mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
GH-45619: [Python] Use f-string instead of string.format (#45629)
### Rationale for this change See https://github.com/apache/arrow/issues/45619. ### What changes are included in this PR? Refactor using f-string instead of `string.format`. But do not use f-string for following case, `string.format` allows passing parameters, making the code more reusable. https://github.com/apache/arrow/blob/0fbf9823542233c5f32c26534c34cc97ce3f0be2/python/pyarrow/parquet/core.py#L1624-L1695 ### Are these changes tested? Via CI. ### Are there any user-facing changes? No. * GitHub Issue: #45619 Lead-authored-by: Chilin Chiou <chilin.chiou@gmail.com> Co-authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
This commit is contained in:
1 parent
c3855b917e
commit
992bee2c83
110 files changed
+753
-948
No files matched your search
@@ -171,7 +171,7 @@ class DarwinSymbolizer(Symbolizer):
|
||||
# foo(type1, type2) (in object.name) (filename.cc:80)
|
||||
match = re.match(r'^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line)
|
||||
if DEBUG:
|
||||
print('atos_line: {0}'.format(atos_line))
|
||||
print(f'atos_line: {atos_line}')
|
||||
if match:
|
||||
function_name = match.group(1)
|
||||
function_name = re.sub(r'\(.*?\)', '', function_name)
|
||||
|
||||
@@ -31,13 +31,12 @@ def process_dir(corpus_dir, zip_output):
|
||||
|
||||
for child in corpus_dir.iterdir():
|
||||
if not child.is_file():
|
||||
raise IOError("Not a file: {0}".format(child))
|
||||
raise IOError(f"Not a file: {child}")
|
||||
with child.open('rb') as f:
|
||||
data = f.read()
|
||||
arcname = hashlib.sha1(data).hexdigest()
|
||||
if arcname in seen:
|
||||
raise ValueError("Duplicate hash: {0} (in file {1})"
|
||||
.format(arcname, child))
|
||||
raise ValueError(f"Duplicate hash: {arcname} (in file {child})")
|
||||
zip_output.writestr(str(arcname), data)
|
||||
seen.add(arcname)
|
||||
|
||||
@@ -49,6 +48,6 @@ def main(corpus_dir, zip_output_name):
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: {0} <corpus dir> <output zip file>".format(sys.argv[0]))
|
||||
print(f"Usage: {sys.argv[0]} <corpus dir> <output zip file>")
|
||||
sys.exit(1)
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
@@ -204,7 +204,7 @@ def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
|
||||
if matches:
|
||||
entries.extend(matches)
|
||||
else:
|
||||
print("{} not in compilation database".format(source))
|
||||
print(f"{source} not in compilation database")
|
||||
# TODO: As long as there is no complete compilation database available this check cannot be performed
|
||||
pass
|
||||
#print('WARNING: \'%s\' not found in compilation database.', source)
|
||||
|
||||
@@ -121,7 +121,7 @@ def lint_files():
|
||||
if __name__ == '__main__':
|
||||
failures = list(lint_files())
|
||||
for path, why, i, line in failures:
|
||||
print('File {0} failed C++/CLI lint check: {1}\n'
|
||||
'Line {2}: {3}'.format(path, why, i + 1, line))
|
||||
print(f'File {path} failed C++/CLI lint check: {why}\n'
|
||||
f'Line {i + 1}: {line}')
|
||||
if failures:
|
||||
exit(1)
|
||||
@@ -38,8 +38,7 @@ def _check_one_file(filename, formatted):
|
||||
original.decode('utf8').splitlines(True),
|
||||
formatted.decode('utf8').splitlines(True),
|
||||
fromfile=filename,
|
||||
tofile="{} (after clang format)".format(
|
||||
filename)))
|
||||
tofile=f"{filename} (after clang format)"))
|
||||
else:
|
||||
diff = None
|
||||
|
||||
@@ -85,8 +84,7 @@ if __name__ == "__main__":
|
||||
|
||||
if arguments.fix:
|
||||
if not arguments.quiet:
|
||||
print("\n".join(map(lambda x: "Formatting {}".format(x),
|
||||
formatted_filenames)))
|
||||
print("\n".join(f"Formatting {x}" for x in formatted_filenames))
|
||||
|
||||
# Break clang-format invocations into chunks: each invocation formats
|
||||
# 16 files. Wait for all processes to complete
|
||||
@@ -122,9 +120,9 @@ if __name__ == "__main__":
|
||||
# check the output from each invocation of clang-format in parallel
|
||||
for filename, diff in pool.starmap(_check_one_file, checker_args):
|
||||
if not arguments.quiet:
|
||||
print("Checking {}".format(filename))
|
||||
print(f"Checking {filename}")
|
||||
if diff:
|
||||
print("{} had clang-format style issues".format(filename))
|
||||
print(f"{filename} had clang-format style issues")
|
||||
# Print out the diff to stderr
|
||||
error = True
|
||||
# pad with a newline
|
||||
|
||||
@@ -78,7 +78,7 @@ if __name__ == "__main__":
|
||||
if arguments.quiet:
|
||||
cmd.append('--quiet')
|
||||
else:
|
||||
print("\n".join(map(lambda x: "Linting {}".format(x),
|
||||
print("\n".join(map(lambda x: f"Linting {x}",
|
||||
linted_filenames)))
|
||||
|
||||
# lint files in chunks: each invocation of cpplint will process 16 files
|
||||
|
||||
@@ -73,7 +73,7 @@ namespace internal {
|
||||
|
||||
|
||||
print("inline const uint8_t* unpack0_64(const uint8_t* in, uint64_t* out) {")
|
||||
print(" for(int k = 0; k < {0} ; k += 1) {{".format(howmany(0)))
|
||||
print(f" for(int k = 0; k < {howmany(0)} ; k += 1) {{")
|
||||
print(" out[k] = 0;")
|
||||
print(" }")
|
||||
print(" return in;")
|
||||
@@ -81,47 +81,44 @@ print("}")
|
||||
|
||||
for bit in range(1, 65):
|
||||
print("")
|
||||
print(
|
||||
"inline const uint8_t* unpack{0}_64(const uint8_t* in, uint64_t* out) {{".format(bit))
|
||||
print(f"inline const uint8_t* unpack{bit}_64(const uint8_t* in, uint64_t* out) {{")
|
||||
|
||||
if(bit < 64):
|
||||
print(" const uint64_t mask = {0}ULL;".format((1 << bit)-1))
|
||||
print(f" const uint64_t mask = {((1 << bit)-1)}ULL;")
|
||||
maskstr = " & mask"
|
||||
if (bit == 64):
|
||||
maskstr = "" # no need
|
||||
|
||||
for k in range(howmanywords(bit)-1):
|
||||
print(" uint64_t w{0} = util::SafeLoadAs<uint64_t>(in);".format(k))
|
||||
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
|
||||
print(" in += 8;".format(k))
|
||||
print(f" uint64_t w{k} = util::SafeLoadAs<uint64_t>(in);")
|
||||
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
|
||||
print(" in += 8;")
|
||||
k = howmanywords(bit) - 1
|
||||
if (bit % 2 == 0):
|
||||
print(" uint64_t w{0} = util::SafeLoadAs<uint64_t>(in);".format(k))
|
||||
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
|
||||
print(" in += 8;".format(k))
|
||||
print(f" uint64_t w{k} = util::SafeLoadAs<uint64_t>(in);")
|
||||
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
|
||||
print(" in += 8;")
|
||||
else:
|
||||
print(" uint64_t w{0} = util::SafeLoadAs<uint32_t>(in);".format(k))
|
||||
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
|
||||
print(" in += 4;".format(k))
|
||||
print(f" uint64_t w{k} = util::SafeLoadAs<uint32_t>(in);")
|
||||
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
|
||||
print(" in += 4;")
|
||||
|
||||
for j in range(howmany(bit)):
|
||||
firstword = j * bit // 64
|
||||
secondword = (j * bit + bit - 1)//64
|
||||
firstshift = (j*bit) % 64
|
||||
firstshiftstr = " >> {0}".format(firstshift)
|
||||
firstshiftstr = f" >> {firstshift}"
|
||||
if(firstshift == 0):
|
||||
firstshiftstr = "" # no need
|
||||
if(firstword == secondword):
|
||||
if(firstshift + bit == 64):
|
||||
print(" out[{0}] = w{1}{2};".format(
|
||||
j, firstword, firstshiftstr, firstshift))
|
||||
print(f" out[{j}] = w{firstword}{firstshiftstr};")
|
||||
else:
|
||||
print(" out[{0}] = (w{1}{2}){3};".format(
|
||||
j, firstword, firstshiftstr, maskstr))
|
||||
print(f" out[{j}] = (w{firstword}{firstshiftstr}){maskstr};")
|
||||
else:
|
||||
secondshift = (64-firstshift)
|
||||
print(" out[{0}] = ((w{1}{2}) | (w{3} << {4})){5};".format(
|
||||
j, firstword, firstshiftstr, firstword+1, secondshift, maskstr))
|
||||
print(f" out[{j}] = ((w{firstword}{firstshiftstr}) | "
|
||||
f"(w{firstword+1} << {secondshift})){maskstr};")
|
||||
print("")
|
||||
print(" return in;")
|
||||
print("}")
|
||||
|
||||
@@ -37,8 +37,8 @@ def apply_template(template, data):
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
raise ValueError("Usage: {0} <template file> <data file> "
|
||||
"<output file>".format(sys.argv[0]))
|
||||
raise ValueError(f"Usage: {sys.argv[0]} <template file> <data file> "
|
||||
"<output file>")
|
||||
with open(sys.argv[1], "rb") as f:
|
||||
template = f.read()
|
||||
with open(sys.argv[2], "rb") as f:
|
||||
|
||||
@@ -23,30 +23,30 @@ DEFAULT_THRESHOLD = 0.05
|
||||
|
||||
def items_per_seconds_fmt(value):
|
||||
if value < 1000:
|
||||
return "{} items/sec".format(value)
|
||||
return f"{value} items/sec"
|
||||
if value < 1000**2:
|
||||
return "{:.3f}K items/sec".format(value / 1000)
|
||||
return f"{value / 1000:.3f}K items/sec"
|
||||
if value < 1000**3:
|
||||
return "{:.3f}M items/sec".format(value / 1000**2)
|
||||
return f"{value / 1000**2:.3f}M items/sec"
|
||||
else:
|
||||
return "{:.3f}G items/sec".format(value / 1000**3)
|
||||
return f"{value / 1000**3:.3f}G items/sec"
|
||||
|
||||
|
||||
def bytes_per_seconds_fmt(value):
|
||||
if value < 1024:
|
||||
return "{} bytes/sec".format(value)
|
||||
return f"{value} bytes/sec"
|
||||
if value < 1024**2:
|
||||
return "{:.3f} KiB/sec".format(value / 1024)
|
||||
return f"{value / 1024:.3f} KiB/sec"
|
||||
if value < 1024**3:
|
||||
return "{:.3f} MiB/sec".format(value / 1024**2)
|
||||
return f"{value / 1024**2:.3f} MiB/sec"
|
||||
if value < 1024**4:
|
||||
return "{:.3f} GiB/sec".format(value / 1024**3)
|
||||
return f"{value / 1024**3:.3f} GiB/sec"
|
||||
else:
|
||||
return "{:.3f} TiB/sec".format(value / 1024**4)
|
||||
return f"{value / 1024**4:.3f} TiB/sec"
|
||||
|
||||
|
||||
def change_fmt(value):
|
||||
return "{:.3%}".format(value)
|
||||
return f"{value:.3%}"
|
||||
|
||||
|
||||
def formatter_for_unit(unit):
|
||||
|
||||
@@ -43,7 +43,7 @@ class Benchmark:
|
||||
return self.median
|
||||
|
||||
def __repr__(self):
|
||||
return "Benchmark[name={},value={}]".format(self.name, self.value)
|
||||
return f"Benchmark[name={self.name},value={self.value}]"
|
||||
|
||||
|
||||
class BenchmarkSuite:
|
||||
@@ -52,6 +52,4 @@ class BenchmarkSuite:
|
||||
self.benchmarks = benchmarks
|
||||
|
||||
def __repr__(self):
|
||||
return "BenchmarkSuite[name={}, benchmarks={}]".format(
|
||||
self.name, self.benchmarks
|
||||
)
|
||||
return f"BenchmarkSuite[name={self.name}, benchmarks={self.benchmarks}]"
|
||||
@@ -45,7 +45,7 @@ class GoogleBenchmarkCommand(Command):
|
||||
def list_benchmarks(self):
|
||||
argv = ["--benchmark_list_tests"]
|
||||
if self.benchmark_filter:
|
||||
argv.append("--benchmark_filter={}".format(self.benchmark_filter))
|
||||
argv.append(f"--benchmark_filter={self.benchmark_filter}")
|
||||
result = self.run(*argv, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
return str.splitlines(result.stdout.decode("utf-8"))
|
||||
@@ -166,7 +166,7 @@ class GoogleBenchmark(Benchmark):
|
||||
counters)
|
||||
|
||||
def __repr__(self):
|
||||
return "GoogleBenchmark[name={},runs={}]".format(self.names, self.runs)
|
||||
return f"GoogleBenchmark[name={self.name},runs={self.runs}]"
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, payload):
|
||||
|
||||
@@ -58,7 +58,7 @@ class JavaMicrobenchmarkHarnessCommand(Command):
|
||||
def list_benchmarks(self):
|
||||
argv = []
|
||||
if self.benchmark_filter:
|
||||
argv.append("-Dbenchmark.filter={}".format(self.benchmark_filter))
|
||||
argv.append(f"-Dbenchmark.filter={self.benchmark_filter}")
|
||||
result = self.build.list(
|
||||
*argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
@@ -77,13 +77,11 @@ class JavaMicrobenchmarkHarnessCommand(Command):
|
||||
|
||||
def results(self, repetitions):
|
||||
with NamedTemporaryFile(suffix=".json") as out:
|
||||
argv = ["-Dbenchmark.runs={}".format(repetitions),
|
||||
"-Dbenchmark.resultfile={}".format(out.name),
|
||||
argv = [f"-Dbenchmark.runs={repetitions}",
|
||||
f"-Dbenchmark.resultfile={out.name}",
|
||||
"-Dbenchmark.resultformat=json"]
|
||||
if self.benchmark_filter:
|
||||
argv.append(
|
||||
"-Dbenchmark.filter={}".format(self.benchmark_filter)
|
||||
)
|
||||
argv.append(f"-Dbenchmark.filter={self.benchmark_filter}")
|
||||
|
||||
self.build.benchmark(*argv, check=True)
|
||||
return json.load(out)
|
||||
@@ -187,8 +185,7 @@ class JavaMicrobenchmarkHarness(Benchmark):
|
||||
counters)
|
||||
|
||||
def __repr__(self):
|
||||
return "JavaMicrobenchmark[name={},runs={}]".format(
|
||||
self.name, self.runs)
|
||||
return f"JavaMicrobenchmark[name={self.name},runs={self.runs}]"
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, payload):
|
||||
|
||||
@@ -69,7 +69,7 @@ class StaticBenchmarkRunner(BenchmarkRunner):
|
||||
def list_benchmarks(self):
|
||||
for suite in self._suites:
|
||||
for benchmark in suite.benchmarks:
|
||||
yield "{}.{}".format(suite.name, benchmark.name)
|
||||
yield f"{suite.name}.{benchmark.name}"
|
||||
|
||||
@property
|
||||
def suites(self):
|
||||
@@ -102,7 +102,7 @@ class StaticBenchmarkRunner(BenchmarkRunner):
|
||||
return BenchmarkRunnerCodec.decode(loaded, **kwargs)
|
||||
|
||||
def __repr__(self):
|
||||
return "BenchmarkRunner[suites={}]".format(list(self.suites))
|
||||
return f"BenchmarkRunner[suites={list(self.suites)}]"
|
||||
|
||||
|
||||
class CppBenchmarkRunner(BenchmarkRunner):
|
||||
@@ -165,7 +165,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
|
||||
for suite_name, suite_bin in self.suites_binaries.items():
|
||||
suite_cmd = GoogleBenchmarkCommand(suite_bin)
|
||||
for benchmark_name in suite_cmd.list_benchmarks():
|
||||
yield "{}.{}".format(suite_name, benchmark_name)
|
||||
yield f"{suite_name}.{benchmark_name}"
|
||||
|
||||
@property
|
||||
def suites(self):
|
||||
@@ -176,7 +176,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
|
||||
suite_and_binaries = self.suites_binaries
|
||||
for suite_name in suite_and_binaries:
|
||||
if not suite_matcher(suite_name):
|
||||
logger.debug("Ignoring suite {}".format(suite_name))
|
||||
logger.debug(f"Ignoring suite {suite_name}")
|
||||
continue
|
||||
|
||||
suite_bin = suite_and_binaries[suite_name]
|
||||
@@ -184,8 +184,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
|
||||
|
||||
# Filter may exclude all benchmarks
|
||||
if not suite:
|
||||
logger.debug("Suite {} executed but no results"
|
||||
.format(suite_name))
|
||||
logger.debug(f"Suite {suite_name} executed but no results")
|
||||
continue
|
||||
|
||||
suite_found = True
|
||||
@@ -274,7 +273,7 @@ class JavaBenchmarkRunner(BenchmarkRunner):
|
||||
suite_cmd = JavaMicrobenchmarkHarnessCommand(self.build)
|
||||
benchmark_names = suite_cmd.list_benchmarks()
|
||||
for benchmark_name in benchmark_names:
|
||||
yield "{}".format(benchmark_name)
|
||||
yield f"{benchmark_name}"
|
||||
|
||||
@property
|
||||
def suites(self):
|
||||
@@ -284,8 +283,7 @@ class JavaBenchmarkRunner(BenchmarkRunner):
|
||||
|
||||
# Filter may exclude all benchmarks
|
||||
if not suite:
|
||||
logger.debug("Suite {} executed but no results"
|
||||
.format(suite_name))
|
||||
logger.debug(f"Suite {suite_name} executed but no results")
|
||||
return
|
||||
|
||||
yield suite
|
||||
|
||||
@@ -230,7 +230,7 @@ class CommentBot:
|
||||
self.github = github.Github(**kwargs)
|
||||
|
||||
def parse_command(self, payload):
|
||||
mention = '@{}'.format(self.name)
|
||||
mention = f'@{self.name}'
|
||||
comment = payload['comment']
|
||||
|
||||
if payload['sender']['login'] == self.name:
|
||||
@@ -261,7 +261,7 @@ class CommentBot:
|
||||
elif event == 'pull_request_review_comment':
|
||||
return self.handle_review_comment(command, payload)
|
||||
else:
|
||||
raise ValueError("Unexpected event type {}".format(event))
|
||||
raise ValueError(f"Unexpected event type {event}")
|
||||
|
||||
def handle_issue_comment(self, command, payload):
|
||||
repo = self.github.get_repo(payload['repository']['id'], lazy=True)
|
||||
@@ -355,7 +355,7 @@ def _clone_arrow_and_crossbow(dest, crossbow_repo, arrow_repo_url, pr_number):
|
||||
git.checkout(local_branch, git_dir=arrow_path)
|
||||
|
||||
# 2. clone crossbow repository
|
||||
crossbow_url = 'https://github.com/{}'.format(crossbow_repo)
|
||||
crossbow_url = f'https://github.com/{crossbow_repo}'
|
||||
git.clone(crossbow_url, str(queue_path))
|
||||
|
||||
# 3. initialize crossbow objects
|
||||
|
||||
@@ -270,7 +270,7 @@ def decorate_lint_command(cmd):
|
||||
Decorate the lint() command function to add individual per-check options.
|
||||
"""
|
||||
for check in lint_checks:
|
||||
option = click.option("--{0}/--no-{0}".format(check.option_name),
|
||||
option = click.option(f"--{check.option_name}/--no-{check.option_name}",
|
||||
default=None, help=check.help)
|
||||
cmd = option(cmd)
|
||||
return cmd
|
||||
@@ -414,7 +414,7 @@ def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras,
|
||||
""" List benchmark suite.
|
||||
"""
|
||||
with tmpdir(preserve=preserve) as root:
|
||||
logger.debug("Running benchmark {}".format(rev_or_path))
|
||||
logger.debug(f"Running benchmark {rev_or_path}")
|
||||
|
||||
if language == "cpp":
|
||||
conf = CppBenchmarkRunner.default_configuration(
|
||||
@@ -495,7 +495,7 @@ def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras,
|
||||
archery benchmark run --output=run.json
|
||||
"""
|
||||
with tmpdir(preserve=preserve) as root:
|
||||
logger.debug("Running benchmark {}".format(rev_or_path))
|
||||
logger.debug(f"Running benchmark {rev_or_path}")
|
||||
|
||||
if language == "cpp":
|
||||
conf = CppBenchmarkRunner.default_configuration(
|
||||
@@ -626,8 +626,7 @@ def benchmark_diff(ctx, src, preserve, output, language, cmake_extras,
|
||||
archery --quiet benchmark diff WORKSPACE run.json > result.json
|
||||
"""
|
||||
with tmpdir(preserve=preserve) as root:
|
||||
logger.debug("Comparing {} (contender) with {} (baseline)"
|
||||
.format(contender, baseline))
|
||||
logger.debug(f"Comparing {contender} (contender) with {baseline} (baseline)")
|
||||
|
||||
if language == "cpp":
|
||||
conf = CppBenchmarkRunner.default_configuration(
|
||||
@@ -704,7 +703,7 @@ def _format_comparisons_with_pandas(comparisons_json, no_counters,
|
||||
def labelled(title, df):
|
||||
if len(df) == 0:
|
||||
return ''
|
||||
title += ': ({})'.format(len(df))
|
||||
title += f': ({len(df)})'
|
||||
df_str = df.to_string(index=False)
|
||||
bar = '-' * df_str.index('\n')
|
||||
return '\n'.join([bar, title, bar, df_str])
|
||||
|
||||
@@ -150,11 +150,10 @@ def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version,
|
||||
queue.put(job, prefix=job_prefix)
|
||||
|
||||
if no_push:
|
||||
click.echo('Branches and commits created but not pushed: `{}`'
|
||||
.format(job.branch))
|
||||
click.echo(f'Branches and commits created but not pushed: `{job.branch}`')
|
||||
else:
|
||||
queue.push()
|
||||
click.echo('Pushed job identifier is: `{}`'.format(job.branch))
|
||||
click.echo(f'Pushed job identifier is: `{job.branch}`')
|
||||
|
||||
|
||||
@crossbow.command()
|
||||
@@ -263,7 +262,7 @@ def render(obj, task, config_path, arrow_version, arrow_remote, arrow_branch,
|
||||
for task_name, rendered_files in job.render_tasks().items():
|
||||
for path, content in _flatten(rendered_files).items():
|
||||
click.echo('#' * 80)
|
||||
click.echo('### {:^72} ###'.format("/".join(path)))
|
||||
click.echo(f"### {'/'.join(path):^72} ###")
|
||||
click.echo('#' * 80)
|
||||
click.echo(highlight(content))
|
||||
|
||||
@@ -534,8 +533,8 @@ def download_artifacts(obj, job_name, target_dir, dry_run, fetch,
|
||||
else:
|
||||
break
|
||||
|
||||
click.echo('Downloading {}\'s artifacts.'.format(job_name))
|
||||
click.echo('Destination directory is {}'.format(target_dir))
|
||||
click.echo(f'Downloading {job_name}\'s artifacts.')
|
||||
click.echo(f'Destination directory is {target_dir}')
|
||||
click.echo()
|
||||
|
||||
report = ConsoleReport(job, task_filters=task_filters)
|
||||
|
||||
@@ -269,8 +269,8 @@ class Repo:
|
||||
def origin(self):
|
||||
remote = self.repo.remotes['origin']
|
||||
if self.require_https and remote.url.startswith('git@github.com'):
|
||||
raise CrossbowError("Change SSH origin URL to HTTPS to use "
|
||||
"Crossbow: {}".format(remote.url))
|
||||
raise CrossbowError(f"Change SSH origin URL to HTTPS to use "
|
||||
f"Crossbow: {remote.url}")
|
||||
return remote
|
||||
|
||||
def fetch(self, retry=3):
|
||||
@@ -299,9 +299,9 @@ class Repo:
|
||||
try:
|
||||
self.origin.push(refs + self._updated_refs, callbacks=callbacks)
|
||||
except pygit2.GitError:
|
||||
raise RuntimeError('Failed to push updated references, '
|
||||
'potentially because of credential issues: {}'
|
||||
.format(self._updated_refs))
|
||||
raise RuntimeError("Failed to push updated references, "
|
||||
"potentially because of credential issues: "
|
||||
f"{self._updated_refs}")
|
||||
else:
|
||||
self.updated_refs = []
|
||||
|
||||
@@ -329,9 +329,9 @@ class Repo:
|
||||
return self.repo.remotes[self.branch.upstream.remote_name]
|
||||
except (AttributeError, KeyError):
|
||||
raise CrossbowError(
|
||||
'Cannot determine git remote for the Arrow repository to '
|
||||
'clone or push to, try to push the `{}` branch first to have '
|
||||
'a remote tracking counterpart.'.format(self.branch.name)
|
||||
"Cannot determine git remote for the Arrow repository to "
|
||||
f"clone or push to, try to push the `{self.branch.name}` "
|
||||
"branch first to have a remote tracking counterpart."
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -422,7 +422,7 @@ class Repo:
|
||||
branch = self.repo.create_branch(branch_name, commit)
|
||||
|
||||
# append to the pushable references
|
||||
self._updated_refs.append('refs/heads/{}'.format(branch_name))
|
||||
self._updated_refs.append(f"refs/heads/{branch_name}")
|
||||
|
||||
return branch
|
||||
|
||||
@@ -438,7 +438,7 @@ class Repo:
|
||||
message)
|
||||
|
||||
# append to the pushable references
|
||||
self._updated_refs.append('refs/tags/{}'.format(tag_name))
|
||||
self._updated_refs.append(f"refs/tags/{tag_name}")
|
||||
|
||||
return self.repo[tag_id]
|
||||
|
||||
@@ -505,12 +505,11 @@ class Repo:
|
||||
result = release.upload_asset(name=name, asset=fp,
|
||||
content_type=mime)
|
||||
except github3.exceptions.ResponseError as e:
|
||||
logger.error('Attempt {} has failed with message: {}.'
|
||||
.format(i + 1, str(e)))
|
||||
logger.error('Error message {}'.format(e.msg))
|
||||
logger.error('List of errors provided by GitHub:')
|
||||
logger.error(f"Attempt {i + 1} has failed with message: {e}.")
|
||||
logger.error(f"Error message {e.msg}")
|
||||
logger.error("List of errors provided by GitHub:")
|
||||
for err in e.errors:
|
||||
logger.error(' - {}'.format(err))
|
||||
logger.error(f" - {err}")
|
||||
|
||||
if e.code == 422:
|
||||
# 422 Validation Failed, probably raised because
|
||||
@@ -518,16 +517,15 @@ class Repo:
|
||||
# reattempting the asset upload
|
||||
for asset in release.assets():
|
||||
if asset.name == name:
|
||||
logger.info('Release asset {} already exists, '
|
||||
'removing it...'.format(name))
|
||||
logger.info(f"Release asset {name} already exists, "
|
||||
"removing it...")
|
||||
asset.delete()
|
||||
logger.info('Asset {} removed.'.format(name))
|
||||
logger.info(f"Asset {name} removed.")
|
||||
break
|
||||
except github3.exceptions.ConnectionError as e:
|
||||
logger.error('Attempt {} has failed with message: {}.'
|
||||
.format(i + 1, str(e)))
|
||||
logger.error(f"Attempt {i + 1} has failed with message: {e}.")
|
||||
else:
|
||||
logger.info('Attempt {} has finished.'.format(i + 1))
|
||||
logger.info(f"Attempt {i + 1} has finished.")
|
||||
return result
|
||||
|
||||
time.sleep(retry_backoff)
|
||||
@@ -536,14 +534,14 @@ class Repo:
|
||||
|
||||
def github_upload_asset_curl(self, release, path, name, mime):
|
||||
upload_url, _ = release.upload_url.split('{?')
|
||||
upload_url += '?name={}'.format(name)
|
||||
upload_url += f"?name={name}"
|
||||
|
||||
command = [
|
||||
'curl',
|
||||
'--fail',
|
||||
'-H', "Authorization: token {}".format(self.github_token),
|
||||
'-H', "Content-Type: {}".format(mime),
|
||||
'--data-binary', '@{}'.format(path),
|
||||
'-H', f"Authorization: token {self.github_token}",
|
||||
'-H', f"Content-Type: {mime}",
|
||||
'--data-binary', f'@{path}',
|
||||
upload_url
|
||||
]
|
||||
return subprocess.run(command, shell=False, check=True)
|
||||
@@ -576,8 +574,8 @@ class Repo:
|
||||
mime = mimetypes.guess_type(name)[0] or 'application/zip'
|
||||
|
||||
logger.info(
|
||||
'Uploading asset `{}` with mimetype {} and size {}...'
|
||||
.format(name, mime, size)
|
||||
f"Uploading asset `{name}` with mimetype {mime} and size "
|
||||
f"{size}..."
|
||||
)
|
||||
|
||||
if method == 'requests':
|
||||
@@ -588,7 +586,7 @@ class Repo:
|
||||
mime=mime)
|
||||
else:
|
||||
raise CrossbowError(
|
||||
'Unsupported upload method {}'.format(method)
|
||||
f"Unsupported upload method {method}"
|
||||
)
|
||||
|
||||
def github_pr(self, title, head=None, base=None, body=None,
|
||||
@@ -645,12 +643,12 @@ class Queue(Repo):
|
||||
def _next_job_id(self, prefix):
|
||||
"""Auto increments the branch's identifier based on the prefix"""
|
||||
latest_id = self._latest_prefix_id(prefix)
|
||||
return '{}-{}'.format(prefix, latest_id + 1)
|
||||
return f"{prefix}-{latest_id + 1}"
|
||||
|
||||
def _new_hex_id(self, prefix):
|
||||
"""Append a new id to branch's identifier based on the prefix"""
|
||||
hex_id = uuid.uuid4().hex[:10]
|
||||
return '{}-{}'.format(prefix, hex_id)
|
||||
return f"{prefix}-{hex_id}"
|
||||
|
||||
def latest_for_prefix(self, prefix):
|
||||
prefix_date = self._prefix_contains_date(prefix)
|
||||
@@ -667,13 +665,13 @@ class Queue(Repo):
|
||||
raise RuntimeError(
|
||||
f"No job has been submitted with prefix '{prefix}' yet"
|
||||
)
|
||||
job_name = '{}-{}'.format(prefix, latest_id)
|
||||
job_name = f"{prefix}-{latest_id}"
|
||||
return self.get(job_name)
|
||||
|
||||
def date_of(self, job):
|
||||
# it'd be better to bound to the queue repository on deserialization
|
||||
# and reorganize these methods to Job
|
||||
branch_name = 'origin/{}'.format(job.branch)
|
||||
branch_name = f'origin/{job.branch}'
|
||||
branch = self.repo.branches[branch_name]
|
||||
commit = self.repo[branch.target]
|
||||
return date.fromtimestamp(commit.commit_time)
|
||||
@@ -691,13 +689,13 @@ class Queue(Repo):
|
||||
yield self.get(name)
|
||||
|
||||
def get(self, job_name):
|
||||
branch_name = 'origin/{}'.format(job_name)
|
||||
branch_name = f"origin/{job_name}"
|
||||
branch = self.repo.branches[branch_name]
|
||||
try:
|
||||
content = self.file_contents(branch.target, 'job.yml')
|
||||
except KeyError:
|
||||
raise CrossbowError(
|
||||
'No job is found with name: {}'.format(job_name)
|
||||
f"No job is found with name: {job_name}"
|
||||
)
|
||||
|
||||
buffer = StringIO(content.decode('utf-8'))
|
||||
@@ -724,7 +722,7 @@ class Queue(Repo):
|
||||
for task_name, task in job.tasks.items():
|
||||
# adding CI's name to the end of the branch in order to use skip
|
||||
# patterns on travis and circleci
|
||||
task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
|
||||
task.branch = f"{job.branch}-{task.ci}-{task_name}"
|
||||
params = {
|
||||
**job.params,
|
||||
"arrow": job.target,
|
||||
@@ -772,14 +770,14 @@ def get_version(root, **kwargs):
|
||||
if 'dev' not in tag:
|
||||
major += 1
|
||||
|
||||
return "{}.{}.{}.dev{}".format(major, minor, patch, version.distance or 0)
|
||||
return f"{major}.{minor}.{patch}.dev{version.distance or 0}"
|
||||
|
||||
|
||||
class Serializable:
|
||||
|
||||
@classmethod
|
||||
def to_yaml(cls, representer, data):
|
||||
tag = '!{}'.format(cls.__name__)
|
||||
tag = f"!{cls.__name__}"
|
||||
dct = {k: v for k, v in data.__dict__.items() if not k.startswith('_')}
|
||||
return representer.represent_mapping(tag, dct)
|
||||
|
||||
@@ -943,9 +941,8 @@ class Task(Serializable):
|
||||
params=params)
|
||||
except jinja2.TemplateError as e:
|
||||
raise RuntimeError(
|
||||
'Failed to render template `{}` with {}: {}'.format(
|
||||
self.template, e.__class__.__name__, str(e)
|
||||
)
|
||||
f"Failed to render template `{self.template}` with "
|
||||
f"{e.__class__.__name__}: {e}"
|
||||
)
|
||||
|
||||
tree = {**_default_tree, self.filename: rendered}
|
||||
@@ -1092,8 +1089,8 @@ class TaskAssets(dict):
|
||||
self[pattern] = github_assets[matches[0].group(0)]
|
||||
else:
|
||||
raise CrossbowError(
|
||||
'Only a single asset should match pattern `{}`, there are '
|
||||
'multiple ones: {}'.format(pattern, ', '.join(matches))
|
||||
f"Only a single asset should match pattern `{pattern}`, "
|
||||
f"there are multiple ones: {', '.join(matches)}"
|
||||
)
|
||||
|
||||
def missing_patterns(self):
|
||||
@@ -1239,12 +1236,12 @@ class Job(Serializable):
|
||||
|
||||
waited_for_minutes = (time.time() - started_at) / 60
|
||||
if waited_for_minutes > poll_max_minutes:
|
||||
msg = ('Exceeded the maximum amount of time waiting for job '
|
||||
'to finish, waited for {} minutes.')
|
||||
raise RuntimeError(msg.format(waited_for_minutes))
|
||||
msg = (f'Exceeded the maximum amount of time waiting for job '
|
||||
f'to finish, waited for {waited_for_minutes} minutes.')
|
||||
raise RuntimeError(msg)
|
||||
|
||||
logger.info('Waiting {} minutes and then checking again'
|
||||
.format(poll_interval_minutes))
|
||||
logger.info(f'Waiting {poll_interval_minutes} minutes and then '
|
||||
'checking again')
|
||||
time.sleep(poll_interval_minutes * 60)
|
||||
|
||||
|
||||
@@ -1278,9 +1275,8 @@ class Config(dict):
|
||||
requested_groups = set(group_allowlist)
|
||||
invalid_groups = requested_groups - valid_groups
|
||||
if invalid_groups:
|
||||
msg = 'Invalid group(s) {!r}. Must be one of {!r}'.format(
|
||||
invalid_groups, valid_groups
|
||||
)
|
||||
msg = (f"Invalid group(s) {invalid_groups!r}. Must be one of "
|
||||
f"{valid_groups!r}")
|
||||
raise CrossbowError(msg)
|
||||
|
||||
# treat the task names as glob patterns to select tasks more easily
|
||||
@@ -1291,7 +1287,7 @@ class Config(dict):
|
||||
requested_tasks.update(matches)
|
||||
else:
|
||||
raise CrossbowError(
|
||||
"Unable to match any tasks for `{}`".format(pattern)
|
||||
f"Unable to match any tasks for `{pattern}`"
|
||||
)
|
||||
|
||||
requested_group_tasks = set()
|
||||
@@ -1309,7 +1305,7 @@ class Config(dict):
|
||||
requested_group_tasks.update(matches)
|
||||
else:
|
||||
raise CrossbowError(
|
||||
"Unable to match any tasks for `{}`".format(pattern)
|
||||
f"Unable to match any tasks for `{pattern}`"
|
||||
)
|
||||
|
||||
# remove any tasks that are negated with ~task-name
|
||||
@@ -1320,7 +1316,7 @@ class Config(dict):
|
||||
matches)
|
||||
else:
|
||||
raise CrossbowError(
|
||||
"Unable to match any tasks for `{}`".format(pattern)
|
||||
f"Unable to match any tasks for `{pattern}`"
|
||||
)
|
||||
|
||||
requested_tasks = requested_tasks.union(requested_group_tasks)
|
||||
@@ -1328,9 +1324,7 @@ class Config(dict):
|
||||
# validate that the passed and matched tasks are defined in the config
|
||||
invalid_tasks = requested_tasks - valid_tasks
|
||||
if invalid_tasks:
|
||||
msg = 'Invalid task(s) {!r}. Must be one of {!r}'.format(
|
||||
invalid_tasks, valid_tasks
|
||||
)
|
||||
msg = f'Invalid task(s) {invalid_tasks!r}. Must be one of {valid_tasks!r}'
|
||||
raise CrossbowError(msg)
|
||||
|
||||
return {
|
||||
@@ -1346,9 +1340,9 @@ class Config(dict):
|
||||
tasks = self.select(tasks=[pattern])
|
||||
if not tasks:
|
||||
raise CrossbowError(
|
||||
"The pattern `{}` defined for task group `{}` is not "
|
||||
"matching any of the tasks defined in the "
|
||||
"configuration file.".format(pattern, group_name)
|
||||
f"The pattern `{pattern}` defined for task group "
|
||||
f"`{group_name}` is not matching any of the tasks "
|
||||
"defined in the configuration file."
|
||||
)
|
||||
|
||||
# validate that the tasks are constructible
|
||||
@@ -1357,9 +1351,9 @@ class Config(dict):
|
||||
Task(task_name, **task)
|
||||
except Exception as e:
|
||||
raise CrossbowError(
|
||||
'Unable to construct a task object from the '
|
||||
'definition of task `{}`. The original error message '
|
||||
'is: `{}`'.format(task_name, str(e))
|
||||
"Unable to construct a task object from the "
|
||||
f"definition of task `{task_name}`. The original error "
|
||||
f"message is: `{e}`"
|
||||
)
|
||||
|
||||
# Get the default branch name from the repository
|
||||
@@ -1393,8 +1387,8 @@ class Config(dict):
|
||||
)
|
||||
)
|
||||
if not files:
|
||||
raise CrossbowError('No files have been rendered for task `{}`'
|
||||
.format(task_name))
|
||||
raise CrossbowError("No files have been rendered for task "
|
||||
f"`{task_name}`")
|
||||
|
||||
|
||||
# configure yaml serializer
|
||||
|
||||
@@ -62,10 +62,10 @@ class Report:
|
||||
return url[:-4] if url.endswith('.git') else url
|
||||
|
||||
def url(self, query):
|
||||
return '{}/branches/all?query={}'.format(self.repo_url, query)
|
||||
return f'{self.repo_url}/branches/all?query={query}'
|
||||
|
||||
def branch_url(self, branch):
|
||||
return '{}/tree/{}'.format(self.repo_url, branch)
|
||||
return f'{self.repo_url}/tree/{branch}'
|
||||
|
||||
def task_url(self, task):
|
||||
build_links = task.status().build_links
|
||||
@@ -155,7 +155,7 @@ class ConsoleReport(Report):
|
||||
line = self.HEADER.format(
|
||||
state=state.upper(),
|
||||
branch=branch,
|
||||
content='uploaded {} / {}'.format(n_uploaded, n_expected)
|
||||
content=f'uploaded {n_uploaded} / {n_expected}'
|
||||
)
|
||||
return click.style(line, fg=self.COLORS[state.lower()])
|
||||
|
||||
@@ -166,7 +166,7 @@ class ConsoleReport(Report):
|
||||
content='Artifacts'
|
||||
)
|
||||
delimiter = '-' * len(header)
|
||||
return '{}\n{}'.format(header, delimiter)
|
||||
return f"{header}\n{delimiter}"
|
||||
|
||||
def artifact(self, state, pattern, asset):
|
||||
if asset is None:
|
||||
@@ -324,9 +324,9 @@ class CommentReport(Report):
|
||||
url = 'https://github.com/{repo}/branches/all?query={branch}'
|
||||
sha = self.job.target.head
|
||||
|
||||
msg = 'Revision: {}\n\n'.format(sha)
|
||||
msg = f'Revision: {sha}\n\n'
|
||||
msg += 'Submitted crossbow builds: [{repo} @ {branch}]'
|
||||
msg += '({})\n'.format(url)
|
||||
msg += f'({url})\n'
|
||||
msg += '\n|Task|Status|\n|----|------|'
|
||||
|
||||
tasks = sorted(self.job.tasks.items(), key=operator.itemgetter(0))
|
||||
@@ -342,8 +342,8 @@ class CommentReport(Report):
|
||||
url=self.task_url(task)
|
||||
)
|
||||
except KeyError:
|
||||
badge = 'unsupported CI service `{}`'.format(task.ci)
|
||||
badge = f'unsupported CI service `{task.ci}`'
|
||||
|
||||
msg += '\n|{}|{}|'.format(key, badge)
|
||||
msg += f'\n|{key}|{badge}|'
|
||||
|
||||
return msg.format(repo=self.crossbow_repo, branch=self.job.branch)
|
||||
@@ -76,7 +76,7 @@ def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli,
|
||||
if not config_path.exists():
|
||||
raise click.ClickException(
|
||||
"Docker compose configuration cannot be found in directory {}, "
|
||||
"try to pass the arrow source directory explicitly.".format(src)
|
||||
f"try to pass the arrow source directory explicitly. {src}"
|
||||
)
|
||||
|
||||
# take the Docker Compose parameters like PYTHON, PANDAS, UBUNTU from the
|
||||
@@ -124,7 +124,7 @@ def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
|
||||
except UndefinedImage as e:
|
||||
raise click.ClickException(
|
||||
"There is no service/image defined in docker-compose.yml with "
|
||||
"name: {}".format(str(e))
|
||||
f"name: {e}"
|
||||
)
|
||||
except RuntimeError as e:
|
||||
raise click.ClickException(str(e))
|
||||
@@ -157,7 +157,7 @@ def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
|
||||
except UndefinedImage as e:
|
||||
raise click.ClickException(
|
||||
"There is no service/image defined in docker-compose.yml with "
|
||||
"name: {}".format(str(e))
|
||||
f"name: {e}"
|
||||
)
|
||||
except RuntimeError as e:
|
||||
raise click.ClickException(str(e))
|
||||
@@ -251,7 +251,7 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
|
||||
except UndefinedImage as e:
|
||||
raise click.ClickException(
|
||||
"There is no service/image defined in docker-compose.yml with "
|
||||
"name: {}".format(str(e))
|
||||
f"name: {e}"
|
||||
)
|
||||
except RuntimeError as e:
|
||||
raise click.ClickException(str(e))
|
||||
|
||||
@@ -117,18 +117,18 @@ class ComposeConfig:
|
||||
for name in self.with_gpus:
|
||||
if name not in services:
|
||||
errors.append(
|
||||
'Service `{}` defined in `x-with-gpus` bot not in '
|
||||
'`services`'.format(name)
|
||||
f'Service `{name}` defined in `x-with-gpus` bot not in '
|
||||
'`services`'
|
||||
)
|
||||
for name in nodes - services:
|
||||
errors.append(
|
||||
'Service `{}` is defined in `x-hierarchy` bot not in '
|
||||
'`services`'.format(name)
|
||||
f'Service `{name}` is defined in `x-hierarchy` bot not in '
|
||||
'`services`'
|
||||
)
|
||||
for name in services - nodes:
|
||||
errors.append(
|
||||
'Service `{}` is defined in `services` but not in '
|
||||
'`x-hierarchy`'.format(name)
|
||||
f'Service `{name}` is defined in `services` but not in '
|
||||
'`x-hierarchy`'
|
||||
)
|
||||
|
||||
# trigger Docker Compose's own validation
|
||||
@@ -147,9 +147,9 @@ class ComposeConfig:
|
||||
errors += result.stderr.decode().splitlines()
|
||||
|
||||
if errors:
|
||||
msg = '\n'.join([' - {}'.format(msg) for msg in errors])
|
||||
msg = '\n'.join([f' - {msg}' for msg in errors])
|
||||
raise ValueError(
|
||||
'Found errors with docker-compose:\n{}'.format(msg)
|
||||
f'Found errors with docker-compose:\n{msg}'
|
||||
)
|
||||
|
||||
rendered_config = StringIO(result.stdout.decode())
|
||||
@@ -230,9 +230,7 @@ class DockerCompose(Command):
|
||||
result.check_returncode()
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise RuntimeError(
|
||||
"{} exited with non-zero exit code {}".format(
|
||||
' '.join(e.cmd), e.returncode
|
||||
)
|
||||
f'{e.cmd} exited with non-zero exit code {e.returncode}'
|
||||
)
|
||||
|
||||
def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True):
|
||||
@@ -294,14 +292,12 @@ class DockerCompose(Command):
|
||||
|
||||
if self.config.using_buildx:
|
||||
for k, v in service['build'].get('args', {}).items():
|
||||
args.extend(['--build-arg', '{}={}'.format(k, v)])
|
||||
args.extend(['--build-arg', f'{k}={v}'])
|
||||
|
||||
if use_cache:
|
||||
cache_ref = '{}-cache'.format(service['image'])
|
||||
cache_from = 'type=registry,ref={}'.format(cache_ref)
|
||||
cache_to = (
|
||||
'type=registry,ref={},mode=max'.format(cache_ref)
|
||||
)
|
||||
cache_ref = f'{service["image"]}-cache'
|
||||
cache_from = f'type=registry,ref={cache_ref}'
|
||||
cache_to = f'type=registry,ref={cache_ref},mode=max'
|
||||
args.extend([
|
||||
'--cache-from', cache_from,
|
||||
'--cache-to', cache_to,
|
||||
@@ -319,9 +315,9 @@ class DockerCompose(Command):
|
||||
if self.config.debug and os.name != "nt":
|
||||
args.append("--progress=plain")
|
||||
for k, v in service['build'].get('args', {}).items():
|
||||
args.extend(['--build-arg', '{}={}'.format(k, v)])
|
||||
args.extend(['--build-arg', f'{k}={v}'])
|
||||
for img in cache_from:
|
||||
args.append('--cache-from="{}"'.format(img))
|
||||
args.append(f'--cache-from="{img}"')
|
||||
args.extend([
|
||||
'-f', arrow_path(service['build']['dockerfile']),
|
||||
'-t', service['image'],
|
||||
@@ -358,13 +354,13 @@ class DockerCompose(Command):
|
||||
# append env variables from the compose conf
|
||||
for k, v in service.get('environment', {}).items():
|
||||
if v is not None:
|
||||
args.extend(['-e', '{}={}'.format(k, v)])
|
||||
args.extend(['-e', f'{k}={v}'])
|
||||
|
||||
# append volumes from the compose conf
|
||||
for v in service.get('volumes', []):
|
||||
if not isinstance(v, str):
|
||||
# if not the compact string volume definition
|
||||
v = "{}:{}".format(v['source'], v['target'])
|
||||
v = f'{v["source"]}:{v["target"]}'
|
||||
args.extend(['-v', v])
|
||||
|
||||
# append capabilities from the compose conf
|
||||
@@ -393,7 +389,7 @@ class DockerCompose(Command):
|
||||
|
||||
if env is not None:
|
||||
for k, v in env.items():
|
||||
args.extend(['-e', '{}={}'.format(k, v)])
|
||||
args.extend(['-e', f'{k}={v}'])
|
||||
|
||||
if volumes is not None:
|
||||
for volume in volumes:
|
||||
@@ -443,8 +439,7 @@ class DockerCompose(Command):
|
||||
self._execute_docker('login', '-u', user, '-p', password)
|
||||
except subprocess.CalledProcessError:
|
||||
# hide credentials
|
||||
msg = ('Failed to push `{}`, check the passed credentials'
|
||||
.format(service_name))
|
||||
msg = f'Failed to push `{service_name}`, check the passed credentials'
|
||||
raise RuntimeError(msg) from None
|
||||
|
||||
service = self.config.get(service_name)
|
||||
|
||||
@@ -199,7 +199,7 @@ def create_config(directory, yml_content, env_content=None):
|
||||
if env_content is not None:
|
||||
with env_path.open('w') as fp:
|
||||
for k, v in env_content.items():
|
||||
fp.write("{}={}\n".format(k, v))
|
||||
fp.write(f"{k}={v}\n")
|
||||
|
||||
return config_path
|
||||
|
||||
|
||||
@@ -1592,8 +1592,7 @@ def generate_null_trivial_case(batch_sizes):
|
||||
|
||||
def generate_decimal32_case():
|
||||
fields = [
|
||||
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
|
||||
bit_width=32)
|
||||
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=32)
|
||||
for i, precision in enumerate(range(3, 10))
|
||||
]
|
||||
|
||||
@@ -1603,8 +1602,7 @@ def generate_decimal32_case():
|
||||
|
||||
def generate_decimal64_case():
|
||||
fields = [
|
||||
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
|
||||
bit_width=64)
|
||||
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=64)
|
||||
for i, precision in enumerate(range(3, 19))
|
||||
]
|
||||
|
||||
@@ -1614,8 +1612,7 @@ def generate_decimal64_case():
|
||||
|
||||
def generate_decimal128_case():
|
||||
fields = [
|
||||
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
|
||||
bit_width=128)
|
||||
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=128)
|
||||
for i, precision in enumerate(range(3, 39))
|
||||
]
|
||||
|
||||
@@ -1628,8 +1625,7 @@ def generate_decimal128_case():
|
||||
|
||||
def generate_decimal256_case():
|
||||
fields = [
|
||||
DecimalField(name='f{}'.format(i), precision=precision, scale=5,
|
||||
bit_width=256)
|
||||
DecimalField(name=f'f{i}', precision=precision, scale=5, bit_width=256)
|
||||
for i, precision in enumerate(range(37, 70))
|
||||
]
|
||||
|
||||
|
||||
@@ -77,8 +77,7 @@ class IntegrationRunner(object):
|
||||
self.match = match
|
||||
|
||||
if self.match is not None:
|
||||
print("-- Only running tests with {} in their name"
|
||||
.format(self.match))
|
||||
print(f"-- Only running tests with {self.match} in their name")
|
||||
self.json_files = [json_file for json_file in self.json_files
|
||||
if self.match in json_file.name]
|
||||
|
||||
@@ -115,7 +114,7 @@ class IntegrationRunner(object):
|
||||
with group(f"Integration: Test: IPC: Gold: {consumer.name}"):
|
||||
log('\n')
|
||||
log('******************************************************')
|
||||
log('Tests against golden files in {}'.format(gold_dir))
|
||||
log(f'Tests against golden files in {gold_dir}')
|
||||
log('******************************************************')
|
||||
|
||||
def run_gold(_, consumer, test_case: datagen.File):
|
||||
@@ -280,8 +279,7 @@ class IntegrationRunner(object):
|
||||
"""
|
||||
with group(f"Integration: Test: IPC: {producer.name} -> {consumer.name}"):
|
||||
log('##########################################################')
|
||||
log('IPC: {0} producing, {1} consuming'
|
||||
.format(producer.name, consumer.name))
|
||||
log(f'IPC: {producer.name} producing, {consumer.name} consuming')
|
||||
log('##########################################################')
|
||||
|
||||
case_runner = partial(self._run_ipc_test_case,
|
||||
@@ -302,7 +300,7 @@ class IntegrationRunner(object):
|
||||
|
||||
json_path = test_case.path
|
||||
log('=' * 70)
|
||||
log('Testing file {0}'.format(json_path))
|
||||
log(f'Testing file {json_path}')
|
||||
|
||||
if test_case.should_skip(producer.name, SKIP_IPC):
|
||||
log(f'-- Skipping test because producer {producer.name} does '
|
||||
@@ -405,8 +403,7 @@ class IntegrationRunner(object):
|
||||
):
|
||||
with group(f"Integration: Test: Flight: {producer.name} -> {consumer.name}"):
|
||||
log('##########################################################')
|
||||
log('Flight: {0} serving, {1} requesting'
|
||||
.format(producer.name, consumer.name))
|
||||
log(f'Flight: {producer.name} serving, {consumer.name} requesting')
|
||||
log('##########################################################')
|
||||
|
||||
case_runner = partial(self._run_flight_test_case, producer, consumer)
|
||||
@@ -423,7 +420,7 @@ class IntegrationRunner(object):
|
||||
outcome = Outcome()
|
||||
|
||||
log('=' * 70)
|
||||
log('Testing file {0}'.format(test_case.name))
|
||||
log(f'Testing file {test_case.name}')
|
||||
|
||||
if test_case.should_skip(producer.name, SKIP_FLIGHT):
|
||||
log(f'-- Skipping test because producer {producer.name} does '
|
||||
|
||||
@@ -120,9 +120,7 @@ class CppTester(Tester):
|
||||
out, err = server.communicate()
|
||||
raise RuntimeError(
|
||||
"Flight-C++ server did not start properly, "
|
||||
"stdout:\n{}\n\nstderr:\n{}\n".format(
|
||||
output + out.decode(), err.decode()
|
||||
)
|
||||
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
|
||||
)
|
||||
port = int(output.split(":")[1])
|
||||
yield port
|
||||
|
||||
@@ -233,10 +233,8 @@ class CSharpTester(Tester):
|
||||
server.kill()
|
||||
out, err = server.communicate()
|
||||
raise RuntimeError(
|
||||
'.NET Flight server did not start properly, '
|
||||
'stdout: \n{}\n\nstderr:\n{}\n'.format(
|
||||
output + out.decode(), err.decode()
|
||||
)
|
||||
".NET Flight server did not start properly, "
|
||||
f"stdout: \n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
|
||||
)
|
||||
port = int(output.split(':')[-1])
|
||||
yield port
|
||||
|
||||
@@ -106,10 +106,8 @@ class GoTester(Tester):
|
||||
server.kill()
|
||||
out, err = server.communicate()
|
||||
raise RuntimeError(
|
||||
'Flight-Go server did not start properly, '
|
||||
'stdout: \n{}\n\nstderr:\n{}\n'.format(
|
||||
output + out.decode(), err.decode()
|
||||
)
|
||||
"Flight-Go server did not start properly, "
|
||||
f"stdout: \n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
|
||||
)
|
||||
port = int(output.split(':')[1])
|
||||
yield port
|
||||
|
||||
@@ -339,10 +339,8 @@ class JavaTester(Tester):
|
||||
server.kill()
|
||||
out, err = server.communicate()
|
||||
raise RuntimeError(
|
||||
'Flight-Java server did not start properly, '
|
||||
'stdout:\n{}\n\nstderr:\n{}\n'.format(
|
||||
output + out.decode(), err.decode()
|
||||
)
|
||||
"Flight-Java server did not start properly, "
|
||||
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
|
||||
)
|
||||
port = int(output.split(':')[1])
|
||||
yield port
|
||||
|
||||
@@ -105,10 +105,8 @@ class RustTester(Tester):
|
||||
server.kill()
|
||||
out, err = server.communicate()
|
||||
raise RuntimeError(
|
||||
'Flight-Rust server did not start properly, '
|
||||
'stdout:\n{}\n\nstderr:\n{}\n'.format(
|
||||
output + out.decode(), err.decode()
|
||||
)
|
||||
"Flight-Rust server did not start properly, "
|
||||
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
|
||||
)
|
||||
port = int(output.split(':')[1])
|
||||
yield port
|
||||
|
||||
@@ -164,7 +164,7 @@ class CppConfiguration:
|
||||
return self._cc
|
||||
|
||||
if self.with_fuzzing:
|
||||
return "clang-{}".format(LLVM_VERSION)
|
||||
return f"clang-{LLVM_VERSION}"
|
||||
|
||||
return None
|
||||
|
||||
@@ -174,7 +174,7 @@ class CppConfiguration:
|
||||
return self._cxx
|
||||
|
||||
if self.with_fuzzing:
|
||||
return "clang++-{}".format(LLVM_VERSION)
|
||||
return f"clang++-{LLVM_VERSION}"
|
||||
|
||||
return None
|
||||
|
||||
@@ -277,7 +277,7 @@ class CppConfiguration:
|
||||
@property
|
||||
def definitions(self):
|
||||
extras = list(self.cmake_extras) if self.cmake_extras else []
|
||||
definitions = ["-D{}={}".format(d[0], d[1]) for d in self._gen_defs()]
|
||||
definitions = [f"-D{d[0]}={d[1]}" for d in self._gen_defs()]
|
||||
return definitions + extras
|
||||
|
||||
@property
|
||||
|
||||
@@ -249,7 +249,7 @@ class NumpyDoc:
|
||||
try:
|
||||
obj = Docstring._load_obj(symbol)
|
||||
except (ImportError, AttributeError):
|
||||
print('{} is not available for import'.format(symbol))
|
||||
print(f'{symbol} is not available for import')
|
||||
else:
|
||||
self.traverse(callback, obj, from_package=from_package)
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ def release_changelog_regenerate(obj):
|
||||
continue
|
||||
release = Release(version, repo=repo,
|
||||
issue_tracker=issue_tracker)
|
||||
click.echo('Querying changelog for version: {}'.format(version))
|
||||
click.echo(f'Querying changelog for version: {version}')
|
||||
changelogs.append(release.changelog())
|
||||
|
||||
click.echo('Rendering new CHANGELOG.md file...')
|
||||
@@ -139,4 +139,4 @@ def release_cherry_pick(obj, version, dry_run, recreate):
|
||||
else:
|
||||
click.echo(f'git checkout -b {release.branch} {release.base_branch}')
|
||||
for commit in release.commits_to_pick():
|
||||
click.echo('git cherry-pick {}'.format(commit.hexsha))
|
||||
click.echo(f'git cherry-pick {commit.hexsha}')
|
||||
@@ -183,7 +183,7 @@ class CommitTitle:
|
||||
matches = _TITLE_REGEX.match(headline)
|
||||
if matches is None:
|
||||
warnings.warn(
|
||||
"Unable to parse commit message `{}`".format(headline)
|
||||
f"Unable to parse commit message `{headline}`"
|
||||
)
|
||||
return CommitTitle(headline)
|
||||
|
||||
@@ -203,10 +203,10 @@ class CommitTitle:
|
||||
def to_string(self, with_issue=True, with_components=True):
|
||||
out = ""
|
||||
if with_issue and self.issue:
|
||||
out += "{}: ".format(self.issue)
|
||||
out += f"{self.issue}: "
|
||||
if with_components and self.components:
|
||||
for component in self.components:
|
||||
out += "[{}]".format(component)
|
||||
out += f"[{component}]"
|
||||
out += " "
|
||||
out += self.summary
|
||||
return out
|
||||
@@ -225,13 +225,12 @@ class Commit:
|
||||
return getattr(self._wrapped, attr)
|
||||
|
||||
def __repr__(self):
|
||||
template = '<Commit sha={!r} issue={!r} components={!r} summary={!r}>'
|
||||
return template.format(self.hexsha, self.issue, self.components,
|
||||
self.summary)
|
||||
return (f'<Commit sha={self.hexsha!r} issue={self.issue!r} '
|
||||
f'components={self.components!r} summary={self.summary!r}>')
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return 'https://github.com/apache/arrow/commit/{}'.format(self.hexsha)
|
||||
return f'https://github.com/apache/arrow/commit/{self.hexsha}'
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
|
||||
@@ -50,7 +50,7 @@ def set_env_vars():
|
||||
|
||||
|
||||
def github_url(path):
|
||||
return 'https://api.github.com:443/{}'.format(path.strip('/'))
|
||||
return f'https://api.github.com:443/{path.strip("/")}'
|
||||
|
||||
|
||||
@group()
|
||||
|
||||
@@ -70,7 +70,7 @@ class Cache:
|
||||
"""
|
||||
def download(path):
|
||||
""" Tiny wrapper that download a file and save as key. """
|
||||
logger.debug("Downloading {} as {}".format(url, path))
|
||||
logger.debug(f"Downloading {url} as {path}")
|
||||
conn = urlopen(url)
|
||||
# Ensure the download is completed before writing to disks.
|
||||
content = conn.read()
|
||||
|
||||
@@ -79,7 +79,7 @@ class CMakeDefinition:
|
||||
def arguments(self):
|
||||
"""" Return the arguments to cmake invocation. """
|
||||
arguments = [
|
||||
"-G{}".format(self.generator),
|
||||
f"-G{self.generator}",
|
||||
] + self.definitions + [
|
||||
self.source
|
||||
]
|
||||
@@ -99,13 +99,9 @@ class CMakeDefinition:
|
||||
if os.path.exists(build_dir):
|
||||
# Extra safety to ensure we're deleting a build folder.
|
||||
if not CMakeBuild.is_build_dir(build_dir):
|
||||
raise FileExistsError(
|
||||
"{} is not a cmake build".format(build_dir)
|
||||
)
|
||||
raise FileExistsError(f"{build_dir} is not a cmake build")
|
||||
if not force:
|
||||
raise FileExistsError(
|
||||
"{} exists use force=True".format(build_dir)
|
||||
)
|
||||
raise FileExistsError(f"{build_dir} exists use force=True")
|
||||
rmtree(build_dir)
|
||||
|
||||
os.mkdir(build_dir)
|
||||
@@ -116,7 +112,7 @@ class CMakeDefinition:
|
||||
**kwargs)
|
||||
|
||||
def __repr__(self):
|
||||
return "CMakeDefinition[source={}]".format(self.source)
|
||||
return f"CMakeDefinition[source={self.source}]"
|
||||
|
||||
|
||||
CMAKE_BUILD_TYPE_RE = re.compile("CMAKE_BUILD_TYPE:STRING=([a-zA-Z]+)")
|
||||
@@ -194,7 +190,7 @@ class CMakeBuild(CMake):
|
||||
be lost. Only build_type is recovered.
|
||||
"""
|
||||
if not CMakeBuild.is_build_dir(path):
|
||||
raise ValueError("Not a valid CMakeBuild path: {}".format(path))
|
||||
raise ValueError(f"Not a valid CMakeBuild path: {path}")
|
||||
|
||||
build_type = None
|
||||
# Infer build_type by looking at CMakeCache.txt and looking for a magic
|
||||
@@ -207,9 +203,6 @@ class CMakeBuild(CMake):
|
||||
return CMakeBuild(path, build_type)
|
||||
|
||||
def __repr__(self):
|
||||
return ("CMakeBuild["
|
||||
"build = {},"
|
||||
"build_type = {},"
|
||||
"definition = {}]".format(self.build_dir,
|
||||
self.build_type,
|
||||
self.definition))
|
||||
return (f"CMakeBuild[build = {self.build_dir},"
|
||||
f"build_type = {self.build_type},"
|
||||
f"definition = {self.definition}]")
|
||||
@@ -25,7 +25,7 @@ from .logger import logger, ctx
|
||||
|
||||
def default_bin(name, default):
|
||||
assert default
|
||||
env_name = "ARCHERY_{0}_BIN".format(default.upper())
|
||||
env_name = f"ARCHERY_{default.upper()}_BIN"
|
||||
return name if name else os.environ.get(env_name, default)
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class Command:
|
||||
if "check" not in kwargs:
|
||||
kwargs["check"] = True
|
||||
|
||||
logger.debug("Executing `{}`".format(invocation))
|
||||
logger.debug(f"Executing `{invocation}`")
|
||||
return subprocess.run(invocation, **kwargs)
|
||||
|
||||
@property
|
||||
|
||||
@@ -355,18 +355,16 @@ def python_numpydoc(symbols=None, allow_rules=None, disallow_rules=None):
|
||||
qualname_with_signature = '.'.join([module, cython_signature])
|
||||
click.echo(
|
||||
click.style(
|
||||
'-> {}'.format(qualname_with_signature),
|
||||
f'-> {qualname_with_signature}',
|
||||
fg='yellow'
|
||||
)
|
||||
)
|
||||
|
||||
for error in errors:
|
||||
number_of_violations += 1
|
||||
click.echo('{}: {}'.format(*error))
|
||||
click.echo(f'{error[0]}: {error[1]}')
|
||||
|
||||
msg = 'Total number of docstring violations: {}'.format(
|
||||
number_of_violations
|
||||
)
|
||||
msg = f'Total number of docstring violations: {number_of_violations}'
|
||||
click.echo()
|
||||
click.echo(click.style(msg, fg='red'))
|
||||
|
||||
@@ -392,7 +390,7 @@ def rat_linter(src, root):
|
||||
|
||||
violations = list(report.validate(exclusion=exclusion))
|
||||
for violation in violations:
|
||||
print("apache-rat license violation: {}".format(violation))
|
||||
print(f"apache-rat license violation: {violation}")
|
||||
|
||||
yield LintResult(len(violations) == 0)
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ class MavenDefinition:
|
||||
# Extra safety to ensure we're deleting a build folder.
|
||||
if not MavenBuild.is_build_dir(build_dir):
|
||||
raise FileExistsError(
|
||||
"{} is not a maven build".format(build_dir)
|
||||
f"{build_dir} is not a maven build"
|
||||
)
|
||||
|
||||
cmd_kwargs = cmd_kwargs if cmd_kwargs else {}
|
||||
@@ -113,7 +113,7 @@ class MavenDefinition:
|
||||
return arguments
|
||||
|
||||
def __repr__(self):
|
||||
return "MavenDefinition[source={}]".format(self.source)
|
||||
return f"MavenDefinition[source={self.source}]"
|
||||
|
||||
|
||||
class MavenBuild(Maven):
|
||||
@@ -193,12 +193,11 @@ class MavenBuild(Maven):
|
||||
be lost.
|
||||
"""
|
||||
if not MavenBuild.is_build_dir(path):
|
||||
raise ValueError("Not a valid MavenBuild path: {}".format(path))
|
||||
raise ValueError(f"Not a valid MavenBuild path: {path}")
|
||||
|
||||
return MavenBuild(path, definition=None)
|
||||
|
||||
def __repr__(self):
|
||||
return ("MavenBuild["
|
||||
"build = {},"
|
||||
"definition = {}]".format(self.build_dir,
|
||||
self.definition))
|
||||
f"build = {self.build_dir},"
|
||||
f"definition = {self.definition}]")
|
||||
@@ -24,7 +24,7 @@ from .cache import Cache
|
||||
from .command import capture_stdout
|
||||
|
||||
RAT_VERSION = 0.13
|
||||
RAT_JAR_FILENAME = "apache-rat-{}.jar".format(RAT_VERSION)
|
||||
RAT_JAR_FILENAME = f"apache-rat-{RAT_VERSION}.jar"
|
||||
RAT_URL_ = "https://repo1.maven.org/maven2/org/apache/rat/apache-rat"
|
||||
RAT_URL = "/".join([RAT_URL_, str(RAT_VERSION), RAT_JAR_FILENAME])
|
||||
|
||||
@@ -54,7 +54,7 @@ class RatReport:
|
||||
self.tree = ElementTree.fromstring(xml)
|
||||
|
||||
def __repr__(self):
|
||||
return "RatReport({})".format(self.xml)
|
||||
return f"RatReport({self.xml})"
|
||||
|
||||
def validate(self, exclusion=None):
|
||||
for r in self.tree.findall('resource'):
|
||||
|
||||
@@ -32,7 +32,7 @@ class Report(metaclass=ABCMeta):
|
||||
def __init__(self, **kwargs):
|
||||
for field in self.fields:
|
||||
if field not in kwargs:
|
||||
raise ValueError('Missing keyword argument {}'.format(field))
|
||||
raise ValueError(f'Missing keyword argument {field}')
|
||||
self._data = kwargs
|
||||
|
||||
def __getattr__(self, key):
|
||||
|
||||
@@ -64,7 +64,7 @@ class ArrowSources:
|
||||
# validate by checking a specific path in the arrow source tree
|
||||
if not (path / 'cpp' / 'CMakeLists.txt').exists():
|
||||
raise InvalidArrowSource(
|
||||
"No Arrow C++ sources found in {}.".format(path)
|
||||
f"No Arrow C++ sources found in {path}."
|
||||
)
|
||||
self.path = path
|
||||
|
||||
@@ -116,7 +116,7 @@ class ArrowSources:
|
||||
def archive(self, path, dereference=False, compressor=None, revision=None):
|
||||
""" Saves a git archive at path. """
|
||||
if not self.git_backed:
|
||||
raise ValueError("{} is not backed by git".format(self))
|
||||
raise ValueError(f"{self} is not backed by git")
|
||||
|
||||
rev = revision if revision else "HEAD"
|
||||
archive = git.archive("--prefix=apache-arrow.tmp/", rev,
|
||||
@@ -160,7 +160,7 @@ class ArrowSources:
|
||||
Path to checkout the local clone.
|
||||
"""
|
||||
if not self.git_backed:
|
||||
raise ValueError("{} is not backed by git".format(self))
|
||||
raise ValueError(f"{self} is not backed by git")
|
||||
|
||||
if revision == ArrowSources.WORKSPACE:
|
||||
return self, False
|
||||
@@ -226,10 +226,10 @@ class ArrowSources:
|
||||
except InvalidArrowSource:
|
||||
pass
|
||||
|
||||
searched_paths = "\n".join([" - {}".format(p) for p in paths])
|
||||
searched_paths = "\n".join([f" - {p}" for p in paths])
|
||||
raise InvalidArrowSource(
|
||||
"Unable to locate Arrow's source directory. "
|
||||
"Searched paths are:\n{}".format(searched_paths)
|
||||
f"Searched paths are:\n{searched_paths}"
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
|
||||
+10
-11
@@ -235,13 +235,12 @@ def format_issue_output(issue_type, issue_id, status,
|
||||
|
||||
url = f'https://github.com/{ORG_NAME}/{PROJECT_NAME}/issues/{url_id}'
|
||||
|
||||
return """=== {} {} ===
|
||||
Summary\t\t{}
|
||||
Assignee\t{}
|
||||
Components\t{}
|
||||
Status\t\t{}
|
||||
URL\t\t{}""".format(issue_type.upper(), issue_id, summary, assignee,
|
||||
components, status, url)
|
||||
return f"""=== {issue_type.upper()} {issue_id} ===
|
||||
Summary\t\t{summary}
|
||||
Assignee\t{assignee}
|
||||
Components\t{components}
|
||||
Status\t\t{status}
|
||||
URL\t\t{url}"""
|
||||
|
||||
|
||||
class GitHubAPI(object):
|
||||
@@ -263,7 +262,7 @@ class GitHubAPI(object):
|
||||
'(GitHub personal access token):')
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
'Authorization': 'token {0}'.format(token),
|
||||
'Authorization': f'token {token}',
|
||||
}
|
||||
self.headers = headers
|
||||
|
||||
@@ -429,7 +428,7 @@ class PullRequest(object):
|
||||
return GitHubIssue(self._github_api, github_id, self.cmd)
|
||||
|
||||
self.cmd.fail("PR title should be prefixed by a GitHub ID, like: "
|
||||
"GH-XXX, but found {0}".format(self.title))
|
||||
f"GH-XXX, but found {self.title}")
|
||||
|
||||
def merge(self):
|
||||
"""
|
||||
@@ -459,7 +458,7 @@ class PullRequest(object):
|
||||
reverse=True)
|
||||
|
||||
for i, author in enumerate(distinct_authors):
|
||||
print("Author {}: {}".format(i + 1, author))
|
||||
print(f"Author {i + 1}: {author}")
|
||||
|
||||
if len(distinct_authors) > 1:
|
||||
primary_author, distinct_other_authors = get_primary_author(
|
||||
@@ -533,7 +532,7 @@ def get_primary_author(cmd, distinct_authors):
|
||||
|
||||
if author_pat.match(primary_author):
|
||||
break
|
||||
print('Bad author "{}", please try again'.format(primary_author))
|
||||
print(f'Bad author "{primary_author}", please try again')
|
||||
|
||||
# When primary author is specified manually, de-dup it from
|
||||
# author list and put it at the head of author list.
|
||||
|
||||
@@ -103,7 +103,7 @@ class Downloader:
|
||||
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
print("Downloading {} to {}".format(path, dest_path))
|
||||
print(f"Downloading {path} to {dest_path}")
|
||||
|
||||
url = f"{self.URL_ROOT}/{path}"
|
||||
self._download_url(url, dest_path)
|
||||
@@ -272,7 +272,7 @@ def download_rc_binaries(
|
||||
repository=None,
|
||||
tag=None,
|
||||
):
|
||||
version_string = "{}-rc{}".format(version, rc_number)
|
||||
version_string = f'{version}-rc{rc_number}'
|
||||
version_pattern = re.compile(r"\d+\.\d+\.\d+")
|
||||
if target_package_type:
|
||||
package_types = [target_package_type]
|
||||
|
||||
+1
-1
@@ -382,7 +382,7 @@ html_context = {
|
||||
# The name for this set of Sphinx documents.
|
||||
# "<project> v<release> documentation" by default.
|
||||
#
|
||||
html_title = u'Apache Arrow v{}'.format(version)
|
||||
html_title = f"Apache Arrow v{version}"
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#
|
||||
|
||||
@@ -37,7 +37,7 @@ class ParquetWriteBinary(object):
|
||||
i in range(nuniques)], dtype='O')
|
||||
values = unique_values[np.random.randint(0, nuniques, size=length)]
|
||||
self.table = pa.table([pa.array(values) for i in range(num_cols)],
|
||||
names=['f{}'.format(i) for i in range(num_cols)])
|
||||
names=[f'f{i}' for i in range(num_cols)])
|
||||
self.table_df = self.table.to_pandas()
|
||||
|
||||
def time_write_binary_table(self):
|
||||
@@ -72,7 +72,7 @@ def generate_dict_table(num_cols, string_size, nunique, length,
|
||||
random_order=random_order)
|
||||
return pa.table([
|
||||
data for i in range(num_cols)
|
||||
], names=['f{}'.format(i) for i in range(num_cols)])
|
||||
], names=[f'f{i}' for i in range(num_cols)])
|
||||
|
||||
|
||||
class ParquetWriteDictionaries(object):
|
||||
|
||||
@@ -99,8 +99,7 @@ class FlightServer(pyarrow.flight.FlightServerBase):
|
||||
|
||||
def do_action(self, context, action):
|
||||
if action.type == "clear":
|
||||
raise NotImplementedError(
|
||||
"{} is not implemented.".format(action.type))
|
||||
raise NotImplementedError(f"{action.type} is not implemented.")
|
||||
elif action.type == "healthcheck":
|
||||
pass
|
||||
elif action.type == "shutdown":
|
||||
@@ -109,7 +108,7 @@ class FlightServer(pyarrow.flight.FlightServerBase):
|
||||
# request
|
||||
threading.Thread(target=self._shutdown).start()
|
||||
else:
|
||||
raise KeyError("Unknown action {!r}".format(action.type))
|
||||
raise KeyError(f"Unknown action {action.type!r}")
|
||||
|
||||
def _shutdown(self):
|
||||
"""Shut down after a delay."""
|
||||
@@ -141,7 +140,7 @@ def main():
|
||||
tls_private_key = key_file.read()
|
||||
tls_certificates.append((tls_cert_chain, tls_private_key))
|
||||
|
||||
location = "{}://{}:{}".format(scheme, args.host, args.port)
|
||||
location = f"{scheme}://{args.host}:{args.port}"
|
||||
|
||||
server = FlightServer(args.host, location,
|
||||
tls_certificates=tls_certificates,
|
||||
|
||||
@@ -408,7 +408,7 @@ def get_library_dirs():
|
||||
if not library_dir.startswith("-L"):
|
||||
raise ValueError(
|
||||
"pkg-config --libs-only-L returned unexpected "
|
||||
"value {!r}".format(library_dir))
|
||||
f"value {library_dir!r}")
|
||||
append_library_dir(library_dir[2:])
|
||||
|
||||
if _sys.platform == 'win32':
|
||||
|
||||
@@ -553,7 +553,7 @@ cdef class Declaration(_Weakrefable):
|
||||
return frombytes(GetResultValue(DeclarationToString(self.decl)))
|
||||
|
||||
def __repr__(self):
|
||||
return "<pyarrow.acero.Declaration>\n{0}".format(str(self))
|
||||
return f"<pyarrow.acero.Declaration>\n{self}"
|
||||
|
||||
def to_table(self, bint use_threads=True):
|
||||
"""
|
||||
|
||||
+11
-26
@@ -63,14 +63,10 @@ def _pas():
|
||||
|
||||
|
||||
def _forbid_instantiation(klass, subclasses_instead=True):
|
||||
msg = '{} is an abstract class thus cannot be initialized.'.format(
|
||||
klass.__name__
|
||||
)
|
||||
msg = f'{klass.__name__} is an abstract class thus cannot be initialized.'
|
||||
if subclasses_instead:
|
||||
subclasses = [cls.__name__ for cls in klass.__subclasses__]
|
||||
msg += ' Use one of the subclasses instead: {}'.format(
|
||||
', '.join(subclasses)
|
||||
)
|
||||
msg += f' Use one of the subclasses instead: {", ".join(subclasses)}'
|
||||
raise TypeError(msg)
|
||||
|
||||
|
||||
@@ -201,8 +197,7 @@ cdef class Kernel(_Weakrefable):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
|
||||
|
||||
|
||||
cdef class ScalarKernel(Kernel):
|
||||
@@ -212,8 +207,7 @@ cdef class ScalarKernel(Kernel):
|
||||
self.kernel = kernel
|
||||
|
||||
def __repr__(self):
|
||||
return ("ScalarKernel<{}>"
|
||||
.format(frombytes(self.kernel.signature.get().ToString())))
|
||||
return f"ScalarKernel<{frombytes(self.kernel.signature.get().ToString())}>"
|
||||
|
||||
|
||||
cdef class VectorKernel(Kernel):
|
||||
@@ -223,8 +217,7 @@ cdef class VectorKernel(Kernel):
|
||||
self.kernel = kernel
|
||||
|
||||
def __repr__(self):
|
||||
return ("VectorKernel<{}>"
|
||||
.format(frombytes(self.kernel.signature.get().ToString())))
|
||||
return f"VectorKernel<{frombytes(self.kernel.signature.get().ToString())}>"
|
||||
|
||||
|
||||
cdef class ScalarAggregateKernel(Kernel):
|
||||
@@ -234,8 +227,7 @@ cdef class ScalarAggregateKernel(Kernel):
|
||||
self.kernel = kernel
|
||||
|
||||
def __repr__(self):
|
||||
return ("ScalarAggregateKernel<{}>"
|
||||
.format(frombytes(self.kernel.signature.get().ToString())))
|
||||
return f"ScalarAggregateKernel<{frombytes(self.kernel.signature.get().ToString())}>"
|
||||
|
||||
|
||||
cdef class HashAggregateKernel(Kernel):
|
||||
@@ -245,8 +237,7 @@ cdef class HashAggregateKernel(Kernel):
|
||||
self.kernel = kernel
|
||||
|
||||
def __repr__(self):
|
||||
return ("HashAggregateKernel<{}>"
|
||||
.format(frombytes(self.kernel.signature.get().ToString())))
|
||||
return f"HashAggregateKernel<{frombytes(self.kernel.signature.get().ToString())}>"
|
||||
|
||||
|
||||
FunctionDoc = namedtuple(
|
||||
@@ -298,17 +289,14 @@ cdef class Function(_Weakrefable):
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
|
||||
|
||||
cdef void init(self, const shared_ptr[CFunction]& sp_func) except *:
|
||||
self.sp_func = sp_func
|
||||
self.base_func = sp_func.get()
|
||||
|
||||
def __repr__(self):
|
||||
return ("arrow.compute.Function<name={}, kind={}, "
|
||||
"arity={}, num_kernels={}>"
|
||||
.format(self.name, self.kind, self.arity, self.num_kernels))
|
||||
return f"arrow.compute.Function<name={self.name}, kind={self.kind}, arity={self.arity}, num_kernels={self.num_kernels}>"
|
||||
|
||||
def __reduce__(self):
|
||||
# Reduction uses the global registry
|
||||
@@ -2568,9 +2556,7 @@ cdef class Expression(_Weakrefable):
|
||||
return frombytes(self.expr.ToString())
|
||||
|
||||
def __repr__(self):
|
||||
return "<pyarrow.compute.{0} {1}>".format(
|
||||
self.__class__.__name__, str(self)
|
||||
)
|
||||
return f"<pyarrow.compute.{self.__class__.__name__} {self}>"
|
||||
|
||||
@staticmethod
|
||||
def from_substrait(object message not None):
|
||||
@@ -2874,8 +2860,7 @@ cdef class UdfContext:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
|
||||
|
||||
cdef void init(self, const CUdfContext &c_context):
|
||||
self.c_context = c_context
|
||||
|
||||
@@ -1150,9 +1150,8 @@ cdef class CSVStreamingReader(RecordBatchReader):
|
||||
Schema schema
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, "
|
||||
"use pyarrow.csv.open_csv() instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use pyarrow.csv.open_csv() instead.")
|
||||
|
||||
# Note about cancellation: we cannot create a SignalStopHandler
|
||||
# by default here, as several CSVStreamingReader instances may be
|
||||
|
||||
@@ -841,8 +841,7 @@ cdef class BufferWriter(NativeFile):
|
||||
offset = offset + position
|
||||
else:
|
||||
with gil:
|
||||
raise ValueError("Invalid value of whence: {0}"
|
||||
.format(whence))
|
||||
raise ValueError(f"Invalid value of whence: {whence}")
|
||||
check_status(self.writer.Seek(offset))
|
||||
return self.tell()
|
||||
|
||||
|
||||
@@ -1128,7 +1128,7 @@ cdef class FileSystemDataset(Dataset):
|
||||
elif not isinstance(root_partition, Expression):
|
||||
raise TypeError(
|
||||
"Argument 'root_partition' has incorrect type (expected "
|
||||
"Expression, got {0})".format(type(root_partition))
|
||||
f"Expression, got {type(root_partition)})"
|
||||
)
|
||||
|
||||
for fragment in fragments:
|
||||
@@ -1220,8 +1220,8 @@ cdef class FileSystemDataset(Dataset):
|
||||
]:
|
||||
if not isinstance(arg, class_):
|
||||
raise TypeError(
|
||||
"Argument '{0}' has incorrect type (expected {1}, "
|
||||
"got {2})".format(name, class_.__name__, type(arg))
|
||||
f"Argument '{name}' has incorrect type (expected {class_.__name__}, "
|
||||
f"got {type(arg)})"
|
||||
)
|
||||
|
||||
partitions = partitions or [_true] * len(paths)
|
||||
@@ -1988,9 +1988,7 @@ cdef class FileFragment(Fragment):
|
||||
)
|
||||
if partition:
|
||||
partition = f" partition=[{partition}]"
|
||||
return "<pyarrow.dataset.{0}{1} path={2}{3}>".format(
|
||||
self.__class__.__name__, typ, self.path, partition
|
||||
)
|
||||
return f"<pyarrow.dataset.{self.__class__.__name__}{typ} path={self.path}{partition}>"
|
||||
|
||||
def __reduce__(self):
|
||||
buffer = self.buffer
|
||||
@@ -3385,7 +3383,7 @@ cdef class FileSystemDatasetFactory(DatasetFactory):
|
||||
)
|
||||
else:
|
||||
raise TypeError('Must pass either paths or a FileSelector, but '
|
||||
'passed {}'.format(type(paths_or_selector)))
|
||||
f'passed {type(paths_or_selector)}')
|
||||
|
||||
self.init(GetResultValue(result))
|
||||
|
||||
@@ -3528,7 +3526,7 @@ cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
|
||||
if not isinstance(expr, Expression):
|
||||
raise TypeError(
|
||||
"Expected an Expression for a 'column' dictionary "
|
||||
"value, got {} instead".format(type(expr))
|
||||
f"value, got {type(expr)} instead"
|
||||
)
|
||||
c_exprs.push_back((<Expression> expr).unwrap())
|
||||
|
||||
@@ -3540,7 +3538,7 @@ cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
|
||||
else:
|
||||
raise ValueError(
|
||||
"Expected a list or a dict for 'columns', "
|
||||
"got {} instead.".format(type(columns))
|
||||
f"got {type(columns)} instead."
|
||||
)
|
||||
|
||||
check_status(builder.BatchSize(batch_size))
|
||||
|
||||
@@ -336,7 +336,7 @@ class RowGroupInfo:
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return "RowGroupInfo({})".format(self.id)
|
||||
return f"RowGroupInfo({self.id})"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, int):
|
||||
@@ -671,9 +671,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions):
|
||||
self._set_arrow_properties()
|
||||
|
||||
def __repr__(self):
|
||||
return "<pyarrow.dataset.ParquetFileWriteOptions {0}>".format(
|
||||
" ".join([f"{key}={value}" for key, value in self._properties.items()])
|
||||
)
|
||||
return f"<pyarrow.dataset.ParquetFileWriteOptions {self._properties}>"
|
||||
|
||||
|
||||
cdef set _PARQUET_READ_OPTIONS = {
|
||||
|
||||
+28
-41
@@ -88,8 +88,7 @@ _FLIGHT_SERVER_ERROR_REGEX = re.compile(
|
||||
def _munge_grpc_python_error(message):
|
||||
m = _FLIGHT_SERVER_ERROR_REGEX.match(message)
|
||||
if m:
|
||||
return ('Flight RPC failed with Python exception \"{}: {}\"'
|
||||
.format(m.group(2), m.group(1)))
|
||||
return f'Flight RPC failed with Python exception "{m.group(2)}: {m.group(1)}"'
|
||||
else:
|
||||
return message
|
||||
|
||||
@@ -130,8 +129,7 @@ cdef class FlightCallOptions(_Weakrefable):
|
||||
self.options.write_options = c_write_options.c_options
|
||||
if read_options is not None:
|
||||
if not isinstance(read_options, IpcReadOptions):
|
||||
raise TypeError("expected IpcReadOptions, got {}"
|
||||
.format(type(read_options)))
|
||||
raise TypeError(f"expected IpcReadOptions, got {type(read_options)}")
|
||||
self.options.read_options = read_options.c_options
|
||||
if headers is not None:
|
||||
self.options.headers = headers
|
||||
@@ -142,8 +140,7 @@ cdef class FlightCallOptions(_Weakrefable):
|
||||
return &DEFAULT_CALL_OPTIONS
|
||||
elif isinstance(obj, FlightCallOptions):
|
||||
return &((<FlightCallOptions> obj).options)
|
||||
raise TypeError("Expected a FlightCallOptions object, not "
|
||||
"'{}'".format(type(obj)))
|
||||
raise TypeError(f"Expected a FlightCallOptions object, not '{type(obj)}'")
|
||||
|
||||
|
||||
_CertKeyPair = collections.namedtuple('_CertKeyPair', ['cert', 'key'])
|
||||
@@ -182,7 +179,7 @@ cdef class FlightError(Exception):
|
||||
self.extra_info = tobytes(extra_info)
|
||||
|
||||
cdef CStatus to_status(self):
|
||||
message = tobytes("Flight error: {}".format(str(self)))
|
||||
message = tobytes(f"Flight error: {self}")
|
||||
return CStatus_UnknownError(message)
|
||||
|
||||
|
||||
@@ -280,8 +277,7 @@ cdef class Action(_Weakrefable):
|
||||
@staticmethod
|
||||
cdef CAction unwrap(action) except *:
|
||||
if not isinstance(action, Action):
|
||||
raise TypeError("Must provide Action, not '{}'".format(
|
||||
type(action)))
|
||||
raise TypeError(f"Must provide Action, not '{type(action)}'")
|
||||
return (<Action> action).action
|
||||
|
||||
def serialize(self):
|
||||
@@ -493,10 +489,9 @@ cdef class FlightDescriptor(_Weakrefable):
|
||||
CFlightDescriptor descriptor
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
|
||||
"`pyarrow.flight.FlightDescriptor.for_{path,command}` "
|
||||
"function instead."
|
||||
.format(self.__class__.__name__))
|
||||
"function instead.")
|
||||
|
||||
@staticmethod
|
||||
def for_path(*path):
|
||||
@@ -552,8 +547,8 @@ cdef class FlightDescriptor(_Weakrefable):
|
||||
@staticmethod
|
||||
cdef CFlightDescriptor unwrap(descriptor) except *:
|
||||
if not isinstance(descriptor, FlightDescriptor):
|
||||
raise TypeError("Must provide a FlightDescriptor, not '{}'".format(
|
||||
type(descriptor)))
|
||||
raise TypeError(
|
||||
f"Must provide a FlightDescriptor, not '{type(descriptor)}'")
|
||||
return (<FlightDescriptor> descriptor).descriptor
|
||||
|
||||
def serialize(self):
|
||||
@@ -693,8 +688,7 @@ cdef class Location(_Weakrefable):
|
||||
CLocation.Parse(tobytes(location)).Value(&c_location))
|
||||
return c_location
|
||||
elif not isinstance(location, Location):
|
||||
raise TypeError("Must provide a Location, not '{}'".format(
|
||||
type(location)))
|
||||
raise TypeError(f"Must provide a Location, not '{type(location)}'")
|
||||
return (<Location> location).location
|
||||
|
||||
|
||||
@@ -733,7 +727,7 @@ cdef class FlightEndpoint(_Weakrefable):
|
||||
self.endpoint.ticket.ticket = tobytes(ticket)
|
||||
else:
|
||||
raise TypeError("Argument ticket must be a Ticket instance, string or bytes, "
|
||||
"not '{}'".format(type(ticket)))
|
||||
f"not '{type(ticket)}'")
|
||||
|
||||
for location in locations:
|
||||
if isinstance(location, Location):
|
||||
@@ -744,7 +738,7 @@ cdef class FlightEndpoint(_Weakrefable):
|
||||
CLocation.Parse(tobytes(location)).Value(&c_location))
|
||||
else:
|
||||
raise TypeError("Argument locations must contain Location instances, strings or bytes, "
|
||||
"not '{}'".format(type(location)))
|
||||
f"not '{type(location)}'")
|
||||
self.endpoint.locations.push_back(c_location)
|
||||
|
||||
if expiration_time is not None:
|
||||
@@ -753,11 +747,11 @@ cdef class FlightEndpoint(_Weakrefable):
|
||||
expiration_time.cast(timestamp("ns")).value)
|
||||
else:
|
||||
raise TypeError("Argument expiration_time must be a TimestampScalar, "
|
||||
"not '{}'".format(type(expiration_time)))
|
||||
f"not '{type(expiration_time)}'")
|
||||
|
||||
if not isinstance(app_metadata, (str, bytes)):
|
||||
raise TypeError("Argument app_metadata must be a string or bytes, "
|
||||
"not '{}'".format(type(app_metadata)))
|
||||
f"not '{type(app_metadata)}'")
|
||||
self.endpoint.app_metadata = tobytes(app_metadata)
|
||||
|
||||
@property
|
||||
@@ -919,8 +913,8 @@ cdef class FlightInfo(_Weakrefable):
|
||||
if isinstance(endpoint, FlightEndpoint):
|
||||
c_endpoints.push_back((<FlightEndpoint> endpoint).endpoint)
|
||||
else:
|
||||
raise TypeError('Endpoint {} is not instance of'
|
||||
' FlightEndpoint'.format(endpoint))
|
||||
raise TypeError(
|
||||
f'Endpoint {endpoint} is not instance of FlightEndpoint')
|
||||
|
||||
check_flight_status(CreateFlightInfo(c_schema,
|
||||
descriptor.descriptor,
|
||||
@@ -1048,8 +1042,7 @@ cdef class FlightStreamChunk(_Weakrefable):
|
||||
return iter((self.data, self.app_metadata))
|
||||
|
||||
def __repr__(self):
|
||||
return "<FlightStreamChunk with data: {} with metadata: {}>".format(
|
||||
self.chunk.data != NULL, self.chunk.app_metadata != NULL)
|
||||
return f"<FlightStreamChunk with data: {self.chunk.data != NULL} with metadata: {self.chunk.app_metadata != NULL}>"
|
||||
|
||||
|
||||
cdef class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin):
|
||||
@@ -1527,8 +1520,7 @@ cdef class FlightClient(_Weakrefable):
|
||||
|
||||
if not isinstance(auth_handler, ClientAuthHandler):
|
||||
raise TypeError(
|
||||
"FlightClient.authenticate takes a ClientAuthHandler, "
|
||||
"not '{}'".format(type(auth_handler)))
|
||||
f"FlightClient.authenticate takes a ClientAuthHandler, not '{type(auth_handler)}'")
|
||||
handler.reset((<ClientAuthHandler> auth_handler).to_handler())
|
||||
with nogil:
|
||||
check_flight_status(
|
||||
@@ -1831,7 +1823,7 @@ cdef class RecordBatchStream(FlightDataStream):
|
||||
if (not isinstance(data_source, RecordBatchReader) and
|
||||
not isinstance(data_source, lib.Table)):
|
||||
raise TypeError("Expected RecordBatchReader or Table, "
|
||||
"but got: {}".format(type(data_source)))
|
||||
f"but got: {type(data_source)}")
|
||||
self.data_source = data_source
|
||||
self.write_options = _get_options(options).c_options
|
||||
|
||||
@@ -1844,8 +1836,8 @@ cdef class RecordBatchStream(FlightDataStream):
|
||||
table = (<Table> self.data_source).table
|
||||
reader.reset(new TableBatchReader(deref(table)))
|
||||
else:
|
||||
raise RuntimeError("Can't construct RecordBatchStream "
|
||||
"from type {}".format(type(self.data_source)))
|
||||
raise RuntimeError(
|
||||
f"Can't construct RecordBatchStream from type {type(self.data_source)}")
|
||||
return new CRecordBatchStream(reader, self.write_options)
|
||||
|
||||
|
||||
@@ -2127,8 +2119,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
|
||||
raise ValueError("Got a FlightDataStream whose schema "
|
||||
"does not match the declared schema of this "
|
||||
"GeneratorStream. "
|
||||
"Got: {}\nExpected: {}".format(
|
||||
substream_schema, stream_schema))
|
||||
f"Got: {substream_schema}\nExpected: {stream_schema}")
|
||||
stream.current_stream.reset(
|
||||
new CPyFlightDataStream(result, move(data_stream)))
|
||||
# Loop around and try again
|
||||
@@ -2139,8 +2130,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
|
||||
raise ValueError("Got a RecordBatch whose schema does not "
|
||||
"match the declared schema of this "
|
||||
"GeneratorStream. "
|
||||
"Got: {}\nExpected: {}".format(batch.schema,
|
||||
stream_schema))
|
||||
f"Got: {batch.schema}\nExpected: {stream_schema}")
|
||||
check_flight_status(GetRecordBatchPayload(
|
||||
deref(batch.batch),
|
||||
stream.c_options,
|
||||
@@ -2152,7 +2142,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
|
||||
raise TypeError("GeneratorStream must be initialized with "
|
||||
"an iterator of FlightDataStream, Table, "
|
||||
"RecordBatch, or RecordBatchStreamReader objects, "
|
||||
"not {}.".format(type(result)))
|
||||
f"not {type(result)}.")
|
||||
# Don't loop around
|
||||
return CStatus_OK()
|
||||
# Ran out of attempts (the RPC handler kept yielding empty tables/readers)
|
||||
@@ -2174,8 +2164,7 @@ cdef CStatus _list_flights(void* self, const CServerCallContext& context,
|
||||
for info in result:
|
||||
if not isinstance(info, FlightInfo):
|
||||
raise TypeError("FlightServerBase.list_flights must return "
|
||||
"FlightInfo instances, but got {}".format(
|
||||
type(info)))
|
||||
f"FlightInfo instances, but got {type(info)}")
|
||||
flights.push_back(deref((<FlightInfo> info).info.get()))
|
||||
listing.reset(new CSimpleFlightListing(flights))
|
||||
except FlightError as flight_error:
|
||||
@@ -2199,8 +2188,7 @@ cdef CStatus _get_flight_info(void* self, const CServerCallContext& context,
|
||||
return (<FlightError> flight_error).to_status()
|
||||
if not isinstance(result, FlightInfo):
|
||||
raise TypeError("FlightServerBase.get_flight_info must return "
|
||||
"a FlightInfo instance, but got {}".format(
|
||||
type(result)))
|
||||
f"a FlightInfo instance, but got {type(result)}")
|
||||
info.reset(new CFlightInfo(deref((<FlightInfo> result).info.get())))
|
||||
return CStatus_OK()
|
||||
|
||||
@@ -2216,8 +2204,7 @@ cdef CStatus _get_schema(void* self, const CServerCallContext& context,
|
||||
py_descriptor)
|
||||
if not isinstance(result, SchemaResult):
|
||||
raise TypeError("FlightServerBase.get_schema_info must return "
|
||||
"a SchemaResult instance, but got {}".format(
|
||||
type(result)))
|
||||
f"a SchemaResult instance, but got {type(result)}")
|
||||
info.reset(new CSchemaResult(deref((<SchemaResult> result).result.get())))
|
||||
return CStatus_OK()
|
||||
|
||||
@@ -2943,7 +2930,7 @@ cdef class FlightServerBase(_Weakrefable):
|
||||
if auth_handler:
|
||||
if not isinstance(auth_handler, ServerAuthHandler):
|
||||
raise TypeError("auth_handler must be a ServerAuthHandler, "
|
||||
"not a '{}'".format(type(auth_handler)))
|
||||
f"not a '{type(auth_handler)}'")
|
||||
c_options.get().auth_handler.reset(
|
||||
(<ServerAuthHandler> auth_handler).to_handler())
|
||||
|
||||
|
||||
+7
-10
@@ -180,8 +180,7 @@ cdef class FileInfo(_Weakrefable):
|
||||
@staticmethod
|
||||
cdef CFileInfo unwrap_safe(obj):
|
||||
if not isinstance(obj, FileInfo):
|
||||
raise TypeError("Expected FileInfo instance, got {0}"
|
||||
.format(type(obj)))
|
||||
raise TypeError(f"Expected FileInfo instance, got {type(obj)}")
|
||||
return (<FileInfo> obj).unwrap()
|
||||
|
||||
def __repr__(self):
|
||||
@@ -406,8 +405,7 @@ cdef class FileSelector(_Weakrefable):
|
||||
self.selector.recursive = recursive
|
||||
|
||||
def __repr__(self):
|
||||
return ("<FileSelector base_dir={0.base_dir!r} "
|
||||
"recursive={0.recursive}>".format(self))
|
||||
return f"<FileSelector base_dir={self.base_dir!r} recursive={self.recursive}>"
|
||||
|
||||
|
||||
cdef class FileSystem(_Weakrefable):
|
||||
@@ -449,7 +447,7 @@ cdef class FileSystem(_Weakrefable):
|
||||
--------
|
||||
Create a new FileSystem subclass from a URI:
|
||||
|
||||
>>> uri = 'file:///{}/pyarrow-fs-example.dat'.format(local_path)
|
||||
>>> uri = f'file:///{local_path}/pyarrow-fs-example.dat'
|
||||
>>> local_new, path_new = fs.FileSystem.from_uri(uri)
|
||||
>>> local_new
|
||||
<pyarrow._fs.LocalFileSystem object at ...
|
||||
@@ -564,7 +562,7 @@ cdef class FileSystem(_Weakrefable):
|
||||
--------
|
||||
>>> local
|
||||
<pyarrow._fs.LocalFileSystem object at ...>
|
||||
>>> local.get_file_info("/{}/pyarrow-fs-example.dat".format(local_path))
|
||||
>>> local.get_file_info(f"/{local_path}/pyarrow-fs-example.dat")
|
||||
<FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
|
||||
"""
|
||||
cdef:
|
||||
@@ -1192,8 +1190,7 @@ cdef class SubTreeFileSystem(FileSystem):
|
||||
self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
|
||||
|
||||
def __repr__(self):
|
||||
return ("SubTreeFileSystem(base_path={}, base_fs={}"
|
||||
.format(self.base_path, self.base_fs))
|
||||
return f"SubTreeFileSystem(base_path={self.base_path}, base_fs={self.base_fs})"
|
||||
|
||||
def __reduce__(self):
|
||||
return SubTreeFileSystem, (
|
||||
@@ -1260,8 +1257,8 @@ cdef class PyFileSystem(FileSystem):
|
||||
shared_ptr[CPyFileSystem] wrapped
|
||||
|
||||
if not isinstance(handler, FileSystemHandler):
|
||||
raise TypeError("Expected a FileSystemHandler instance, got {0}"
|
||||
.format(type(handler)))
|
||||
raise TypeError(
|
||||
f"Expected a FileSystemHandler instance, got {type(handler)}")
|
||||
|
||||
vtable.get_type_name = _cb_get_type_name
|
||||
vtable.equals = _cb_equals
|
||||
|
||||
@@ -74,7 +74,7 @@ cdef class HadoopFileSystem(FileSystem):
|
||||
|
||||
if not host.startswith(('hdfs://', 'viewfs://')) and host != "default":
|
||||
# TODO(kszucs): do more sanitization
|
||||
host = 'hdfs://{}'.format(host)
|
||||
host = f'hdfs://{host}'
|
||||
|
||||
options.ConfigureEndPoint(tobytes(host), int(port))
|
||||
options.ConfigureReplication(replication)
|
||||
|
||||
@@ -216,8 +216,8 @@ cdef class ParseOptions(_Weakrefable):
|
||||
v = CUnexpectedFieldBehavior_InferType
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unexpected value `{}` for `unexpected_field_behavior`, pass "
|
||||
"either `ignore`, `error` or `infer`.".format(value)
|
||||
f"Unexpected value `{value}` for `unexpected_field_behavior`, pass "
|
||||
f"either `ignore`, `error` or `infer`."
|
||||
)
|
||||
|
||||
self.options.unexpected_field_behavior = v
|
||||
|
||||
+64
-111
@@ -59,25 +59,16 @@ cdef class Statistics(_Weakrefable):
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
return """{}
|
||||
has_min_max: {}
|
||||
min: {}
|
||||
max: {}
|
||||
null_count: {}
|
||||
distinct_count: {}
|
||||
num_values: {}
|
||||
physical_type: {}
|
||||
logical_type: {}
|
||||
converted_type (legacy): {}""".format(object.__repr__(self),
|
||||
self.has_min_max,
|
||||
self.min,
|
||||
self.max,
|
||||
self.null_count,
|
||||
self.distinct_count,
|
||||
self.num_values,
|
||||
self.physical_type,
|
||||
str(self.logical_type),
|
||||
self.converted_type)
|
||||
return f"""{object.__repr__(self)}
|
||||
has_min_max: {self.has_min_max}
|
||||
min: {self.min}
|
||||
max: {self.max}
|
||||
null_count: {self.null_count}
|
||||
distinct_count: {self.distinct_count}
|
||||
num_values: {self.num_values}
|
||||
physical_type: {self.physical_type}
|
||||
logical_type: {self.logical_type}
|
||||
converted_type (legacy): {self.converted_type}"""
|
||||
|
||||
def to_dict(self):
|
||||
"""
|
||||
@@ -234,7 +225,7 @@ cdef class ParquetLogicalType(_Weakrefable):
|
||||
self.type = type
|
||||
|
||||
def __repr__(self):
|
||||
return "{}\n {}".format(object.__repr__(self), str(self))
|
||||
return f"{object.__repr__(self)}\n {self}"
|
||||
|
||||
def __str__(self):
|
||||
return frombytes(self.type.get().ToString(), safe=True)
|
||||
@@ -455,39 +446,24 @@ cdef class ColumnChunkMetaData(_Weakrefable):
|
||||
def __repr__(self):
|
||||
statistics = indent(repr(self.statistics), 4 * ' ')
|
||||
geo_statistics = indent(repr(self.geo_statistics), 4 * ' ')
|
||||
return """{0}
|
||||
file_offset: {1}
|
||||
file_path: {2}
|
||||
physical_type: {3}
|
||||
num_values: {4}
|
||||
path_in_schema: {5}
|
||||
is_stats_set: {6}
|
||||
return f"""{object.__repr__(self)}
|
||||
file_offset: {self.file_offset}
|
||||
file_path: {self.file_path}
|
||||
physical_type: {self.physical_type}
|
||||
num_values: {self.num_values}
|
||||
path_in_schema: {self.path_in_schema}
|
||||
is_stats_set: {self.is_stats_set}
|
||||
statistics:
|
||||
{7}
|
||||
{statistics}
|
||||
geo_statistics:
|
||||
{8}
|
||||
compression: {9}
|
||||
encodings: {10}
|
||||
has_dictionary_page: {11}
|
||||
dictionary_page_offset: {12}
|
||||
data_page_offset: {13}
|
||||
total_compressed_size: {14}
|
||||
total_uncompressed_size: {15}""".format(object.__repr__(self),
|
||||
self.file_offset,
|
||||
self.file_path,
|
||||
self.physical_type,
|
||||
self.num_values,
|
||||
self.path_in_schema,
|
||||
self.is_stats_set,
|
||||
statistics,
|
||||
geo_statistics,
|
||||
self.compression,
|
||||
self.encodings,
|
||||
self.has_dictionary_page,
|
||||
self.dictionary_page_offset,
|
||||
self.data_page_offset,
|
||||
self.total_compressed_size,
|
||||
self.total_uncompressed_size)
|
||||
{geo_statistics}
|
||||
compression: {self.compression}
|
||||
encodings: {self.encodings}
|
||||
has_dictionary_page: {self.has_dictionary_page}
|
||||
dictionary_page_offset: {self.dictionary_page_offset}
|
||||
data_page_offset: {self.data_page_offset}
|
||||
total_compressed_size: {self.total_compressed_size}
|
||||
total_uncompressed_size: {self.total_uncompressed_size}"""
|
||||
|
||||
def to_dict(self):
|
||||
"""
|
||||
@@ -791,16 +767,15 @@ cdef class SortingColumn:
|
||||
elif descending == "ascending":
|
||||
descending = False
|
||||
else:
|
||||
raise ValueError("Invalid sort key direction: {0}"
|
||||
.format(descending))
|
||||
raise ValueError(f"Invalid sort key direction: {descending}")
|
||||
else:
|
||||
raise ValueError("Invalid sort key: {0}".format(sort_key))
|
||||
raise ValueError(f"Invalid sort key: {sort_key}")
|
||||
|
||||
try:
|
||||
column_index = col_map[name]
|
||||
except KeyError:
|
||||
raise ValueError("Sort key name '{0}' not found in schema:\n{1}"
|
||||
.format(name, schema))
|
||||
raise ValueError(
|
||||
f"Sort key name '{name}' not found in schema:\n{schema}")
|
||||
|
||||
sorting_columns.append(
|
||||
cls(column_index, descending=descending, nulls_first=nulls_first)
|
||||
@@ -851,9 +826,8 @@ cdef class SortingColumn:
|
||||
return tuple(sort_keys), null_placement
|
||||
|
||||
def __repr__(self):
|
||||
return """{}(column_index={}, descending={}, nulls_first={})""".format(
|
||||
self.__class__.__name__,
|
||||
self.column_index, self.descending, self.nulls_first)
|
||||
return f"{self.__class__.__name__}(column_index={self.column_index}, " \
|
||||
f"descending={self.descending}, nulls_first={self.nulls_first})"
|
||||
|
||||
def __eq__(self, SortingColumn other):
|
||||
return (self.column_index == other.column_index and
|
||||
@@ -943,21 +917,17 @@ cdef class RowGroupMetaData(_Weakrefable):
|
||||
Metadata for column within this chunk.
|
||||
"""
|
||||
if i < 0 or i >= self.num_columns:
|
||||
raise IndexError('{0} out of bounds'.format(i))
|
||||
raise IndexError(f'{i} out of bounds')
|
||||
cdef ColumnChunkMetaData chunk = ColumnChunkMetaData.__new__(ColumnChunkMetaData)
|
||||
chunk.init(self, i)
|
||||
return chunk
|
||||
|
||||
def __repr__(self):
|
||||
return """{0}
|
||||
num_columns: {1}
|
||||
num_rows: {2}
|
||||
total_byte_size: {3}
|
||||
sorting_columns: {4}""".format(object.__repr__(self),
|
||||
self.num_columns,
|
||||
self.num_rows,
|
||||
self.total_byte_size,
|
||||
self.sorting_columns)
|
||||
return f"""{object.__repr__(self)}
|
||||
num_columns: {self.num_columns}
|
||||
num_rows: {self.num_rows}
|
||||
total_byte_size: {self.total_byte_size}
|
||||
sorting_columns: {self.sorting_columns}"""
|
||||
|
||||
def to_dict(self):
|
||||
"""
|
||||
@@ -1047,17 +1017,13 @@ cdef class FileMetaData(_Weakrefable):
|
||||
self.serialized_size))
|
||||
|
||||
def __repr__(self):
|
||||
return """{0}
|
||||
created_by: {1}
|
||||
num_columns: {2}
|
||||
num_rows: {3}
|
||||
num_row_groups: {4}
|
||||
format_version: {5}
|
||||
serialized_size: {6}""".format(object.__repr__(self),
|
||||
self.created_by, self.num_columns,
|
||||
self.num_rows, self.num_row_groups,
|
||||
self.format_version,
|
||||
self.serialized_size)
|
||||
return f"""{object.__repr__(self)}
|
||||
created_by: {self.created_by}
|
||||
num_columns: {self.num_columns}
|
||||
num_rows: {self.num_rows}
|
||||
num_row_groups: {self.num_row_groups}
|
||||
format_version: {self.format_version}
|
||||
serialized_size: {self.serialized_size}"""
|
||||
|
||||
def to_dict(self):
|
||||
"""
|
||||
@@ -1145,8 +1111,7 @@ cdef class FileMetaData(_Weakrefable):
|
||||
elif version == ParquetVersion_V2_6:
|
||||
return '2.6'
|
||||
else:
|
||||
warnings.warn('Unrecognized file version, assuming 2.6: {}'
|
||||
.format(version))
|
||||
warnings.warn(f'Unrecognized file version, assuming 2.6: {version}')
|
||||
return '2.6'
|
||||
|
||||
@property
|
||||
@@ -1255,9 +1220,7 @@ cdef class ParquetSchema(_Weakrefable):
|
||||
self.schema = container._metadata.schema()
|
||||
|
||||
def __repr__(self):
|
||||
return "{0}\n{1}".format(
|
||||
object.__repr__(self),
|
||||
frombytes(self.schema.ToString(), safe=True))
|
||||
return f"{object.__repr__(self)}\n{frombytes(self.schema.ToString(), safe=True)}"
|
||||
|
||||
def __reduce__(self):
|
||||
return ParquetSchema, (self.parent,)
|
||||
@@ -1329,7 +1292,7 @@ cdef class ParquetSchema(_Weakrefable):
|
||||
column_schema : ColumnSchema
|
||||
"""
|
||||
if i < 0 or i >= len(self):
|
||||
raise IndexError('{0} out of bounds'.format(i))
|
||||
raise IndexError(f'{i} out of bounds')
|
||||
|
||||
return ColumnSchema(self, i)
|
||||
|
||||
@@ -1374,25 +1337,18 @@ cdef class ColumnSchema(_Weakrefable):
|
||||
physical_type = self.physical_type
|
||||
converted_type = self.converted_type
|
||||
if converted_type == 'DECIMAL':
|
||||
converted_type = 'DECIMAL({0}, {1})'.format(self.precision,
|
||||
self.scale)
|
||||
converted_type = f'DECIMAL({self.precision}, {self.scale})'
|
||||
elif physical_type == 'FIXED_LEN_BYTE_ARRAY':
|
||||
converted_type = ('FIXED_LEN_BYTE_ARRAY(length={0})'
|
||||
.format(self.length))
|
||||
converted_type = f'FIXED_LEN_BYTE_ARRAY(length={self.length})'
|
||||
|
||||
return """<ParquetColumnSchema>
|
||||
name: {0}
|
||||
path: {1}
|
||||
max_definition_level: {2}
|
||||
max_repetition_level: {3}
|
||||
physical_type: {4}
|
||||
logical_type: {5}
|
||||
converted_type (legacy): {6}""".format(self.name, self.path,
|
||||
self.max_definition_level,
|
||||
self.max_repetition_level,
|
||||
physical_type,
|
||||
str(self.logical_type),
|
||||
converted_type)
|
||||
return f"""<ParquetColumnSchema>
|
||||
name: {self.name}
|
||||
path: {self.path}
|
||||
max_definition_level: {self.max_definition_level}
|
||||
max_repetition_level: {self.max_repetition_level}
|
||||
physical_type: {physical_type}
|
||||
logical_type: {self.logical_type}
|
||||
converted_type (legacy): {converted_type}"""
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
@@ -2031,8 +1987,8 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
|
||||
elif data_page_version == "2.0":
|
||||
props.data_page_version(ParquetDataPageVersion_V2)
|
||||
else:
|
||||
raise ValueError("Unsupported Parquet data page version: {0}"
|
||||
.format(data_page_version))
|
||||
raise ValueError(
|
||||
f"Unsupported Parquet data page version: {data_page_version}")
|
||||
|
||||
# version
|
||||
|
||||
@@ -2044,8 +2000,7 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
|
||||
elif version == "2.6":
|
||||
props.version(ParquetVersion_V2_6)
|
||||
else:
|
||||
raise ValueError("Unsupported Parquet format version: {0}"
|
||||
.format(version))
|
||||
raise ValueError(f"Unsupported Parquet format version: {version}")
|
||||
|
||||
# compression
|
||||
|
||||
@@ -2220,8 +2175,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
|
||||
elif coerce_timestamps == 'us':
|
||||
arrow_props.coerce_timestamps(TimeUnit_MICRO)
|
||||
elif coerce_timestamps is not None:
|
||||
raise ValueError('Invalid value for coerce_timestamps: {0}'
|
||||
.format(coerce_timestamps))
|
||||
raise ValueError(f'Invalid value for coerce_timestamps: {coerce_timestamps}')
|
||||
|
||||
# allow_truncated_timestamps
|
||||
|
||||
@@ -2243,8 +2197,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
|
||||
warnings.warn("V1 parquet writer engine is a no-op. Use V2.")
|
||||
arrow_props.set_engine_version(ArrowWriterEngineVersion.V1)
|
||||
elif writer_engine_version != "V2":
|
||||
raise ValueError("Unsupported Writer Engine Version: {0}"
|
||||
.format(writer_engine_version))
|
||||
raise ValueError(f"Unsupported Writer Engine Version: {writer_engine_version}")
|
||||
|
||||
arrow_properties = arrow_props.build()
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ cdef cipher_to_name(ParquetCipher cipher):
|
||||
elif ParquetCipher_AES_GCM_CTR_V1 == cipher:
|
||||
return 'AES_GCM_CTR_V1'
|
||||
else:
|
||||
raise ValueError('Invalid cipher value: {0}'.format(cipher))
|
||||
raise ValueError(f'Invalid cipher value: {cipher}')
|
||||
|
||||
cdef class EncryptionConfiguration(_Weakrefable):
|
||||
"""Configuration of the encryption, such as which columns to encrypt"""
|
||||
@@ -100,7 +100,7 @@ cdef class EncryptionConfiguration(_Weakrefable):
|
||||
# to the string defined by the spec
|
||||
# 'key1: col1 , col2; key2: col3 , col4'
|
||||
column_keys = "; ".join(
|
||||
["{}: {}".format(k, ", ".join(v)) for k, v in value.items()])
|
||||
[f"{k}: {', '.join(v)}" for k, v in value.items()])
|
||||
self.configuration.get().column_keys = tobytes(column_keys)
|
||||
|
||||
@property
|
||||
@@ -340,8 +340,7 @@ cdef void _cb_create_kms_client(
|
||||
result = handler(connection_config)
|
||||
if not isinstance(result, KmsClient):
|
||||
raise TypeError(
|
||||
"callable must return KmsClient instances, but got {}".format(
|
||||
type(result)))
|
||||
f"callable must return KmsClient instances, but got {type(result)}")
|
||||
|
||||
out[0] = (<KmsClient> result).unwrap()
|
||||
|
||||
|
||||
@@ -307,8 +307,8 @@ def _perform_join_asof(left_operand, left_on, left_by,
|
||||
columns_collisions = set(left_operand.schema.names) & set(right_columns)
|
||||
if columns_collisions:
|
||||
raise ValueError(
|
||||
"Columns {} present in both tables. AsofJoin does not support "
|
||||
"column collisions.".format(columns_collisions),
|
||||
f"Columns {columns_collisions} present in both tables. "
|
||||
"AsofJoin does not support column collisions."
|
||||
)
|
||||
|
||||
# Add the join node to the execplan
|
||||
|
||||
+22
-23
@@ -340,11 +340,10 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
|
||||
if value_type is not None:
|
||||
warnings.warn(
|
||||
"The dtype of the 'categories' of the passed "
|
||||
"categorical values ({0}) does not match the "
|
||||
"specified type ({1}). For now ignoring the specified "
|
||||
f"categorical values ({values.categories.dtype}) does not match the "
|
||||
f"specified type ({value_type}). For now ignoring the specified "
|
||||
"type, but in the future this mismatch will raise a "
|
||||
"TypeError".format(
|
||||
values.categories.dtype, value_type),
|
||||
"TypeError",
|
||||
FutureWarning, stacklevel=2)
|
||||
dictionary = array(
|
||||
values.categories.values, memory_pool=memory_pool)
|
||||
@@ -1028,9 +1027,9 @@ cdef class Array(_PandasConvertible):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use one of "
|
||||
"the `pyarrow.Array.from_*` functions instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor "
|
||||
"directly, use one of the `pyarrow.Array.from_*` "
|
||||
"functions instead.")
|
||||
|
||||
cdef void init(self, const shared_ptr[CArray]& sp_array) except *:
|
||||
self.sp_array = sp_array
|
||||
@@ -1272,18 +1271,18 @@ cdef class Array(_PandasConvertible):
|
||||
|
||||
if type.num_fields != len(children):
|
||||
raise ValueError("Type's expected number of children "
|
||||
"({0}) did not match the passed number "
|
||||
"({1}).".format(type.num_fields, len(children)))
|
||||
f"({type.num_fields}) did not match the passed number "
|
||||
f"({len(children)})")
|
||||
|
||||
if type.has_variadic_buffers:
|
||||
if type.num_buffers > len(buffers):
|
||||
raise ValueError("Type's expected number of buffers is at least "
|
||||
"{0}, but the passed number is "
|
||||
"{1}.".format(type.num_buffers, len(buffers)))
|
||||
f"{type.num_buffers}, but the passed number is "
|
||||
f"{len(buffers)}.")
|
||||
elif type.num_buffers != len(buffers):
|
||||
raise ValueError("Type's expected number of buffers "
|
||||
"({0}) did not match the passed number "
|
||||
"({1}).".format(type.num_buffers, len(buffers)))
|
||||
f"({type.num_buffers}) did not match the passed number "
|
||||
f"({len(buffers)}).")
|
||||
|
||||
for buf in buffers:
|
||||
# None will produce a null buffer pointer
|
||||
@@ -1355,7 +1354,7 @@ cdef class Array(_PandasConvertible):
|
||||
|
||||
def __repr__(self):
|
||||
type_format = object.__repr__(self)
|
||||
return '{0}\n{1}'.format(type_format, str(self))
|
||||
return f'{type_format}\n{self}'
|
||||
|
||||
def to_string(self, *, int indent=2, int top_level_indent=0, int window=10,
|
||||
int container_window=2, c_bool skip_new_lines=False):
|
||||
@@ -3713,7 +3712,7 @@ cdef class UnionArray(Array):
|
||||
result = (<CUnionArray*> self.ap).field(pos)
|
||||
if result != NULL:
|
||||
return pyarrow_wrap_array(result)
|
||||
raise KeyError("UnionArray does not have child {}".format(pos))
|
||||
raise KeyError(f"UnionArray does not have child {pos}")
|
||||
|
||||
@property
|
||||
def type_codes(self):
|
||||
@@ -4382,8 +4381,8 @@ cdef class RunEndEncodedArray(Array):
|
||||
|
||||
if type.num_fields != len(children):
|
||||
raise ValueError("RunEndEncodedType's expected number of children "
|
||||
"({0}) did not match the passed number "
|
||||
"({1}).".format(type.num_fields, len(children)))
|
||||
f"({type.num_fields}) did not match the passed number "
|
||||
f"({len(children)})")
|
||||
|
||||
# buffers are validated as if we needed to pass them to C++, but
|
||||
# _make_from_arrays will take care of filling in the expected
|
||||
@@ -4395,13 +4394,13 @@ cdef class RunEndEncodedArray(Array):
|
||||
"bitmap, buffers[0] is not None")
|
||||
if type.num_buffers != len(buffers):
|
||||
raise ValueError("RunEndEncodedType's expected number of buffers "
|
||||
"({0}) did not match the passed number "
|
||||
"({1}).".format(type.num_buffers, len(buffers)))
|
||||
f"({type.num_buffers}) did not match the passed number "
|
||||
f"({len(buffers)}).")
|
||||
|
||||
# null_count is also validated as if we needed it
|
||||
if null_count != -1 and null_count != 0:
|
||||
raise ValueError("RunEndEncodedType's expected null_count (0) "
|
||||
"did not match passed number ({0})".format(null_count))
|
||||
f"did not match passed number ({null_count})")
|
||||
|
||||
return RunEndEncodedArray._from_arrays(type, False, length, children[0],
|
||||
children[1], offset)
|
||||
@@ -4484,8 +4483,8 @@ cdef class ExtensionArray(Array):
|
||||
shared_ptr[CExtensionArray] ext_array
|
||||
|
||||
if storage.type != typ.storage_type:
|
||||
raise TypeError("Incompatible storage type {0} "
|
||||
"for extension type {1}".format(storage.type, typ))
|
||||
raise TypeError(f"Incompatible storage type {storage.type} "
|
||||
f"for extension type {typ}")
|
||||
|
||||
ext_array = make_shared[CExtensionArray](typ.sp_type, storage.sp_array)
|
||||
cdef Array result = pyarrow_wrap_array(<shared_ptr[CArray]> ext_array)
|
||||
@@ -4963,7 +4962,7 @@ def concat_arrays(arrays, MemoryPool memory_pool=None):
|
||||
for array in arrays:
|
||||
if not isinstance(array, Array):
|
||||
raise TypeError("Iterable should contain Array objects, "
|
||||
"got {0} instead".format(type(array)))
|
||||
f"got {type(array)} instead")
|
||||
c_arrays.push_back(pyarrow_unwrap_array(array))
|
||||
|
||||
with nogil:
|
||||
|
||||
+14
-15
@@ -142,8 +142,7 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
|
||||
summary = cpp_doc.summary
|
||||
if not summary:
|
||||
arg_str = "arguments" if func.arity > 1 else "argument"
|
||||
summary = ("Call compute function {!r} with the given {}"
|
||||
.format(func.name, arg_str))
|
||||
summary = f"Call compute function {func.name!r} with the given {arg_str}"
|
||||
|
||||
doc_pieces.append(f"{summary}.\n\n")
|
||||
|
||||
@@ -183,11 +182,11 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
|
||||
f"does not have a docstring", RuntimeWarning)
|
||||
options_sig = inspect.signature(options_class)
|
||||
for p in options_sig.parameters.values():
|
||||
doc_pieces.append(dedent("""\
|
||||
{0} : optional
|
||||
Parameter for {1} constructor. Either `options`
|
||||
or `{0}` can be passed, but not both at the same time.
|
||||
""".format(p.name, options_class.__name__)))
|
||||
doc_pieces.append(dedent(f"""\
|
||||
{p.name} : optional
|
||||
Parameter for {options_class.__name__} constructor. Either `options`
|
||||
or `{p.name}` can be passed, but not both at the same time.
|
||||
"""))
|
||||
doc_pieces.append(dedent(f"""\
|
||||
options : pyarrow.compute.{options_class.__name__}, optional
|
||||
Alternative way of passing options.
|
||||
@@ -200,7 +199,8 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
|
||||
|
||||
# 4. Custom addition (e.g. examples)
|
||||
if doc_addition is not None:
|
||||
doc_pieces.append("\n{}\n".format(dedent(doc_addition).strip("\n")))
|
||||
stripped = dedent(doc_addition).strip('\n')
|
||||
doc_pieces.append(f"\n{stripped}\n")
|
||||
|
||||
wrapper.__doc__ = "".join(doc_pieces)
|
||||
return wrapper
|
||||
@@ -213,8 +213,8 @@ def _get_options_class(func):
|
||||
try:
|
||||
return globals()[class_name]
|
||||
except KeyError:
|
||||
warnings.warn("Python binding for {} not exposed"
|
||||
.format(class_name), RuntimeWarning)
|
||||
warnings.warn(f"Python binding for {class_name} not exposed",
|
||||
RuntimeWarning)
|
||||
return None
|
||||
|
||||
|
||||
@@ -222,9 +222,8 @@ def _handle_options(name, options_class, options, args, kwargs):
|
||||
if args or kwargs:
|
||||
if options is not None:
|
||||
raise TypeError(
|
||||
"Function {!r} called with both an 'options' argument "
|
||||
"and additional arguments"
|
||||
.format(name))
|
||||
f"Function {name!r} called with both an 'options' argument "
|
||||
f"and additional arguments")
|
||||
return options_class(*args, **kwargs)
|
||||
|
||||
if options is not None:
|
||||
@@ -233,8 +232,8 @@ def _handle_options(name, options_class, options, args, kwargs):
|
||||
elif isinstance(options, options_class):
|
||||
return options
|
||||
raise TypeError(
|
||||
"Function {!r} expected a {} parameter, got {}"
|
||||
.format(name, options_class, type(options)))
|
||||
f"Function {name!r} expected a {options_class} parameter, "
|
||||
f"got {type(options)}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -233,7 +233,7 @@ def pytest_ignore_collect(collection_path, config):
|
||||
|
||||
# handle cuda, flight, etc
|
||||
for group in doctest_groups:
|
||||
if 'pyarrow/{}'.format(group) in str(collection_path):
|
||||
if f'pyarrow/{group}' in str(collection_path):
|
||||
if not defaults[group]:
|
||||
return True
|
||||
|
||||
|
||||
+16
-21
@@ -114,7 +114,7 @@ def __getattr__(name):
|
||||
raise ImportError(_parquet_msg)
|
||||
|
||||
raise AttributeError(
|
||||
"module 'pyarrow.dataset' has no attribute '{0}'".format(name)
|
||||
f"module 'pyarrow.dataset' has no attribute '{name}'"
|
||||
)
|
||||
|
||||
|
||||
@@ -234,8 +234,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
|
||||
return DirectoryPartitioning.discover(field_names)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Expected list of field names, got {}".format(
|
||||
type(field_names)))
|
||||
f"Expected list of field names, got {type(field_names)}")
|
||||
else:
|
||||
raise ValueError(
|
||||
"For the default directory flavor, need to specify "
|
||||
@@ -253,8 +252,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
|
||||
return FilenamePartitioning.discover(field_names)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Expected list of field names, got {}".format(
|
||||
type(field_names)))
|
||||
f"Expected list of field names, got {type(field_names)}")
|
||||
else:
|
||||
raise ValueError(
|
||||
"For the filename flavor, need to specify "
|
||||
@@ -269,8 +267,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
|
||||
return HivePartitioning(schema, dictionaries)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Expected Schema for 'schema', got {}".format(
|
||||
type(schema)))
|
||||
f"Expected Schema for 'schema', got {type(schema)}")
|
||||
else:
|
||||
return HivePartitioning.discover()
|
||||
else:
|
||||
@@ -292,8 +289,8 @@ def _ensure_partitioning(scheme):
|
||||
elif isinstance(scheme, (Partitioning, PartitioningFactory)):
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Expected Partitioning or PartitioningFactory, got {}"
|
||||
.format(type(scheme)))
|
||||
raise ValueError(
|
||||
f"Expected Partitioning or PartitioningFactory, got {type(scheme)}")
|
||||
return scheme
|
||||
|
||||
|
||||
@@ -317,7 +314,7 @@ def _ensure_format(obj):
|
||||
elif obj == "json":
|
||||
return JsonFileFormat()
|
||||
else:
|
||||
raise ValueError("format '{}' is not supported".format(obj))
|
||||
raise ValueError(f"format '{obj}' is not supported")
|
||||
|
||||
|
||||
def _ensure_multiple_sources(paths, filesystem=None):
|
||||
@@ -382,16 +379,15 @@ def _ensure_multiple_sources(paths, filesystem=None):
|
||||
raise FileNotFoundError(info.path)
|
||||
elif file_type == FileType.Directory:
|
||||
raise IsADirectoryError(
|
||||
'Path {} points to a directory, but only file paths are '
|
||||
f'Path {info.path} points to a directory, but only file paths are '
|
||||
'supported. To construct a nested or union dataset pass '
|
||||
'a list of dataset objects instead.'.format(info.path)
|
||||
'a list of dataset objects instead.'
|
||||
)
|
||||
else:
|
||||
raise IOError(
|
||||
'Path {} exists but its type is unknown (could be a '
|
||||
f'Path {info.path} exists but its type is unknown (could be a '
|
||||
'special file such as a Unix socket or character device, '
|
||||
'or Windows NUL / CON / ...)'.format(info.path)
|
||||
)
|
||||
'or Windows NUL / CON / ...)')
|
||||
|
||||
return filesystem, paths
|
||||
|
||||
@@ -802,18 +798,18 @@ RecordBatch or Table, iterable of RecordBatch, RecordBatchReader, or URI
|
||||
return _in_memory_dataset(source, **kwargs)
|
||||
else:
|
||||
unique_types = set(type(elem).__name__ for elem in source)
|
||||
type_names = ', '.join('{}'.format(t) for t in unique_types)
|
||||
type_names = ', '.join(f'{t}' for t in unique_types)
|
||||
raise TypeError(
|
||||
'Expected a list of path-like or dataset objects, or a list '
|
||||
'of batches or tables. The given list contains the following '
|
||||
'types: {}'.format(type_names)
|
||||
f'types: {type_names}'
|
||||
)
|
||||
elif isinstance(source, (pa.RecordBatch, pa.Table)):
|
||||
return _in_memory_dataset(source, **kwargs)
|
||||
else:
|
||||
raise TypeError(
|
||||
'Expected a path-like, list of path-likes or a list of Datasets '
|
||||
'instead of the given type: {}'.format(type(source).__name__)
|
||||
f'instead of the given type: {type(source).__name__}'
|
||||
)
|
||||
|
||||
|
||||
@@ -987,9 +983,8 @@ Table/RecordBatch, or iterable of RecordBatch
|
||||
file_options = format.make_write_options()
|
||||
|
||||
if format != file_options.format:
|
||||
raise TypeError("Supplied FileWriteOptions have format {}, "
|
||||
"which doesn't match supplied FileFormat {}".format(
|
||||
format, file_options))
|
||||
raise TypeError(f"Supplied FileWriteOptions have format {format}, "
|
||||
f"which doesn't match supplied FileFormat {file_options}")
|
||||
|
||||
if basename_template is None:
|
||||
basename_template = "part-{i}." + format.default_extname
|
||||
|
||||
@@ -73,7 +73,7 @@ cdef class Device(_Weakrefable):
|
||||
return self.device.get().Equals(deref((<Device>other).device.get()))
|
||||
|
||||
def __repr__(self):
|
||||
return "<pyarrow.Device: {}>".format(frombytes(self.device.get().ToString()))
|
||||
return f"<pyarrow.Device: {frombytes(self.device.get().ToString())}>"
|
||||
|
||||
@property
|
||||
def type_name(self):
|
||||
@@ -137,9 +137,8 @@ cdef class MemoryManager(_Weakrefable):
|
||||
return self.memory_manager
|
||||
|
||||
def __repr__(self):
|
||||
return "<pyarrow.MemoryManager device: {}>".format(
|
||||
frombytes(self.memory_manager.get().device().get().ToString())
|
||||
)
|
||||
device_str = frombytes(self.memory_manager.get().device().get().ToString())
|
||||
return f"<pyarrow.MemoryManager device: {device_str}>"
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
|
||||
+11
-14
@@ -70,10 +70,8 @@ class FeatherDataset:
|
||||
|
||||
def validate_schemas(self, piece, table):
|
||||
if not self.schema.equals(table.schema):
|
||||
raise ValueError('Schema in {!s} was different. \n'
|
||||
'{!s}\n\nvs\n\n{!s}'
|
||||
.format(piece, self.schema,
|
||||
table.schema))
|
||||
raise ValueError(f'Schema in {piece} was different. \n'
|
||||
f'{self.schema}\n\nvs\n\n{table.schema}')
|
||||
|
||||
def read_pandas(self, columns=None, use_threads=True):
|
||||
"""
|
||||
@@ -100,14 +98,15 @@ def check_chunked_overflow(name, col):
|
||||
return
|
||||
|
||||
if col.type in (ext.binary(), ext.string()):
|
||||
raise ValueError("Column '{}' exceeds 2GB maximum capacity of "
|
||||
raise ValueError(f"Column '{name}' exceeds 2GB maximum capacity of "
|
||||
"a Feather binary column. This restriction may be "
|
||||
"lifted in the future".format(name))
|
||||
"lifted in the future")
|
||||
else:
|
||||
# TODO(wesm): Not sure when else this might be reached
|
||||
raise ValueError("Column '{}' of type {} was chunked on conversion "
|
||||
"to Arrow and cannot be currently written to "
|
||||
"Feather format".format(name, str(col.type)))
|
||||
raise ValueError(
|
||||
f"Column '{name}' of type {col.type} was chunked on conversion to Arrow "
|
||||
"and cannot be currently written to Feather format"
|
||||
)
|
||||
|
||||
|
||||
_FEATHER_SUPPORTED_CODECS = {'lz4', 'zstd', 'uncompressed'}
|
||||
@@ -180,9 +179,8 @@ def write_feather(df, dest, compression=None, compression_level=None,
|
||||
compression = 'lz4'
|
||||
elif (compression is not None and
|
||||
compression not in _FEATHER_SUPPORTED_CODECS):
|
||||
raise ValueError('compression="{}" not supported, must be '
|
||||
'one of {}'.format(compression,
|
||||
_FEATHER_SUPPORTED_CODECS))
|
||||
raise ValueError(f'compression="{compression}" not supported, must be '
|
||||
f'one of {_FEATHER_SUPPORTED_CODECS}')
|
||||
|
||||
try:
|
||||
_feather.write_feather(table, dest, compression=compression,
|
||||
@@ -268,8 +266,7 @@ def read_table(source, columns=None, memory_map=False, use_threads=True):
|
||||
else:
|
||||
column_type_names = [t.__name__ for t in column_types]
|
||||
raise TypeError("Columns must be indices or names. "
|
||||
"Got columns {} of types {}"
|
||||
.format(columns, column_type_names))
|
||||
f"Got columns {columns} of types {column_type_names}")
|
||||
|
||||
# Feather v1 already respects the column selection
|
||||
if reader.version < 3:
|
||||
|
||||
+10
-12
@@ -74,11 +74,11 @@ def __getattr__(name):
|
||||
if name in _not_imported:
|
||||
raise ImportError(
|
||||
"The pyarrow installation is not built with support for "
|
||||
"'{0}'".format(name)
|
||||
f"'{name}'"
|
||||
)
|
||||
|
||||
raise AttributeError(
|
||||
"module 'pyarrow.fs' has no attribute '{0}'".format(name)
|
||||
f"module 'pyarrow.fs' has no attribute '{name}'"
|
||||
)
|
||||
|
||||
|
||||
@@ -93,10 +93,9 @@ def _filesystem_from_str(uri):
|
||||
if prefix_info.type != FileType.Directory:
|
||||
raise ValueError(
|
||||
"The path component of the filesystem URI must point to a "
|
||||
"directory but it has a type: `{}`. The path component "
|
||||
"is `{}` and the given filesystem URI is `{}`".format(
|
||||
prefix_info.type.name, prefix_info.path, uri
|
||||
)
|
||||
f"directory but it has a type: `{prefix_info.type.name}`. The path "
|
||||
f"component is `{prefix_info.path}` and the given filesystem URI is "
|
||||
f"`{uri}`"
|
||||
)
|
||||
filesystem = SubTreeFileSystem(prefix, filesystem)
|
||||
return filesystem
|
||||
@@ -126,9 +125,8 @@ def _ensure_filesystem(filesystem, *, use_mmap=False):
|
||||
return PyFileSystem(FSSpecHandler(filesystem))
|
||||
|
||||
raise TypeError(
|
||||
"Unrecognized filesystem: {}. `filesystem` argument must be a "
|
||||
"FileSystem instance or a valid file system URI'".format(
|
||||
type(filesystem))
|
||||
f"Unrecognized filesystem: {type(filesystem)}. `filesystem` argument must be a "
|
||||
"FileSystem instance or a valid file system URI"
|
||||
)
|
||||
|
||||
|
||||
@@ -234,7 +232,7 @@ def copy_files(source, destination,
|
||||
Copy one file from S3 bucket to a local directory:
|
||||
|
||||
>>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
|
||||
... "file:///{}/index_copy.ndjson".format(local_path))
|
||||
... f"file:///{local_path}/index_copy.ndjson")
|
||||
|
||||
>>> fs.LocalFileSystem().get_file_info(str(local_path)+
|
||||
... '/index_copy.ndjson')
|
||||
@@ -243,7 +241,7 @@ def copy_files(source, destination,
|
||||
Copy file using a FileSystem object:
|
||||
|
||||
>>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
|
||||
... "file:///{}/index_copy.ndjson".format(local_path),
|
||||
... f"file:///{local_path}/index_copy.ndjson",
|
||||
... source_filesystem=fs.S3FileSystem())
|
||||
"""
|
||||
source_fs, source_path = _resolve_filesystem_and_path(
|
||||
@@ -297,7 +295,7 @@ class FSSpecHandler(FileSystemHandler):
|
||||
protocol = self.fs.protocol
|
||||
if isinstance(protocol, list):
|
||||
protocol = protocol[0]
|
||||
return "fsspec+{0}".format(protocol)
|
||||
return f"fsspec+{protocol}"
|
||||
|
||||
def normalize_path(self, path):
|
||||
return path
|
||||
|
||||
+15
-19
@@ -80,9 +80,8 @@ cdef class Node(_Weakrefable):
|
||||
shared_ptr[CNode] node
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use the "
|
||||
"TreeExprBuilder API directly"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use the "
|
||||
"TreeExprBuilder API directly")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CNode] node):
|
||||
@@ -95,7 +94,7 @@ cdef class Node(_Weakrefable):
|
||||
|
||||
def __repr__(self):
|
||||
type_format = object.__repr__(self)
|
||||
return '{0}\n{1}'.format(type_format, str(self))
|
||||
return f"{type_format}\n{self}"
|
||||
|
||||
def return_type(self):
|
||||
return pyarrow_wrap_data_type(self.node.get().return_type())
|
||||
@@ -113,7 +112,7 @@ cdef class Expression(_Weakrefable):
|
||||
|
||||
def __repr__(self):
|
||||
type_format = object.__repr__(self)
|
||||
return '{0}\n{1}'.format(type_format, str(self))
|
||||
return f"{type_format}\n{self}"
|
||||
|
||||
def root(self):
|
||||
return Node.create(self.expression.get().root())
|
||||
@@ -127,9 +126,8 @@ cdef class Condition(_Weakrefable):
|
||||
shared_ptr[CCondition] condition
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use the "
|
||||
"TreeExprBuilder API instead"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use the "
|
||||
"TreeExprBuilder API instead")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CCondition] condition):
|
||||
@@ -142,7 +140,7 @@ cdef class Condition(_Weakrefable):
|
||||
|
||||
def __repr__(self):
|
||||
type_format = object.__repr__(self)
|
||||
return '{0}\n{1}'.format(type_format, str(self))
|
||||
return f"{type_format}\n{self}"
|
||||
|
||||
def root(self):
|
||||
return Node.create(self.condition.get().root())
|
||||
@@ -156,8 +154,8 @@ cdef class SelectionVector(_Weakrefable):
|
||||
shared_ptr[CSelectionVector] selection_vector
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(
|
||||
f"Do not call {self.__class__.__name__}'s constructor directly.")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CSelectionVector] selection_vector):
|
||||
@@ -176,9 +174,8 @@ cdef class Projector(_Weakrefable):
|
||||
MemoryPool pool
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
"make_projector instead"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
|
||||
"make_projector instead")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CProjector] projector, MemoryPool pool):
|
||||
@@ -226,9 +223,8 @@ cdef class Filter(_Weakrefable):
|
||||
shared_ptr[CFilter] filter
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
"make_filter instead"
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
|
||||
"make_filter instead")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CFilter] filter):
|
||||
@@ -712,8 +708,8 @@ cdef class FunctionSignature(_Weakrefable):
|
||||
shared_ptr[CFunctionSignature] signature
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(
|
||||
f"Do not call {self.__class__.__name__}'s constructor directly.")
|
||||
|
||||
@staticmethod
|
||||
cdef create(shared_ptr[CFunctionSignature] signature):
|
||||
|
||||
@@ -81,7 +81,7 @@ cdef inline CSelectionVector_Mode _ensure_selection_mode(str name) except *:
|
||||
elif uppercase == 'UINT64':
|
||||
return CSelectionVector_Mode_UINT64
|
||||
else:
|
||||
raise ValueError('Invalid value for Selection Mode: {!r}'.format(name))
|
||||
raise ValueError(f'Invalid value for Selection Mode: {name!r}')
|
||||
|
||||
cdef inline str _selection_mode_name(CSelectionVector_Mode ctype):
|
||||
if ctype == CSelectionVector_Mode_NONE:
|
||||
|
||||
@@ -308,7 +308,7 @@ class _PyArrowColumn:
|
||||
kind = DtypeKind.DATETIME
|
||||
ts = dtype.unit[0]
|
||||
tz = dtype.tz if dtype.tz else ""
|
||||
f_string = "ts{ts}:{tz}".format(ts=ts, tz=tz)
|
||||
f_string = f"ts{ts}:{tz}"
|
||||
return kind, bit_width, f_string, Endianness.NATIVE
|
||||
elif pa.types.is_dictionary(dtype):
|
||||
kind = DtypeKind.CATEGORICAL
|
||||
|
||||
+13
-18
@@ -335,8 +335,7 @@ cdef class NativeFile(_Weakrefable):
|
||||
offset = offset + position
|
||||
else:
|
||||
with gil:
|
||||
raise ValueError("Invalid value of whence: {0}"
|
||||
.format(whence))
|
||||
raise ValueError(f"Invalid value of whence: {whence}")
|
||||
check_status(handle.get().Seek(offset))
|
||||
|
||||
return self.tell()
|
||||
@@ -702,8 +701,7 @@ cdef class NativeFile(_Weakrefable):
|
||||
# the passed buffer, so it's hard for us to avoid doubling the memory
|
||||
buf = <uint8_t*> malloc(buffer_size)
|
||||
if buf == NULL:
|
||||
raise MemoryError("Failed to allocate {0} bytes"
|
||||
.format(buffer_size))
|
||||
raise MemoryError(f"Failed to allocate {buffer_size} bytes")
|
||||
|
||||
writer_thread.start()
|
||||
|
||||
@@ -770,8 +768,7 @@ cdef class NativeFile(_Weakrefable):
|
||||
# the passed buffer, so it's hard for us to avoid doubling the memory
|
||||
buf = <uint8_t*> malloc(buffer_size)
|
||||
if buf == NULL:
|
||||
raise MemoryError("Failed to allocate {0} bytes"
|
||||
.format(buffer_size))
|
||||
raise MemoryError(f"Failed to allocate {buffer_size} bytes")
|
||||
|
||||
cdef int64_t total_bytes = 0
|
||||
cdef int32_t c_buffer_size = buffer_size
|
||||
@@ -951,7 +948,7 @@ cdef class PythonFile(NativeFile):
|
||||
elif inferred_mode.startswith('r'):
|
||||
kind = 'r'
|
||||
else:
|
||||
raise ValueError('Invalid file mode: {0}'.format(mode))
|
||||
raise ValueError(f'Invalid file mode: {mode}')
|
||||
|
||||
# If mode was given, check it matches the given file
|
||||
if mode is not None:
|
||||
@@ -1087,7 +1084,7 @@ cdef class MemoryMappedFile(NativeFile):
|
||||
self.is_readable = True
|
||||
self.is_writable = True
|
||||
else:
|
||||
raise ValueError('Invalid file mode: {0}'.format(mode))
|
||||
raise ValueError(f'Invalid file mode: {mode}')
|
||||
|
||||
with nogil:
|
||||
handle = GetResultValue(CMemoryMappedFile.Open(c_path, c_mode))
|
||||
@@ -1149,8 +1146,7 @@ def memory_map(path, mode='r'):
|
||||
|
||||
cdef _check_is_file(path):
|
||||
if os.path.isdir(path):
|
||||
raise IOError("Expected file path, but {0} is a directory"
|
||||
.format(path))
|
||||
raise IOError(f"Expected file path, but {path} is a directory")
|
||||
|
||||
|
||||
def create_memory_map(path, size):
|
||||
@@ -1247,7 +1243,7 @@ cdef class OSFile(NativeFile):
|
||||
elif mode in ('a', 'ab'):
|
||||
self._open_writable(c_path, append=True)
|
||||
else:
|
||||
raise ValueError('Invalid file mode: {0}'.format(mode))
|
||||
raise ValueError(f'Invalid file mode: {mode}')
|
||||
|
||||
cdef _open_readable(self, c_string path, CMemoryPool* pool):
|
||||
cdef shared_ptr[ReadableFile] handle
|
||||
@@ -2212,8 +2208,7 @@ cdef get_writer(object source, shared_ptr[COutputStream]* writer):
|
||||
nf = source
|
||||
writer[0] = nf.get_output_stream()
|
||||
else:
|
||||
raise TypeError('Unable to write to object of type: {0}'
|
||||
.format(type(source)))
|
||||
raise TypeError(f'Unable to write to object of type: {type(source)}')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
@@ -2248,7 +2243,7 @@ cdef CCompressionType _ensure_compression(str name) except *:
|
||||
elif uppercase == 'ZSTD':
|
||||
return CCompressionType_ZSTD
|
||||
else:
|
||||
raise ValueError('Invalid value for compression: {!r}'.format(name))
|
||||
raise ValueError(f'Invalid value for compression: {name!r}')
|
||||
|
||||
|
||||
cdef class CacheOptions(_Weakrefable):
|
||||
@@ -2810,8 +2805,8 @@ def input_stream(source, compression='detect', buffer_size=None):
|
||||
hasattr(source, 'closed')):
|
||||
stream = PythonFile(source, 'r')
|
||||
else:
|
||||
raise TypeError("pa.input_stream() called with instance of '{}'"
|
||||
.format(source.__class__))
|
||||
raise TypeError(
|
||||
f"pa.input_stream() called with instance of '{source.__class__}'")
|
||||
|
||||
if compression == 'detect':
|
||||
# detect for OSFile too
|
||||
@@ -2902,8 +2897,8 @@ def output_stream(source, compression='detect', buffer_size=None):
|
||||
hasattr(source, 'closed')):
|
||||
stream = PythonFile(source, 'w')
|
||||
else:
|
||||
raise TypeError("pa.output_stream() called with instance of '{}'"
|
||||
.format(source.__class__))
|
||||
raise TypeError(
|
||||
f"pa.output_stream() called with instance of '{source.__class__}'")
|
||||
|
||||
if compression == 'detect':
|
||||
compression = _detect_compression(source_path)
|
||||
|
||||
+13
-15
@@ -301,9 +301,8 @@ cdef class Message(_Weakrefable):
|
||||
pass
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
"`pyarrow.ipc.read_message` function instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
|
||||
"`pyarrow.ipc.read_message` function instead.")
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
@@ -393,9 +392,9 @@ cdef class Message(_Weakrefable):
|
||||
body_len = 0 if body is None else body.size
|
||||
|
||||
return """pyarrow.Message
|
||||
type: {0}
|
||||
metadata length: {1}
|
||||
body length: {2}""".format(self.type, metadata_len, body_len)
|
||||
type: {self.type}
|
||||
metadata length: {metadata_len}
|
||||
body length: {body_len}"""
|
||||
|
||||
|
||||
cdef class MessageReader(_Weakrefable):
|
||||
@@ -410,9 +409,9 @@ cdef class MessageReader(_Weakrefable):
|
||||
pass
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
|
||||
"`pyarrow.ipc.MessageReader.open_stream` function "
|
||||
"instead.".format(self.__class__.__name__))
|
||||
"instead.")
|
||||
|
||||
@staticmethod
|
||||
def open_stream(source):
|
||||
@@ -660,9 +659,8 @@ cdef class RecordBatchReader(_Weakrefable):
|
||||
# cdef block is in lib.pxd
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, "
|
||||
"use one of the RecordBatchReader.from_* functions instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use one of the RecordBatchReader.from_* functions instead.")
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
@@ -1122,7 +1120,7 @@ cdef class _RecordBatchFileReader(_Weakrefable):
|
||||
cdef shared_ptr[CRecordBatch] batch
|
||||
|
||||
if i < 0 or i >= self.num_record_batches:
|
||||
raise ValueError('Batch number {0} out of range'.format(i))
|
||||
raise ValueError(f'Batch number {i} out of range')
|
||||
|
||||
with nogil:
|
||||
batch = GetResultValue(self.reader.get().ReadRecordBatch(i))
|
||||
@@ -1152,7 +1150,7 @@ cdef class _RecordBatchFileReader(_Weakrefable):
|
||||
CRecordBatchWithMetadata batch_with_metadata
|
||||
|
||||
if i < 0 or i >= self.num_record_batches:
|
||||
raise ValueError('Batch number {0} out of range'.format(i))
|
||||
raise ValueError(f'Batch number {i} out of range')
|
||||
|
||||
with nogil:
|
||||
batch_with_metadata = GetResultValue(
|
||||
@@ -1274,8 +1272,8 @@ cdef NativeFile as_native_file(source):
|
||||
source = BufferReader(source)
|
||||
|
||||
if not isinstance(source, NativeFile):
|
||||
raise ValueError('Unable to read message from object with type: {0}'
|
||||
.format(type(source)))
|
||||
raise ValueError(
|
||||
f'Unable to read message from object with type: {type(source)}')
|
||||
return source
|
||||
|
||||
|
||||
|
||||
+12
-16
@@ -81,9 +81,9 @@ metadata : dict | pyarrow.KeyValueMetadata, optional
|
||||
|
||||
|
||||
class RecordBatchStreamWriter(lib._RecordBatchStreamWriter):
|
||||
__doc__ = """Writer for the Arrow streaming binary format
|
||||
__doc__ = f"""Writer for the Arrow streaming binary format
|
||||
|
||||
{}""".format(_ipc_writer_class_doc)
|
||||
{_ipc_writer_class_doc}"""
|
||||
|
||||
def __init__(self, sink, schema, *, options=None):
|
||||
options = _get_legacy_format_default(options)
|
||||
@@ -118,10 +118,9 @@ class RecordBatchFileReader(lib._RecordBatchFileReader):
|
||||
|
||||
class RecordBatchFileWriter(lib._RecordBatchFileWriter):
|
||||
|
||||
__doc__ = """Writer to create the Arrow binary file format
|
||||
__doc__ = f"""Writer to create the Arrow binary file format
|
||||
|
||||
{}
|
||||
""".format(_ipc_file_writer_class_doc)
|
||||
{_ipc_file_writer_class_doc}"""
|
||||
|
||||
def __init__(self, sink, schema, *, options=None, metadata=None):
|
||||
options = _get_legacy_format_default(options)
|
||||
@@ -131,8 +130,7 @@ class RecordBatchFileWriter(lib._RecordBatchFileWriter):
|
||||
def _get_legacy_format_default(options):
|
||||
if options:
|
||||
if not isinstance(options, IpcWriteOptions):
|
||||
raise TypeError("expected IpcWriteOptions, got {}"
|
||||
.format(type(options)))
|
||||
raise TypeError(f"expected IpcWriteOptions, got {type(options)}")
|
||||
return options
|
||||
|
||||
metadata_version = MetadataVersion.V5
|
||||
@@ -146,9 +144,7 @@ def _get_legacy_format_default(options):
|
||||
|
||||
def _ensure_default_ipc_read_options(options):
|
||||
if options and not isinstance(options, IpcReadOptions):
|
||||
raise TypeError(
|
||||
"expected IpcReadOptions, got {}".format(type(options))
|
||||
)
|
||||
raise TypeError(f"expected IpcReadOptions, got {type(options)}")
|
||||
return options or IpcReadOptions()
|
||||
|
||||
|
||||
@@ -157,16 +153,16 @@ def new_stream(sink, schema, *, options=None):
|
||||
options=options)
|
||||
|
||||
|
||||
new_stream.__doc__ = """\
|
||||
new_stream.__doc__ = f"""\
|
||||
Create an Arrow columnar IPC stream writer instance
|
||||
|
||||
{}
|
||||
{_ipc_writer_class_doc}
|
||||
|
||||
Returns
|
||||
-------
|
||||
writer : RecordBatchStreamWriter
|
||||
A writer for the given sink
|
||||
""".format(_ipc_writer_class_doc)
|
||||
"""
|
||||
|
||||
|
||||
def open_stream(source, *, options=None, memory_pool=None):
|
||||
@@ -196,16 +192,16 @@ def new_file(sink, schema, *, options=None, metadata=None):
|
||||
return RecordBatchFileWriter(sink, schema, options=options, metadata=metadata)
|
||||
|
||||
|
||||
new_file.__doc__ = """\
|
||||
new_file.__doc__ = f"""\
|
||||
Create an Arrow columnar IPC file writer instance
|
||||
|
||||
{}
|
||||
{_ipc_file_writer_class_doc}
|
||||
|
||||
Returns
|
||||
-------
|
||||
writer : RecordBatchFileWriter
|
||||
A writer for the given sink
|
||||
""".format(_ipc_file_writer_class_doc)
|
||||
"""
|
||||
|
||||
|
||||
def open_file(source, footer_offset=None, *, options=None, memory_pool=None):
|
||||
|
||||
@@ -239,7 +239,7 @@ def field(jvm_field):
|
||||
typ = pa.decimal128(jvm_type.getPrecision(), jvm_type.getScale())
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Unsupported JVM type: {}".format(type_str))
|
||||
f"Unsupported JVM type: {type_str}")
|
||||
else:
|
||||
# TODO: The following JVM types are not implemented:
|
||||
# Struct, List, FixedSizeList, Union, Dictionary
|
||||
@@ -295,8 +295,8 @@ def array(jvm_array):
|
||||
if jvm_array.getField().getType().isComplex():
|
||||
minor_type_str = jvm_array.getMinorType().toString()
|
||||
raise NotImplementedError(
|
||||
"Cannot convert JVM Arrow array of type {},"
|
||||
" complex types not yet implemented.".format(minor_type_str))
|
||||
f"Cannot convert JVM Arrow array of type {minor_type_str}, "
|
||||
"complex types not yet implemented.")
|
||||
dtype = field(jvm_array.getField()).type
|
||||
buffers = [jvm_buffer(buf)
|
||||
for buf in list(jvm_array.getBuffers(False))]
|
||||
|
||||
@@ -29,9 +29,8 @@ cdef class MemoryPool(_Weakrefable):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, "
|
||||
"use pyarrow.*_memory_pool instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use pyarrow.*_memory_pool instead.")
|
||||
|
||||
cdef void init(self, CMemoryPool* pool):
|
||||
self.pool = pool
|
||||
@@ -126,9 +125,8 @@ cdef class LoggingMemoryPool(MemoryPool):
|
||||
unique_ptr[CLoggingMemoryPool] logging_pool
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, "
|
||||
"use pyarrow.logging_memory_pool instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use pyarrow.logging_memory_pool instead.")
|
||||
|
||||
|
||||
cdef class ProxyMemoryPool(MemoryPool):
|
||||
@@ -141,9 +139,8 @@ cdef class ProxyMemoryPool(MemoryPool):
|
||||
unique_ptr[CProxyMemoryPool] proxy_pool
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, "
|
||||
"use pyarrow.proxy_memory_pool instead."
|
||||
.format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use pyarrow.proxy_memory_pool instead.")
|
||||
|
||||
|
||||
def default_memory_pool():
|
||||
|
||||
@@ -222,7 +222,7 @@ bloom_filter_fpp : double, default 0.05
|
||||
|
||||
|
||||
class ORCWriter:
|
||||
__doc__ = """
|
||||
__doc__ = f"""
|
||||
Writer interface for a single ORC file
|
||||
|
||||
Parameters
|
||||
@@ -231,8 +231,8 @@ where : str or pyarrow.io.NativeFile
|
||||
Writable target. For passing Python file objects or byte buffers,
|
||||
see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream
|
||||
or pyarrow.io.FixedSizeBufferWriter.
|
||||
{}
|
||||
""".format(_orc_writer_args_docs)
|
||||
{_orc_writer_args_docs}
|
||||
"""
|
||||
|
||||
is_open = False
|
||||
|
||||
@@ -369,7 +369,7 @@ def write_table(table, where, *,
|
||||
writer.write(table)
|
||||
|
||||
|
||||
write_table.__doc__ = """
|
||||
write_table.__doc__ = f"""
|
||||
Write a table into an ORC file.
|
||||
|
||||
Parameters
|
||||
@@ -380,5 +380,5 @@ where : str or pyarrow.io.NativeFile
|
||||
Writable target. For passing Python file objects or byte buffers,
|
||||
see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream
|
||||
or pyarrow.io.FixedSizeBufferWriter.
|
||||
{}
|
||||
""".format(_orc_writer_args_docs)
|
||||
{_orc_writer_args_docs}
|
||||
"""
|
||||
@@ -67,14 +67,16 @@ cdef class _PandasAPIShim(object):
|
||||
self._have_pandas = False
|
||||
if raise_:
|
||||
raise ImportError(
|
||||
"pyarrow requires pandas 1.0.0 or above, pandas {} is "
|
||||
"installed".format(self._version)
|
||||
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
|
||||
"installed"
|
||||
)
|
||||
else:
|
||||
warnings.warn(
|
||||
"pyarrow requires pandas 1.0.0 or above, pandas {} is "
|
||||
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
|
||||
"installed. Therefore, pandas-specific integration is not "
|
||||
"used.".format(self._version), stacklevel=2)
|
||||
"used.",
|
||||
stacklevel=2
|
||||
)
|
||||
return
|
||||
|
||||
self._is_v1 = self._loose_version < Version('2.0.0')
|
||||
|
||||
@@ -81,7 +81,7 @@ def get_logical_type(arrow_type):
|
||||
if isinstance(arrow_type, pa.lib.DictionaryType):
|
||||
return 'categorical'
|
||||
elif isinstance(arrow_type, pa.lib.ListType):
|
||||
return 'list[{}]'.format(get_logical_type(arrow_type.value_type))
|
||||
return f'list[{get_logical_type(arrow_type.value_type)}]'
|
||||
elif isinstance(arrow_type, pa.lib.TimestampType):
|
||||
return 'datetimetz' if arrow_type.tz is not None else 'datetime'
|
||||
elif pa.types.is_decimal(arrow_type):
|
||||
@@ -180,9 +180,8 @@ def get_column_metadata(column, name, arrow_type, field_name):
|
||||
and not isinstance(name, str)
|
||||
):
|
||||
raise TypeError(
|
||||
'Column name must be a string. Got column {} of type {}'.format(
|
||||
name, type(name).__name__
|
||||
)
|
||||
f"Column name must be a string. Got column {name} of type "
|
||||
f"{type(name).__name__}"
|
||||
)
|
||||
|
||||
assert isinstance(field_name, str), str(type(field_name))
|
||||
@@ -365,7 +364,7 @@ def _index_level_name(index, i, column_names):
|
||||
if index.name is not None and index.name not in column_names:
|
||||
return _column_name_to_strings(index.name)
|
||||
else:
|
||||
return '__index_level_{:d}__'.format(i)
|
||||
return f'__index_level_{i:d}__'
|
||||
|
||||
|
||||
def _get_columns_to_convert(df, schema, preserve_index, columns):
|
||||
@@ -373,7 +372,7 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
|
||||
|
||||
if not df.columns.is_unique:
|
||||
raise ValueError(
|
||||
'Duplicate column names found: {}'.format(list(df.columns))
|
||||
f'Duplicate column names found: {list(df.columns)}'
|
||||
)
|
||||
|
||||
if schema is not None:
|
||||
@@ -396,7 +395,7 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
|
||||
|
||||
if _pandas_api.is_sparse(col):
|
||||
raise TypeError(
|
||||
"Sparse pandas data (column {}) not supported.".format(name))
|
||||
f"Sparse pandas data (column {name}) not supported.")
|
||||
|
||||
columns_to_convert.append(col)
|
||||
convert_fields.append(None)
|
||||
@@ -458,27 +457,27 @@ def _get_columns_to_convert_given_schema(df, schema, preserve_index):
|
||||
except (KeyError, IndexError):
|
||||
# name not found as index level
|
||||
raise KeyError(
|
||||
"name '{}' present in the specified schema is not found "
|
||||
"in the columns or index".format(name))
|
||||
f"name '{name}' present in the specified schema is not found "
|
||||
"in the columns or index")
|
||||
if preserve_index is False:
|
||||
raise ValueError(
|
||||
"name '{}' present in the specified schema corresponds "
|
||||
f"name '{name}' present in the specified schema corresponds "
|
||||
"to the index, but 'preserve_index=False' was "
|
||||
"specified".format(name))
|
||||
"specified")
|
||||
elif (preserve_index is None and
|
||||
isinstance(col, _pandas_api.pd.RangeIndex)):
|
||||
raise ValueError(
|
||||
"name '{}' is present in the schema, but it is a "
|
||||
f"name '{name}' is present in the schema, but it is a "
|
||||
"RangeIndex which will not be converted as a column "
|
||||
"in the Table, but saved as metadata-only not in "
|
||||
"columns. Specify 'preserve_index=True' to force it "
|
||||
"being added as a column, or remove it from the "
|
||||
"specified schema".format(name))
|
||||
"specified schema")
|
||||
is_index = True
|
||||
|
||||
if _pandas_api.is_sparse(col):
|
||||
raise TypeError(
|
||||
"Sparse pandas data (column {}) not supported.".format(name))
|
||||
f"Sparse pandas data (column {name}) not supported.")
|
||||
|
||||
field = schema.field(name)
|
||||
columns_to_convert.append(col)
|
||||
@@ -621,13 +620,12 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None,
|
||||
except (pa.ArrowInvalid,
|
||||
pa.ArrowNotImplementedError,
|
||||
pa.ArrowTypeError) as e:
|
||||
e.args += ("Conversion failed for column {!s} with type {!s}"
|
||||
.format(col.name, col.dtype),)
|
||||
e.args += (
|
||||
f"Conversion failed for column {col.name} with type {col.dtype}",)
|
||||
raise e
|
||||
if not field_nullable and result.null_count > 0:
|
||||
raise ValueError("Field {} was non-nullable but pandas column "
|
||||
"had {} null values".format(str(field),
|
||||
result.null_count))
|
||||
raise ValueError(f"Field {field} was non-nullable but pandas column "
|
||||
f"had {result.null_count} null values")
|
||||
return result
|
||||
|
||||
def _can_definitely_zero_copy(arr):
|
||||
@@ -1004,8 +1002,7 @@ def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None)
|
||||
# Possibly the result of munged metadata
|
||||
continue
|
||||
else:
|
||||
raise ValueError("Unrecognized index kind: {}"
|
||||
.format(descr['kind']))
|
||||
raise ValueError(f"Unrecognized index kind: {descr['kind']}")
|
||||
index_arrays.append(index_level)
|
||||
index_names.append(index_name)
|
||||
|
||||
|
||||
@@ -181,9 +181,7 @@ def filters_to_expression(filters):
|
||||
elif op == 'not in':
|
||||
return ~field.isin(val)
|
||||
else:
|
||||
raise ValueError(
|
||||
'"{0}" is not a valid operator in predicates.'.format(
|
||||
(col, op, val)))
|
||||
raise ValueError(f'"{col}" is not a valid operator in predicates.')
|
||||
|
||||
disjunction_members = []
|
||||
|
||||
@@ -949,14 +947,14 @@ and write the RecordBatch into the Parquet file:
|
||||
|
||||
class ParquetWriter:
|
||||
|
||||
__doc__ = """
|
||||
__doc__ = f"""
|
||||
Class for incrementally building a Parquet file for Arrow tables.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
where : path or file-like object
|
||||
schema : pyarrow.Schema
|
||||
{}
|
||||
{_parquet_writer_arg_docs}
|
||||
writer_engine_version : unused
|
||||
**options : dict
|
||||
If options contains a key `metadata_collector` then the
|
||||
@@ -966,8 +964,8 @@ writer_engine_version : unused
|
||||
|
||||
Examples
|
||||
--------
|
||||
{}
|
||||
""".format(_parquet_writer_arg_docs, _parquet_writer_example_doc)
|
||||
{_parquet_writer_example_doc}
|
||||
"""
|
||||
|
||||
def __init__(self, where, schema, filesystem=None,
|
||||
flavor=None,
|
||||
@@ -1112,9 +1110,10 @@ Examples
|
||||
assert self.is_open
|
||||
|
||||
if not table.schema.equals(self.schema, check_metadata=False):
|
||||
msg = ('Table schema does not match schema used to create file: '
|
||||
'\ntable:\n{!s} vs. \nfile:\n{!s}'
|
||||
.format(table.schema, self.schema))
|
||||
msg = (
|
||||
"Table schema does not match schema used to create file: \n"
|
||||
f"table:\n{table.schema!s} vs. \nfile:\n{self.schema!s}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
self.writer.write_table(table, row_group_size=row_group_size)
|
||||
@@ -1215,7 +1214,7 @@ create a ParquetDataset object with filter:
|
||||
|
||||
|
||||
class ParquetDataset:
|
||||
__doc__ = """
|
||||
__doc__ = f"""
|
||||
Encapsulates details of reading a complete Parquet dataset possibly
|
||||
consisting of multiple files and partitions in subdirectories.
|
||||
|
||||
@@ -1236,8 +1235,8 @@ filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], defaul
|
||||
exploited to avoid loading files at all if they contain no matching rows.
|
||||
Within-file level filtering and different partitioning schemes are supported.
|
||||
|
||||
{1}
|
||||
{0}
|
||||
{_DNF_filter_doc}
|
||||
{_read_docstring_common}
|
||||
ignore_prefixes : list, optional
|
||||
Files matching any of these prefixes will be ignored by the
|
||||
discovery process.
|
||||
@@ -1276,8 +1275,8 @@ arrow_extensions_enabled : bool, default False
|
||||
|
||||
Examples
|
||||
--------
|
||||
{2}
|
||||
""".format(_read_docstring_common, _DNF_filter_doc, _parquet_dataset_example)
|
||||
{_parquet_dataset_example}
|
||||
"""
|
||||
|
||||
def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None,
|
||||
read_dictionary=None, memory_map=False, buffer_size=None,
|
||||
@@ -1966,7 +1965,7 @@ Defining column encoding per-column:
|
||||
... use_dictionary=False)
|
||||
"""
|
||||
|
||||
write_table.__doc__ = """
|
||||
write_table.__doc__ = f"""
|
||||
Write a Table to Parquet format.
|
||||
|
||||
Parameters
|
||||
@@ -1977,14 +1976,14 @@ row_group_size : int
|
||||
Maximum number of rows in each written row group. If None, the
|
||||
row group size will be the minimum of the Table size and
|
||||
1024 * 1024.
|
||||
{}
|
||||
{_parquet_writer_arg_docs}
|
||||
**kwargs : optional
|
||||
Additional options for ParquetWriter
|
||||
|
||||
Examples
|
||||
--------
|
||||
{}
|
||||
""".format(_parquet_writer_arg_docs, _write_table_example)
|
||||
{_write_table_example}
|
||||
"""
|
||||
|
||||
|
||||
def write_to_dataset(table, root_path, partition_cols=None,
|
||||
|
||||
+11
-18
@@ -26,8 +26,8 @@ cdef class Scalar(_Weakrefable):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise TypeError("Do not call {}'s constructor directly, use "
|
||||
"pa.scalar() instead.".format(self.__class__.__name__))
|
||||
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
|
||||
"use pa.scalar() instead.")
|
||||
|
||||
cdef void init(self, const shared_ptr[CScalar]& wrapped):
|
||||
self.wrapped = wrapped
|
||||
@@ -117,9 +117,7 @@ cdef class Scalar(_Weakrefable):
|
||||
check_status(self.wrapped.get().Validate())
|
||||
|
||||
def __repr__(self):
|
||||
return '<pyarrow.{}: {!r}>'.format(
|
||||
self.__class__.__name__, self.as_py()
|
||||
)
|
||||
return f'<pyarrow.{self.__class__.__name__}: {self.as_py()!r}>'
|
||||
|
||||
def __str__(self):
|
||||
return str(self.as_py())
|
||||
@@ -611,10 +609,10 @@ def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
|
||||
# otherwise safely truncate to microsecond resolution datetime
|
||||
if value % 1000 != 0:
|
||||
raise ValueError(
|
||||
"Nanosecond resolution temporal type {} is not safely "
|
||||
f"Nanosecond resolution temporal type {value} is not safely "
|
||||
"convertible to microseconds to convert to datetime.datetime. "
|
||||
"Install pandas to return as Timestamp with nanosecond "
|
||||
"support or access the .value attribute.".format(value)
|
||||
"support or access the .value attribute."
|
||||
)
|
||||
delta = datetime.timedelta(microseconds=value // 1000)
|
||||
|
||||
@@ -735,9 +733,7 @@ cdef class TimestampScalar(Scalar):
|
||||
type_format = str(_pc().strftime(self, format="%Y-%m-%dT%H:%M:%S%z"))
|
||||
else:
|
||||
type_format = str(_pc().strftime(self))
|
||||
return '<pyarrow.{}: {!r}>'.format(
|
||||
self.__class__.__name__, type_format
|
||||
)
|
||||
return f'<pyarrow.{self.__class__.__name__}: {type_format!r}>'
|
||||
|
||||
|
||||
cdef class DurationScalar(Scalar):
|
||||
@@ -783,10 +779,10 @@ cdef class DurationScalar(Scalar):
|
||||
# otherwise safely truncate to microsecond resolution timedelta
|
||||
if sp.value % 1000 != 0:
|
||||
raise ValueError(
|
||||
"Nanosecond duration {} is not safely convertible to "
|
||||
f"Nanosecond duration {sp.value} is not safely convertible to "
|
||||
"microseconds to convert to datetime.timedelta. Install "
|
||||
"pandas to return as Timedelta with nanosecond support or "
|
||||
"access the .value attribute.".format(sp.value)
|
||||
"access the .value attribute."
|
||||
)
|
||||
return datetime.timedelta(microseconds=sp.value // 1000)
|
||||
|
||||
@@ -1049,9 +1045,7 @@ cdef class StructScalar(Scalar, collections.abc.Mapping):
|
||||
return None
|
||||
|
||||
def __repr__(self):
|
||||
return '<pyarrow.{}: {!r}>'.format(
|
||||
self.__class__.__name__, self._as_py_tuple()
|
||||
)
|
||||
return f'<pyarrow.{self.__class__.__name__}: {self._as_py_tuple()!r}>'
|
||||
|
||||
def __str__(self):
|
||||
return str(self._as_py_tuple())
|
||||
@@ -1356,9 +1350,8 @@ cdef class ExtensionScalar(Scalar):
|
||||
storage = None
|
||||
elif isinstance(value, Scalar):
|
||||
if value.type != typ.storage_type:
|
||||
raise TypeError("Incompatible storage type {0} "
|
||||
"for extension type {1}"
|
||||
.format(value.type, typ))
|
||||
raise TypeError(f"Incompatible storage type {value.type} "
|
||||
f"for extension type {typ}")
|
||||
storage = value
|
||||
else:
|
||||
storage = scalar(value, typ.storage_type)
|
||||
|
||||
+23
-28
@@ -113,7 +113,7 @@ cdef class ChunkedArray(_PandasConvertible):
|
||||
|
||||
def __repr__(self):
|
||||
type_format = object.__repr__(self)
|
||||
return '{0}\n{1}'.format(type_format, str(self))
|
||||
return f"{type_format}\n{self}"
|
||||
|
||||
def to_string(self, *, int indent=0, int window=5, int container_window=2,
|
||||
c_bool skip_new_lines=False):
|
||||
@@ -1568,8 +1568,8 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
|
||||
schema.reset(new CSchema(c_fields, c_meta))
|
||||
return arrays
|
||||
else:
|
||||
raise ValueError('Length of names ({}) does not match '
|
||||
'length of arrays ({})'.format(len(names), K))
|
||||
raise ValueError(f'Length of names ({len(names)}) does not match '
|
||||
f'length of arrays ({K})')
|
||||
|
||||
c_fields.resize(K)
|
||||
|
||||
@@ -1578,8 +1578,8 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
|
||||
'Table or RecordBatch.')
|
||||
|
||||
if len(names) != K:
|
||||
raise ValueError('Length of names ({}) does not match '
|
||||
'length of arrays ({})'.format(len(names), K))
|
||||
raise ValueError(f'Length of names ({len(names)}) does not match '
|
||||
f'length of arrays ({K})')
|
||||
|
||||
converted_arrays = []
|
||||
for i in range(K):
|
||||
@@ -1726,11 +1726,10 @@ cdef class _Tabular(_PandasConvertible):
|
||||
field_indices = self.schema.get_all_field_indices(i)
|
||||
|
||||
if len(field_indices) == 0:
|
||||
raise KeyError("Field \"{}\" does not exist in schema"
|
||||
.format(i))
|
||||
raise KeyError(f'Field "{i}" does not exist in schema')
|
||||
elif len(field_indices) > 1:
|
||||
raise KeyError("Field \"{}\" exists {} times in schema"
|
||||
.format(i, len(field_indices)))
|
||||
raise KeyError(
|
||||
f'Field "{i}" exists {len(field_indices)} times in schema')
|
||||
else:
|
||||
return field_indices[0]
|
||||
elif isinstance(i, int):
|
||||
@@ -2368,15 +2367,13 @@ cdef class _Tabular(_PandasConvertible):
|
||||
show_field_metadata=show_metadata,
|
||||
show_schema_metadata=show_metadata
|
||||
)
|
||||
title = 'pyarrow.{}\n{}'.format(type(self).__name__, schema_as_string)
|
||||
title = f'pyarrow.{type(self).__name__}\n{schema_as_string}'
|
||||
pieces = [title]
|
||||
if preview_cols:
|
||||
pieces.append('----')
|
||||
for i in range(min(self.num_columns, preview_cols)):
|
||||
pieces.append('{}: {}'.format(
|
||||
self.field(i).name,
|
||||
self.column(i).to_string(indent=0, skip_new_lines=True)
|
||||
))
|
||||
pieces.append(
|
||||
f'{self.field(i).name}: {self.column(i).to_string(indent=0, skip_new_lines=True)}')
|
||||
if preview_cols < self.num_columns:
|
||||
pieces.append('...')
|
||||
return '\n'.join(pieces)
|
||||
@@ -2436,7 +2433,7 @@ cdef class _Tabular(_PandasConvertible):
|
||||
for col in columns:
|
||||
idx = self.schema.get_field_index(col)
|
||||
if idx == -1:
|
||||
raise KeyError("Column {!r} not found".format(col))
|
||||
raise KeyError(f"Column {col!r} not found")
|
||||
indices.append(idx)
|
||||
|
||||
indices.sort()
|
||||
@@ -3072,7 +3069,7 @@ cdef class RecordBatch(_Tabular):
|
||||
indices = self.schema.get_all_field_indices(name)
|
||||
|
||||
if not indices:
|
||||
raise KeyError("Column {!r} not found".format(name))
|
||||
raise KeyError(f"Column {name!r} not found")
|
||||
|
||||
for index in indices:
|
||||
idx_to_new_name[index] = new_name
|
||||
@@ -3347,14 +3344,13 @@ cdef class RecordBatch(_Tabular):
|
||||
list newcols = []
|
||||
|
||||
if self.schema.names != target_schema.names:
|
||||
raise ValueError("Target schema's field names are not matching "
|
||||
"the record batch's field names: {!r}, {!r}"
|
||||
.format(self.schema.names, target_schema.names))
|
||||
raise ValueError(f"Target schema's field names are not matching "
|
||||
f"the record batch's field names: {self.schema.names!r}, {target_schema.names!r}")
|
||||
|
||||
for column, field in zip(self.itercolumns(), target_schema):
|
||||
if not field.nullable and column.null_count > 0:
|
||||
raise ValueError("Casting field {!r} with null values to non-nullable"
|
||||
.format(field.name))
|
||||
raise ValueError(
|
||||
f"Casting field {field.name!r} with null values to non-nullable")
|
||||
casted = column.cast(field.type, safe=safe, options=options)
|
||||
newcols.append(casted)
|
||||
|
||||
@@ -3550,7 +3546,7 @@ cdef class RecordBatch(_Tabular):
|
||||
for arr in converted_arrays:
|
||||
if len(arr) != num_rows:
|
||||
raise ValueError('Arrays were not all the same length: '
|
||||
'{0} vs {1}'.format(len(arr), num_rows))
|
||||
f'{len(arr)} vs {num_rows}')
|
||||
c_arrays.push_back(arr.sp_array)
|
||||
|
||||
result = pyarrow_wrap_batch(CRecordBatch.Make(c_schema, num_rows,
|
||||
@@ -4714,14 +4710,13 @@ cdef class Table(_Tabular):
|
||||
list newcols = []
|
||||
|
||||
if self.schema.names != target_schema.names:
|
||||
raise ValueError("Target schema's field names are not matching "
|
||||
"the table's field names: {!r}, {!r}"
|
||||
.format(self.schema.names, target_schema.names))
|
||||
raise ValueError(f"Target schema's field names are not matching "
|
||||
f"the table's field names: {self.schema.names!r}, {target_schema.names!r}")
|
||||
|
||||
for column, field in zip(self.itercolumns(), target_schema):
|
||||
if not field.nullable and column.null_count > 0:
|
||||
raise ValueError("Casting field {!r} with null values to non-nullable"
|
||||
.format(field.name))
|
||||
raise ValueError(
|
||||
f"Casting field {field.name!r} with null values to non-nullable")
|
||||
casted = column.cast(field.type, safe=safe, options=options)
|
||||
newcols.append(casted)
|
||||
|
||||
@@ -5558,7 +5553,7 @@ cdef class Table(_Tabular):
|
||||
indices = self.schema.get_all_field_indices(name)
|
||||
|
||||
if not indices:
|
||||
raise KeyError("Column {!r} not found".format(name))
|
||||
raise KeyError(f"Column {name!r} not found")
|
||||
|
||||
for index in indices:
|
||||
idx_to_new_name[index] = new_name
|
||||
|
||||
+25
-25
@@ -53,10 +53,10 @@ cdef class Tensor(_Weakrefable):
|
||||
return _struct.pack(f"{len(values)}n", *values)
|
||||
|
||||
def __repr__(self):
|
||||
return """<pyarrow.Tensor>
|
||||
type: {0.type}
|
||||
shape: {0.shape}
|
||||
strides: {0.strides}""".format(self)
|
||||
return f"""<pyarrow.Tensor>
|
||||
type: {self.type}
|
||||
shape: {self.shape}
|
||||
strides: {self.strides}"""
|
||||
|
||||
@staticmethod
|
||||
def from_numpy(obj, dim_names=None):
|
||||
@@ -321,8 +321,8 @@ cdef class SparseCOOTensor(_Weakrefable):
|
||||
|
||||
def __repr__(self):
|
||||
return """<pyarrow.SparseCOOTensor>
|
||||
type: {0.type}
|
||||
shape: {0.shape}""".format(self)
|
||||
type: {self.type}
|
||||
shape: {self.shape}"""
|
||||
|
||||
@classmethod
|
||||
def from_dense_numpy(cls, obj, dim_names=None):
|
||||
@@ -394,7 +394,7 @@ shape: {0.shape}""".format(self)
|
||||
import scipy.sparse
|
||||
if not isinstance(obj, scipy.sparse.coo_matrix):
|
||||
raise TypeError(
|
||||
"Expected scipy.sparse.coo_matrix, got {}".format(type(obj)))
|
||||
f"Expected scipy.sparse.coo_matrix, got {type(obj)}")
|
||||
|
||||
cdef shared_ptr[CSparseCOOTensor] csparse_tensor
|
||||
cdef vector[int64_t] c_shape
|
||||
@@ -440,7 +440,7 @@ shape: {0.shape}""".format(self)
|
||||
import sparse
|
||||
if not isinstance(obj, sparse.COO):
|
||||
raise TypeError(
|
||||
"Expected sparse.COO, got {}".format(type(obj)))
|
||||
f"Expected sparse.COO, got {type(obj)}")
|
||||
|
||||
cdef shared_ptr[CSparseCOOTensor] csparse_tensor
|
||||
cdef vector[int64_t] c_shape
|
||||
@@ -624,9 +624,9 @@ cdef class SparseCSRMatrix(_Weakrefable):
|
||||
self.type = pyarrow_wrap_data_type(self.stp.type())
|
||||
|
||||
def __repr__(self):
|
||||
return """<pyarrow.SparseCSRMatrix>
|
||||
type: {0.type}
|
||||
shape: {0.shape}""".format(self)
|
||||
return f"""<pyarrow.SparseCSRMatrix>
|
||||
type: {self.type}
|
||||
shape: {self.shape}"""
|
||||
|
||||
@classmethod
|
||||
def from_dense_numpy(cls, obj, dim_names=None):
|
||||
@@ -705,7 +705,7 @@ shape: {0.shape}""".format(self)
|
||||
import scipy.sparse
|
||||
if not isinstance(obj, scipy.sparse.csr_matrix):
|
||||
raise TypeError(
|
||||
"Expected scipy.sparse.csr_matrix, got {}".format(type(obj)))
|
||||
f"Expected scipy.sparse.csr_matrix, got {type(obj)}")
|
||||
|
||||
cdef shared_ptr[CSparseCSRMatrix] csparse_tensor
|
||||
cdef vector[int64_t] c_shape
|
||||
@@ -865,9 +865,9 @@ cdef class SparseCSCMatrix(_Weakrefable):
|
||||
self.type = pyarrow_wrap_data_type(self.stp.type())
|
||||
|
||||
def __repr__(self):
|
||||
return """<pyarrow.SparseCSCMatrix>
|
||||
type: {0.type}
|
||||
shape: {0.shape}""".format(self)
|
||||
return f"""<pyarrow.SparseCSCMatrix>
|
||||
type: {self.type}
|
||||
shape: {self.shape}"""
|
||||
|
||||
@classmethod
|
||||
def from_dense_numpy(cls, obj, dim_names=None):
|
||||
@@ -946,7 +946,7 @@ shape: {0.shape}""".format(self)
|
||||
import scipy.sparse
|
||||
if not isinstance(obj, scipy.sparse.csc_matrix):
|
||||
raise TypeError(
|
||||
"Expected scipy.sparse.csc_matrix, got {}".format(type(obj)))
|
||||
f"Expected scipy.sparse.csc_matrix, got {type(obj)}")
|
||||
|
||||
cdef shared_ptr[CSparseCSCMatrix] csparse_tensor
|
||||
cdef vector[int64_t] c_shape
|
||||
@@ -1115,9 +1115,9 @@ cdef class SparseCSFTensor(_Weakrefable):
|
||||
self.type = pyarrow_wrap_data_type(self.stp.type())
|
||||
|
||||
def __repr__(self):
|
||||
return """<pyarrow.SparseCSFTensor>
|
||||
type: {0.type}
|
||||
shape: {0.shape}""".format(self)
|
||||
return f"""<pyarrow.SparseCSFTensor>
|
||||
type: {self.type}
|
||||
shape: {self.shape}"""
|
||||
|
||||
@classmethod
|
||||
def from_dense_numpy(cls, obj, dim_names=None):
|
||||
@@ -1183,14 +1183,14 @@ shape: {0.shape}""".format(self)
|
||||
# Enforce preconditions for SparseCSFTensor indices
|
||||
if not (isinstance(indptr, (list, tuple)) and
|
||||
isinstance(indices, (list, tuple))):
|
||||
raise TypeError("Expected list or tuple, got {}, {}"
|
||||
.format(type(indptr), type(indices)))
|
||||
raise TypeError(
|
||||
f"Expected list or tuple, got {type(indptr)}, {type(indices)}")
|
||||
if len(indptr) != len(shape) - 1:
|
||||
raise ValueError("Expected list of {ndim} np.arrays for "
|
||||
"SparseCSFTensor.indptr".format(ndim=len(shape)))
|
||||
raise ValueError(f"Expected list of {len(shape)} np.arrays for "
|
||||
"SparseCSFTensor.indptr")
|
||||
if len(indices) != len(shape):
|
||||
raise ValueError("Expected list of {ndim} np.arrays for "
|
||||
"SparseCSFTensor.indices".format(ndim=len(shape)))
|
||||
raise ValueError(f"Expected list of {len(shape)} np.arrays for "
|
||||
"SparseCSFTensor.indices")
|
||||
if any([x.ndim != 1 for x in indptr]):
|
||||
raise ValueError("Expected a list of 1-dimensional arrays for "
|
||||
"SparseCSFTensor.indptr")
|
||||
|
||||
@@ -83,17 +83,16 @@ def pytest_addoption(parser):
|
||||
elif value in {'0', 'false', 'off', 'no', 'n'}:
|
||||
return False
|
||||
else:
|
||||
raise ValueError('{}={} is not parsable as boolean'
|
||||
.format(name.upper(), value))
|
||||
raise ValueError(f'{name.upper()}={value} is not parsable as boolean')
|
||||
|
||||
for group in groups:
|
||||
default = bool_env('PYARROW_TEST_{}'.format(group), defaults[group])
|
||||
parser.addoption('--enable-{}'.format(group),
|
||||
default = bool_env(f'PYARROW_TEST_{group}', defaults[group])
|
||||
parser.addoption(f'--enable-{group}',
|
||||
action='store_true', default=default,
|
||||
help=('Enable the {} test group'.format(group)))
|
||||
parser.addoption('--disable-{}'.format(group),
|
||||
help=(f'Enable the {group} test group'))
|
||||
parser.addoption(f'--disable-{group}',
|
||||
action='store_true', default=False,
|
||||
help=('Disable the {} test group'.format(group)))
|
||||
help=(f'Disable the {group} test group'))
|
||||
|
||||
|
||||
class PyArrowConfig:
|
||||
@@ -107,7 +106,7 @@ class PyArrowConfig:
|
||||
|
||||
def requires(self, group):
|
||||
if not self.is_enabled[group]:
|
||||
pytest.skip('{} NOT enabled'.format(group))
|
||||
pytest.skip(f'{group} NOT enabled')
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
@@ -119,8 +118,8 @@ def pytest_configure(config):
|
||||
"markers", mark,
|
||||
)
|
||||
|
||||
enable_flag = '--enable-{}'.format(mark)
|
||||
disable_flag = '--disable-{}'.format(mark)
|
||||
enable_flag = f'--enable-{mark}'
|
||||
disable_flag = f'--disable-{mark}'
|
||||
|
||||
is_enabled = (config.getoption(enable_flag) and not
|
||||
config.getoption(disable_flag))
|
||||
@@ -217,7 +216,7 @@ def s3_server(s3_connection, tmpdir_factory):
|
||||
tmpdir = tmpdir_factory.getbasetemp()
|
||||
host, port, access_key, secret_key = s3_connection
|
||||
|
||||
address = '{}:{}'.format(host, port)
|
||||
address = f'{host}:{port}'
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
'MINIO_ACCESS_KEY': access_key,
|
||||
|
||||
@@ -157,7 +157,7 @@ def dataframe_with_lists(include_index=False, parquet_compatible=False):
|
||||
]
|
||||
|
||||
for value_type, data in temporal_pairs:
|
||||
field_name = '{}_list'.format(value_type)
|
||||
field_name = f'{value_type}_list'
|
||||
field_type = pa.list_(value_type)
|
||||
field = pa.field(field_name, field_type)
|
||||
fields.append(field)
|
||||
|
||||
@@ -49,7 +49,7 @@ def s3_bucket(s3_server):
|
||||
host, port, access_key, secret_key = s3_server['connection']
|
||||
s3_client = boto3.client(
|
||||
's3',
|
||||
endpoint_url='http://{}:{}'.format(host, port),
|
||||
endpoint_url=f'http://{host}:{port}',
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=botocore.client.Config(signature_version='s3v4'),
|
||||
@@ -75,11 +75,11 @@ def s3_example_s3fs(s3_server, s3_bucket):
|
||||
key=access_key,
|
||||
secret=secret_key,
|
||||
client_kwargs={
|
||||
'endpoint_url': 'http://{}:{}'.format(host, port)
|
||||
'endpoint_url': f'http://{host}:{port}'
|
||||
}
|
||||
)
|
||||
|
||||
test_path = '{}/{}'.format(s3_bucket, guid())
|
||||
test_path = f'{s3_bucket}/{guid()}'
|
||||
|
||||
fs.mkdir(test_path)
|
||||
yield fs, test_path
|
||||
@@ -95,9 +95,8 @@ def s3_example_fs(s3_server):
|
||||
|
||||
host, port, access_key, secret_key = s3_server['connection']
|
||||
uri = (
|
||||
"s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
|
||||
"&allow_bucket_creation=True"
|
||||
.format(access_key, secret_key, host, port)
|
||||
f"s3://{access_key}:{secret_key}@mybucket/data.parquet?scheme=http"
|
||||
f"&endpoint_override={host}:{port}&allow_bucket_creation=True"
|
||||
)
|
||||
fs, path = FileSystem.from_uri(uri)
|
||||
|
||||
|
||||
@@ -183,7 +183,7 @@ def test_dictionary_array_automatically_read():
|
||||
|
||||
# Make a large dictionary, a little over 4MB of data
|
||||
dict_length = 4000
|
||||
dict_values = pa.array([('x' * 1000 + '_{}'.format(i))
|
||||
dict_values = pa.array([('x' * 1000 + f'_{i}')
|
||||
for i in range(dict_length)])
|
||||
|
||||
num_chunks = 10
|
||||
@@ -220,8 +220,7 @@ def test_decimal_roundtrip(tempdir):
|
||||
util.randdecimal(precision, scale)
|
||||
for _ in range(num_values)
|
||||
]
|
||||
column_name = ('dec_precision_{:d}_scale_{:d}'
|
||||
.format(precision, scale))
|
||||
column_name = f'dec_precision_{precision}_scale_{scale}'
|
||||
columns[column_name] = random_decimal_values
|
||||
|
||||
expected = pd.DataFrame(columns)
|
||||
@@ -255,7 +254,7 @@ def test_decimal_roundtrip_negative_scale(tempdir):
|
||||
|
||||
@pytest.mark.parametrize('dtype', [int, float])
|
||||
def test_single_pylist_column_roundtrip(tempdir, dtype,):
|
||||
filename = tempdir / 'single_{}_column.parquet'.format(dtype.__name__)
|
||||
filename = tempdir / f'single_{dtype.__name__}_column.parquet'
|
||||
data = [pa.array(list(map(dtype, range(5))))]
|
||||
table = pa.Table.from_arrays(data, names=['a'])
|
||||
_write_table(table, filename)
|
||||
|
||||
@@ -566,7 +566,7 @@ def _generate_partition_directories(fs, base_dir, partition_spec, df):
|
||||
|
||||
level_dir = pathsep.join([
|
||||
str(base_dir),
|
||||
'{}={}'.format(name, value)
|
||||
f'{name}={value}'
|
||||
])
|
||||
fs.create_dir(level_dir)
|
||||
|
||||
@@ -648,7 +648,7 @@ def test_read_multiple_files(tempdir):
|
||||
# Hack so that we don't have a dtype cast in v1 files
|
||||
df['uint32'] = df['uint32'].astype(np.int64)
|
||||
|
||||
path = dirpath / '{}.parquet'.format(i)
|
||||
path = dirpath / f'{i}.parquet'
|
||||
|
||||
table = pa.Table.from_pandas(df)
|
||||
_write_table(table, path)
|
||||
@@ -683,7 +683,7 @@ def test_read_multiple_files(tempdir):
|
||||
|
||||
# Test failure modes with non-uniform metadata
|
||||
bad_apple = _test_dataframe(size, seed=i).iloc[:, :4]
|
||||
bad_apple_path = tempdir / '{}.parquet'.format(guid())
|
||||
bad_apple_path = tempdir / f'{guid()}.parquet'
|
||||
|
||||
t = pa.Table.from_pandas(bad_apple)
|
||||
_write_table(t, bad_apple_path)
|
||||
@@ -720,7 +720,7 @@ def test_dataset_read_pandas(tempdir):
|
||||
df.index = np.arange(i * size, (i + 1) * size)
|
||||
df.index.name = 'index'
|
||||
|
||||
path = dirpath / '{}.parquet'.format(i)
|
||||
path = dirpath / f'{i}.parquet'
|
||||
|
||||
table = pa.Table.from_pandas(df)
|
||||
_write_table(table, path)
|
||||
@@ -749,7 +749,7 @@ def test_dataset_memory_map(tempdir):
|
||||
dirpath.mkdir()
|
||||
|
||||
df = _test_dataframe(10, seed=0)
|
||||
path = dirpath / '{}.parquet'.format(0)
|
||||
path = dirpath / '0.parquet'
|
||||
table = pa.Table.from_pandas(df)
|
||||
_write_table(table, path, version='2.6')
|
||||
|
||||
@@ -764,7 +764,7 @@ def test_dataset_enable_buffered_stream(tempdir):
|
||||
dirpath.mkdir()
|
||||
|
||||
df = _test_dataframe(10, seed=0)
|
||||
path = dirpath / '{}.parquet'.format(0)
|
||||
path = dirpath / '0.parquet'
|
||||
table = pa.Table.from_pandas(df)
|
||||
_write_table(table, path, version='2.6')
|
||||
|
||||
@@ -784,7 +784,7 @@ def test_dataset_enable_pre_buffer(tempdir):
|
||||
dirpath.mkdir()
|
||||
|
||||
df = _test_dataframe(10, seed=0)
|
||||
path = dirpath / '{}.parquet'.format(0)
|
||||
path = dirpath / '0.parquet'
|
||||
table = pa.Table.from_pandas(df)
|
||||
_write_table(table, path, version='2.6')
|
||||
|
||||
@@ -801,7 +801,7 @@ def _make_example_multifile_dataset(base_path, nfiles=10, file_nrows=5):
|
||||
paths = []
|
||||
for i in range(nfiles):
|
||||
df = _test_dataframe(file_nrows, seed=i)
|
||||
path = base_path / '{}.parquet'.format(i)
|
||||
path = base_path / f'{i}.parquet'
|
||||
|
||||
test_data.append(_write_table(df, path))
|
||||
paths.append(path)
|
||||
@@ -823,7 +823,7 @@ def test_ignore_private_directories(tempdir, dir_prefix):
|
||||
file_nrows=5)
|
||||
|
||||
# private directory
|
||||
(dirpath / '{}staging'.format(dir_prefix)).mkdir()
|
||||
(dirpath / f'{dir_prefix}staging').mkdir()
|
||||
|
||||
dataset = pq.ParquetDataset(dirpath)
|
||||
|
||||
@@ -873,7 +873,7 @@ def test_ignore_hidden_files_underscore(tempdir):
|
||||
def test_ignore_no_private_directories_in_base_path(tempdir, dir_prefix):
|
||||
# ARROW-8427 - don't ignore explicitly listed files if parent directory
|
||||
# is a private directory
|
||||
dirpath = tempdir / "{0}data".format(dir_prefix) / guid()
|
||||
dirpath = tempdir / f'{dir_prefix}data' / guid()
|
||||
dirpath.mkdir(parents=True)
|
||||
|
||||
paths = _make_example_multifile_dataset(dirpath, nfiles=10,
|
||||
|
||||
@@ -267,7 +267,7 @@ def test_statistics_convert_logical_types(tempdir):
|
||||
for i, (min_val, max_val, typ) in enumerate(cases):
|
||||
t = pa.Table.from_arrays([pa.array([min_val, max_val], type=typ)],
|
||||
['col'])
|
||||
path = str(tempdir / ('example{}.parquet'.format(i)))
|
||||
path = str(tempdir / f'example{i}.parquet')
|
||||
pq.write_table(t, path, version='2.6')
|
||||
pf = pq.ParquetFile(path)
|
||||
stats = pf.metadata.row_group(0).column(0).statistics
|
||||
|
||||
@@ -588,7 +588,7 @@ def test_dataset_read_pandas_common_metadata(
|
||||
np.arange(i * size, (i + 1) * size, dtype="int64"), name='index'
|
||||
)
|
||||
|
||||
path = dirpath / '{}.parquet'.format(i)
|
||||
path = dirpath / f'{i}.parquet'
|
||||
|
||||
table = pa.Table.from_pandas(df, preserve_index=preserve_index)
|
||||
|
||||
|
||||
@@ -246,7 +246,7 @@ def test_to_numpy_writable():
|
||||
@pytest.mark.parametrize('tz', [None, "UTC"])
|
||||
def test_to_numpy_datetime64(unit, tz):
|
||||
arr = pa.array([1, 2, 3], pa.timestamp(unit, tz=tz))
|
||||
expected = np.array([1, 2, 3], dtype="datetime64[{}]".format(unit))
|
||||
expected = np.array([1, 2, 3], dtype=f"datetime64[{unit}]")
|
||||
np_arr = arr.to_numpy()
|
||||
np.testing.assert_array_equal(np_arr, expected)
|
||||
|
||||
@@ -255,7 +255,7 @@ def test_to_numpy_datetime64(unit, tz):
|
||||
@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
|
||||
def test_to_numpy_timedelta64(unit):
|
||||
arr = pa.array([1, 2, 3], pa.duration(unit))
|
||||
expected = np.array([1, 2, 3], dtype="timedelta64[{}]".format(unit))
|
||||
expected = np.array([1, 2, 3], dtype=f"timedelta64[{unit}]")
|
||||
np_arr = arr.to_numpy()
|
||||
np.testing.assert_array_equal(np_arr, expected)
|
||||
|
||||
|
||||
@@ -559,7 +559,7 @@ class BaseTestCSV(abc.ABC):
|
||||
if self.use_threads:
|
||||
row_info = ""
|
||||
else:
|
||||
row_info = "Row #{}: ".format(row)
|
||||
row_info = f"Row #{row}: "
|
||||
return msg_format.format(row_info, *args)
|
||||
|
||||
csv, _ = make_random_csv(4, 100, write_names=True)
|
||||
@@ -1423,7 +1423,7 @@ class BaseCSVTableRead(BaseTestCSV):
|
||||
except AttributeError:
|
||||
clock = time.time
|
||||
num_columns = 10000
|
||||
col_names = ["K{}".format(i) for i in range(num_columns)]
|
||||
col_names = [f"K{i}" for i in range(num_columns)]
|
||||
csv = make_empty_csv(col_names)
|
||||
t1 = clock()
|
||||
convert_options = ConvertOptions(
|
||||
|
||||
@@ -747,8 +747,10 @@ def make_table():
|
||||
dictionary=a2)
|
||||
|
||||
arrays = [a0, a1, a2, a3, a4, a5]
|
||||
schema = pa.schema([('f{}'.format(i), arr.type)
|
||||
for i, arr in enumerate(arrays)])
|
||||
schema = pa.schema([
|
||||
(f'f{i}', arr.type)
|
||||
for i, arr in enumerate(arrays)
|
||||
])
|
||||
batch = pa.record_batch(arrays, schema=schema)
|
||||
table = pa.Table.from_batches([batch])
|
||||
return table
|
||||
|
||||
@@ -123,7 +123,7 @@ def test_cython_api(tmpdir):
|
||||
|
||||
# Check the extension module is loadable from a subprocess without
|
||||
# pyarrow imported first.
|
||||
code = """if 1:
|
||||
code = f"""if 1:
|
||||
import sys
|
||||
import os
|
||||
|
||||
@@ -131,17 +131,16 @@ def test_cython_api(tmpdir):
|
||||
# Add dll directory was added on python 3.8
|
||||
# and is required in order to find extra DLLs
|
||||
# only for win32
|
||||
for dir in {library_dirs}:
|
||||
for dir in {pa.get_library_dirs()}:
|
||||
os.add_dll_directory(dir)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
mod = __import__({mod_name!r})
|
||||
mod = __import__('pyarrow_cython_example')
|
||||
arr = mod.make_null_array(5)
|
||||
assert mod.get_array_length(arr) == 5
|
||||
assert arr.null_count == 5
|
||||
""".format(mod_name='pyarrow_cython_example',
|
||||
library_dirs=pa.get_library_dirs())
|
||||
"""
|
||||
|
||||
path_var = None
|
||||
if sys.platform == 'win32':
|
||||
|
||||
@@ -122,7 +122,7 @@ def mockfs():
|
||||
]
|
||||
|
||||
for i, directory in enumerate(directories):
|
||||
path = '{}/file{}.parquet'.format(directory, i)
|
||||
path = f'{directory}/file{i}.parquet'
|
||||
mockfs.create_dir(directory)
|
||||
with mockfs.open_output_stream(path) as out:
|
||||
data = [
|
||||
@@ -198,15 +198,15 @@ def multisourcefs(request):
|
||||
mockfs.create_dir('plain')
|
||||
n = len(df_a)
|
||||
for i, chunk in enumerate([df_a.iloc[i:i+n//10] for i in range(0, n, n//10)]):
|
||||
path = 'plain/chunk-{}.parquet'.format(i)
|
||||
path = f'plain/chunk-{i}.parquet'
|
||||
with mockfs.open_output_stream(path) as out:
|
||||
pq.write_table(_table_from_pandas(chunk), out)
|
||||
|
||||
# create one with schema partitioning by weekday and color
|
||||
mockfs.create_dir('schema')
|
||||
for part, chunk in df_b.groupby([df_b.date.dt.dayofweek, df_b.color]):
|
||||
folder = 'schema/{}/{}'.format(*part)
|
||||
path = '{}/chunk.parquet'.format(folder)
|
||||
folder = f'schema/{part[0]}/{part[1]}'
|
||||
path = f'{folder}/chunk.parquet'
|
||||
mockfs.create_dir(folder)
|
||||
with mockfs.open_output_stream(path) as out:
|
||||
pq.write_table(_table_from_pandas(chunk), out)
|
||||
@@ -214,8 +214,8 @@ def multisourcefs(request):
|
||||
# create one with hive partitioning by year and month
|
||||
mockfs.create_dir('hive')
|
||||
for part, chunk in df_c.groupby([df_c.date.dt.year, df_c.date.dt.month]):
|
||||
folder = 'hive/year={}/month={}'.format(*part)
|
||||
path = '{}/chunk.parquet'.format(folder)
|
||||
folder = f'hive/year={part[0]}/month={part[1]}'
|
||||
path = f'{folder}/chunk.parquet'
|
||||
mockfs.create_dir(folder)
|
||||
with mockfs.open_output_stream(path) as out:
|
||||
pq.write_table(_table_from_pandas(chunk), out)
|
||||
@@ -223,8 +223,8 @@ def multisourcefs(request):
|
||||
# create one with hive partitioning by color
|
||||
mockfs.create_dir('hive_color')
|
||||
for part, chunk in df_d.groupby("color"):
|
||||
folder = 'hive_color/color={}'.format(part)
|
||||
path = '{}/chunk.parquet'.format(folder)
|
||||
folder = f'hive_color/color={part}'
|
||||
path = f'{folder}/chunk.parquet'
|
||||
mockfs.create_dir(folder)
|
||||
with mockfs.open_output_stream(path) as out:
|
||||
pq.write_table(_table_from_pandas(chunk), out)
|
||||
@@ -1801,8 +1801,8 @@ def test_fragments_repr(tempdir, dataset):
|
||||
fragment = list(dataset.get_fragments())[0]
|
||||
assert (
|
||||
repr(fragment) ==
|
||||
"<pyarrow.dataset.ParquetFileFragment path={}>".format(
|
||||
dataset.filesystem.normalize_path(str(path)))
|
||||
"<pyarrow.dataset.ParquetFileFragment path="
|
||||
f"{dataset.filesystem.normalize_path(str(path))}>"
|
||||
)
|
||||
|
||||
# non-parquet format
|
||||
@@ -1812,8 +1812,8 @@ def test_fragments_repr(tempdir, dataset):
|
||||
fragment = list(dataset.get_fragments())[0]
|
||||
assert (
|
||||
repr(fragment) ==
|
||||
"<pyarrow.dataset.FileFragment type=ipc path={}>".format(
|
||||
dataset.filesystem.normalize_path(str(path)))
|
||||
"<pyarrow.dataset.FileFragment type=ipc path="
|
||||
f"{dataset.filesystem.normalize_path(str(path))}>"
|
||||
)
|
||||
|
||||
|
||||
@@ -2569,7 +2569,7 @@ def _create_partitioned_dataset(basedir):
|
||||
path.mkdir()
|
||||
|
||||
for i in range(3):
|
||||
part = path / "part={}".format(i)
|
||||
part = path / f"part={i}"
|
||||
part.mkdir()
|
||||
pq.write_table(table.slice(3*i, 3), part / "test.parquet")
|
||||
|
||||
@@ -2781,9 +2781,8 @@ def s3_example_simple(s3_server):
|
||||
|
||||
host, port, access_key, secret_key = s3_server['connection']
|
||||
uri = (
|
||||
"s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
|
||||
"&allow_bucket_creation=True"
|
||||
.format(access_key, secret_key, host, port)
|
||||
f"s3://{access_key}:{secret_key}@mybucket/data.parquet?scheme=http"
|
||||
f"&endpoint_override={host}:{port}&allow_bucket_creation=True"
|
||||
)
|
||||
|
||||
fs, path = FileSystem.from_uri(uri)
|
||||
@@ -2833,7 +2832,7 @@ def test_open_dataset_from_uri_s3_fsspec(s3_example_simple):
|
||||
key=access_key,
|
||||
secret=secret_key,
|
||||
client_kwargs={
|
||||
'endpoint_url': 'http://{}:{}'.format(host, port)
|
||||
'endpoint_url': f'http://{host}:{port}'
|
||||
}
|
||||
)
|
||||
|
||||
@@ -2855,10 +2854,10 @@ def test_open_dataset_from_s3_with_filesystem_uri(s3_server):
|
||||
host, port, access_key, secret_key = s3_server['connection']
|
||||
bucket = 'theirbucket'
|
||||
path = 'nested/folder/data.parquet'
|
||||
uri = "s3://{}:{}@{}/{}?scheme=http&endpoint_override={}:{}"\
|
||||
"&allow_bucket_creation=true".format(
|
||||
access_key, secret_key, bucket, path, host, port
|
||||
)
|
||||
uri = (
|
||||
f"s3://{access_key}:{secret_key}@{bucket}/{path}?scheme=http"
|
||||
f"&endpoint_override={host}:{port}&allow_bucket_creation=true"
|
||||
)
|
||||
|
||||
fs, path = FileSystem.from_uri(uri)
|
||||
assert path == 'theirbucket/nested/folder/data.parquet'
|
||||
@@ -3361,7 +3360,7 @@ def test_csv_format(tempdir, dataset_reader):
|
||||
])
|
||||
def test_csv_format_compressed(tempdir, compression, dataset_reader):
|
||||
if not pyarrow.Codec.is_available(compression):
|
||||
pytest.skip("{} support is not built".format(compression))
|
||||
pytest.skip(f"{compression} support is not built")
|
||||
table = pa.table({'a': pa.array([1, 2, 3], type="int64"),
|
||||
'b': pa.array([.1, .2, .3], type="float64")})
|
||||
filesystem = fs.LocalFileSystem()
|
||||
@@ -4778,7 +4777,7 @@ def test_write_dataset_parquet(tempdir):
|
||||
format = ds.ParquetFileFormat()
|
||||
opts = format.make_write_options(version=version)
|
||||
assert "<pyarrow.dataset.ParquetFileWriteOptions" in repr(opts)
|
||||
base_dir = tempdir / 'parquet_dataset_version{0}'.format(version)
|
||||
base_dir = tempdir / f'parquet_dataset_version{version}'
|
||||
ds.write_dataset(table, base_dir, format=format, file_options=opts)
|
||||
meta = pq.read_metadata(base_dir / "part-0.parquet")
|
||||
expected_version = "1.0" if version == "1.0" else "2.6"
|
||||
@@ -5001,7 +5000,7 @@ def test_write_dataset_s3_put_only(s3_server):
|
||||
fs = S3FileSystem(
|
||||
access_key='test_dataset_limited_user',
|
||||
secret_key='limited123',
|
||||
endpoint_override='{}:{}'.format(host, port),
|
||||
endpoint_override=f'{host}:{port}',
|
||||
scheme='http'
|
||||
)
|
||||
|
||||
@@ -5049,7 +5048,7 @@ def test_write_dataset_s3_put_only(s3_server):
|
||||
fs = S3FileSystem(
|
||||
access_key='limited',
|
||||
secret_key='limited123',
|
||||
endpoint_override='{}:{}'.format(host, port),
|
||||
endpoint_override=f'{host}:{port}',
|
||||
scheme='http',
|
||||
allow_bucket_creation=True,
|
||||
)
|
||||
|
||||
@@ -864,7 +864,7 @@ class PeriodType(pa.ExtensionType):
|
||||
return self._freq
|
||||
|
||||
def __arrow_ext_serialize__(self):
|
||||
return "freq={}".format(self.freq).encode()
|
||||
return f"freq={self.freq}".encode()
|
||||
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(cls, storage_type, serialized):
|
||||
|
||||
@@ -443,7 +443,7 @@ def test_all_null_category(version):
|
||||
|
||||
@pytest.mark.pandas
|
||||
def test_multithreaded_read(version):
|
||||
data = {'c{}'.format(i): [''] * 10
|
||||
data = {f'c{i}': [''] * 10
|
||||
for i in range(100)}
|
||||
df = pd.DataFrame(data)
|
||||
_check_pandas_roundtrip(df, use_threads=True, version=version)
|
||||
|
||||
@@ -83,9 +83,9 @@ def read_flight_resource(path):
|
||||
return f.read()
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(
|
||||
"Test resource {} not found; did you initialize the "
|
||||
"test resource submodule?\n{}".format(root / path,
|
||||
traceback.format_exc()))
|
||||
f"Test resource {root / path} not found; did you initialize the "
|
||||
f"test resource submodule?\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
|
||||
def example_tls_certs():
|
||||
@@ -431,7 +431,7 @@ class ExchangeFlightServer(FlightServerBase):
|
||||
return self.exchange_transform(context, reader, writer)
|
||||
else:
|
||||
raise pa.ArrowInvalid(
|
||||
"Unknown command: {}".format(descriptor.command))
|
||||
f"Unknown command: {descriptor.command}")
|
||||
|
||||
def exchange_do_get(self, context, reader, writer):
|
||||
"""Emulate DoGet with DoExchange."""
|
||||
|
||||
@@ -128,13 +128,13 @@ class DummyHandler(FileSystemHandler):
|
||||
def open_input_stream(self, path):
|
||||
if "notfound" in path:
|
||||
raise FileNotFoundError(path)
|
||||
data = "{0}:input_stream".format(path).encode('utf8')
|
||||
data = f"{path}:input_stream".encode('utf8')
|
||||
return pa.BufferReader(data)
|
||||
|
||||
def open_input_file(self, path):
|
||||
if "notfound" in path:
|
||||
raise FileNotFoundError(path)
|
||||
data = "{0}:input_file".format(path).encode('utf8')
|
||||
data = f"{path}:input_file".encode('utf8')
|
||||
return pa.BufferReader(data)
|
||||
|
||||
def open_output_stream(self, path, metadata):
|
||||
@@ -249,7 +249,7 @@ def s3fs(request, s3_server):
|
||||
fs = S3FileSystem(
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
endpoint_override='{}:{}'.format(host, port),
|
||||
endpoint_override=f'{host}:{port}',
|
||||
scheme='http',
|
||||
allow_bucket_creation=True,
|
||||
allow_bucket_deletion=True
|
||||
@@ -382,7 +382,7 @@ def py_fsspec_s3fs(request, s3_server):
|
||||
fs = s3fs.S3FileSystem(
|
||||
key=access_key,
|
||||
secret=secret_key,
|
||||
client_kwargs=dict(endpoint_url='http://{}:{}'.format(host, port))
|
||||
client_kwargs=dict(endpoint_url=f'http://{host}:{port}')
|
||||
)
|
||||
fs = PyFileSystem(FSSpecHandler(fs))
|
||||
fs.create_dir(bucket)
|
||||
@@ -523,7 +523,7 @@ def test_s3fs_limited_permissions_create_bucket(s3_server):
|
||||
fs = S3FileSystem(
|
||||
access_key='test_fs_limited_user',
|
||||
secret_key='limited123',
|
||||
endpoint_override='{}:{}'.format(host, port),
|
||||
endpoint_override=f'{host}:{port}',
|
||||
scheme='http'
|
||||
)
|
||||
fs.create_dir('existing-bucket/test')
|
||||
@@ -766,7 +766,7 @@ def test_get_file_info_with_selector(fs, pathfn):
|
||||
info.path.rstrip("/").endswith(dir_b)):
|
||||
assert info.type == FileType.Directory
|
||||
else:
|
||||
raise ValueError('unexpected path {}'.format(info.path))
|
||||
raise ValueError(f'unexpected path {info.path}')
|
||||
check_mtime_or_absent(info)
|
||||
|
||||
# non-recursive selector -> not selecting the nested file_c
|
||||
@@ -1510,7 +1510,7 @@ def test_hdfs_options(hdfs_connection, pickle_module):
|
||||
host, port, 'me', replication + 1, buffer_size, default_block_size
|
||||
))
|
||||
hdfs5 = HadoopFileSystem(host, port)
|
||||
hdfs6 = HadoopFileSystem.from_uri('hdfs://{}:{}'.format(host, port))
|
||||
hdfs6 = HadoopFileSystem.from_uri(f'hdfs://{host}:{port}')
|
||||
hdfs7 = HadoopFileSystem(host, port, user='localuser')
|
||||
hdfs8 = HadoopFileSystem(host, port, user='localuser',
|
||||
kerb_ticket="cache_path")
|
||||
@@ -1550,7 +1550,7 @@ def test_hdfs_options(hdfs_connection, pickle_module):
|
||||
assert hdfs.get_file_info(FileSelector('/'))
|
||||
|
||||
hdfs = HadoopFileSystem.from_uri(
|
||||
"hdfs://{}:{}/?user={}".format(host, port, user)
|
||||
f"hdfs://{host}:{port}/?user={user}"
|
||||
)
|
||||
assert hdfs.get_file_info(FileSelector('/'))
|
||||
|
||||
@@ -1596,9 +1596,8 @@ def test_filesystem_from_uri_s3(s3_server):
|
||||
|
||||
host, port, access_key, secret_key = s3_server['connection']
|
||||
|
||||
uri = "s3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}"\
|
||||
"&allow_bucket_creation=True" \
|
||||
.format(access_key, secret_key, host, port)
|
||||
uri = f"s3://{access_key}:{secret_key}@mybucket/foo/bar?scheme=http&" \
|
||||
f"endpoint_override={host}:{port}&allow_bucket_creation=True"
|
||||
|
||||
fs, path = FileSystem.from_uri(uri)
|
||||
assert isinstance(fs, S3FileSystem)
|
||||
|
||||
@@ -822,7 +822,7 @@ def test_cache_options_pickling(pickle_module):
|
||||
])
|
||||
def test_compress_decompress(compression):
|
||||
if not Codec.is_available(compression):
|
||||
pytest.skip("{} support is not built".format(compression))
|
||||
pytest.skip(f"{compression} support is not built")
|
||||
|
||||
INPUT_SIZE = 10000
|
||||
test_data = (np.random.randint(0, 255, size=INPUT_SIZE)
|
||||
@@ -863,7 +863,7 @@ def test_compress_decompress(compression):
|
||||
])
|
||||
def test_compression_level(compression):
|
||||
if not Codec.is_available(compression):
|
||||
pytest.skip("{} support is not built".format(compression))
|
||||
pytest.skip(f"{compression} support is not built")
|
||||
|
||||
codec = Codec(compression)
|
||||
if codec.name == "snappy":
|
||||
@@ -1755,7 +1755,7 @@ def test_unknown_compression_raises():
|
||||
])
|
||||
def test_compressed_roundtrip(compression):
|
||||
if not Codec.is_available(compression):
|
||||
pytest.skip("{} support is not built".format(compression))
|
||||
pytest.skip(f"{compression} support is not built")
|
||||
|
||||
data = b"some test data\n" * 10 + b"eof\n"
|
||||
raw = pa.BufferOutputStream()
|
||||
@@ -1776,7 +1776,7 @@ def test_compressed_roundtrip(compression):
|
||||
)
|
||||
def test_compressed_recordbatch_stream(compression):
|
||||
if not Codec.is_available(compression):
|
||||
pytest.skip("{} support is not built".format(compression))
|
||||
pytest.skip(f"{compression} support is not built")
|
||||
|
||||
# ARROW-4836: roundtrip a RecordBatch through a compressed stream
|
||||
table = pa.Table.from_arrays([pa.array([1, 2, 3, 4, 5])], ['a'])
|
||||
|
||||
@@ -45,7 +45,7 @@ def root_allocator():
|
||||
}).text
|
||||
jar_path = os.path.join(
|
||||
arrow_dir, 'java', 'tools', 'target',
|
||||
'arrow-tools-{}-jar-with-dependencies.jar'.format(version))
|
||||
f'arrow-tools-{version}-jar-with-dependencies.jar')
|
||||
jar_path = os.getenv("ARROW_TOOLS_JAR", jar_path)
|
||||
kwargs = {}
|
||||
# This will be the default behaviour in jpype 0.8+
|
||||
@@ -224,7 +224,7 @@ def test_jvm_types(root_allocator, pa_type, jvm_spec, nullable):
|
||||
])
|
||||
def test_jvm_array(root_allocator, pa_type, py_data, jvm_type):
|
||||
# Create vector
|
||||
cls = "org.apache.arrow.vector.{}".format(jvm_type)
|
||||
cls = f"org.apache.arrow.vector.{jvm_type}"
|
||||
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
|
||||
jvm_vector.allocateNew(len(py_data))
|
||||
for i, val in enumerate(py_data):
|
||||
@@ -241,7 +241,7 @@ def test_jvm_array(root_allocator, pa_type, py_data, jvm_type):
|
||||
|
||||
|
||||
def test_jvm_array_empty(root_allocator):
|
||||
cls = "org.apache.arrow.vector.{}".format('IntVector')
|
||||
cls = f"org.apache.arrow.vector.{'IntVector'}"
|
||||
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
|
||||
jvm_vector.allocateNew()
|
||||
jvm_array = pa_jvm.array(jvm_vector)
|
||||
@@ -362,7 +362,7 @@ def test_jvm_array_empty(root_allocator):
|
||||
def test_jvm_record_batch(root_allocator, pa_type, py_data, jvm_type,
|
||||
jvm_spec):
|
||||
# Create vector
|
||||
cls = "org.apache.arrow.vector.{}".format(jvm_type)
|
||||
cls = f"org.apache.arrow.vector.{jvm_type}"
|
||||
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
|
||||
jvm_vector.allocateNew(len(py_data))
|
||||
for i, val in enumerate(py_data):
|
||||
|
||||
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user