GH-45619: [Python] Use f-string instead of string.format (#45629)

### Rationale for this change

See https://github.com/apache/arrow/issues/45619.

### What changes are included in this PR?

Refactor using f-string instead of `string.format`. But do not use f-string for following case,  `string.format` allows passing parameters, making the code more reusable.

https://github.com/apache/arrow/blob/0fbf9823542233c5f32c26534c34cc97ce3f0be2/python/pyarrow/parquet/core.py#L1624-L1695

### Are these changes tested?

Via CI.

### Are there any user-facing changes?

No.

* GitHub Issue: #45619

Lead-authored-by: Chilin Chiou <chilin.chiou@gmail.com>
Co-authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
This commit is contained in:
ChiLin ChiuandRaúl Cumplido authored and GitHub committed 2025-05-12 14:34:49 +02:00
1 parent c3855b917e
commit 992bee2c83
110 files changed
+753 -948

No files matched your search

+1 -1
View File
@@ -171,7 +171,7 @@ class DarwinSymbolizer(Symbolizer):
# foo(type1, type2) (in object.name) (filename.cc:80)
match = re.match(r'^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line)
if DEBUG:
print('atos_line: {0}'.format(atos_line))
print(f'atos_line: {atos_line}')
if match:
function_name = match.group(1)
function_name = re.sub(r'\(.*?\)', '', function_name)
+3 -4
View File
@@ -31,13 +31,12 @@ def process_dir(corpus_dir, zip_output):
for child in corpus_dir.iterdir():
if not child.is_file():
raise IOError("Not a file: {0}".format(child))
raise IOError(f"Not a file: {child}")
with child.open('rb') as f:
data = f.read()
arcname = hashlib.sha1(data).hexdigest()
if arcname in seen:
raise ValueError("Duplicate hash: {0} (in file {1})"
.format(arcname, child))
raise ValueError(f"Duplicate hash: {arcname} (in file {child})")
zip_output.writestr(str(arcname), data)
seen.add(arcname)
@@ -49,6 +48,6 @@ def main(corpus_dir, zip_output_name):
if __name__ == "__main__":
if len(sys.argv) != 3:
print("Usage: {0} <corpus dir> <output zip file>".format(sys.argv[0]))
print(f"Usage: {sys.argv[0]} <corpus dir> <output zip file>")
sys.exit(1)
main(sys.argv[1], sys.argv[2])
+1 -1
View File
@@ -204,7 +204,7 @@ def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
if matches:
entries.extend(matches)
else:
print("{} not in compilation database".format(source))
print(f"{source} not in compilation database")
# TODO: As long as there is no complete compilation database available this check cannot be performed
pass
#print('WARNING: \'%s\' not found in compilation database.', source)
+2 -2
View File
@@ -121,7 +121,7 @@ def lint_files():
if __name__ == '__main__':
failures = list(lint_files())
for path, why, i, line in failures:
print('File {0} failed C++/CLI lint check: {1}\n'
'Line {2}: {3}'.format(path, why, i + 1, line))
print(f'File {path} failed C++/CLI lint check: {why}\n'
f'Line {i + 1}: {line}')
if failures:
exit(1)
+4 -6
View File
@@ -38,8 +38,7 @@ def _check_one_file(filename, formatted):
original.decode('utf8').splitlines(True),
formatted.decode('utf8').splitlines(True),
fromfile=filename,
tofile="{} (after clang format)".format(
filename)))
tofile=f"{filename} (after clang format)"))
else:
diff = None
@@ -85,8 +84,7 @@ if __name__ == "__main__":
if arguments.fix:
if not arguments.quiet:
print("\n".join(map(lambda x: "Formatting {}".format(x),
formatted_filenames)))
print("\n".join(f"Formatting {x}" for x in formatted_filenames))
# Break clang-format invocations into chunks: each invocation formats
# 16 files. Wait for all processes to complete
@@ -122,9 +120,9 @@ if __name__ == "__main__":
# check the output from each invocation of clang-format in parallel
for filename, diff in pool.starmap(_check_one_file, checker_args):
if not arguments.quiet:
print("Checking {}".format(filename))
print(f"Checking {filename}")
if diff:
print("{} had clang-format style issues".format(filename))
print(f"{filename} had clang-format style issues")
# Print out the diff to stderr
error = True
# pad with a newline
+1 -1
View File
@@ -78,7 +78,7 @@ if __name__ == "__main__":
if arguments.quiet:
cmd.append('--quiet')
else:
print("\n".join(map(lambda x: "Linting {}".format(x),
print("\n".join(map(lambda x: f"Linting {x}",
linted_filenames)))
# lint files in chunks: each invocation of cpplint will process 16 files
+17 -20
View File
@@ -73,7 +73,7 @@ namespace internal {
print("inline const uint8_t* unpack0_64(const uint8_t* in, uint64_t* out) {")
print(" for(int k = 0; k < {0} ; k += 1) {{".format(howmany(0)))
print(f" for(int k = 0; k < {howmany(0)} ; k += 1) {{")
print(" out[k] = 0;")
print(" }")
print(" return in;")
@@ -81,47 +81,44 @@ print("}")
for bit in range(1, 65):
print("")
print(
"inline const uint8_t* unpack{0}_64(const uint8_t* in, uint64_t* out) {{".format(bit))
print(f"inline const uint8_t* unpack{bit}_64(const uint8_t* in, uint64_t* out) {{")
if(bit < 64):
print(" const uint64_t mask = {0}ULL;".format((1 << bit)-1))
print(f" const uint64_t mask = {((1 << bit)-1)}ULL;")
maskstr = " & mask"
if (bit == 64):
maskstr = "" # no need
for k in range(howmanywords(bit)-1):
print(" uint64_t w{0} = util::SafeLoadAs<uint64_t>(in);".format(k))
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
print(" in += 8;".format(k))
print(f" uint64_t w{k} = util::SafeLoadAs<uint64_t>(in);")
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
print(" in += 8;")
k = howmanywords(bit) - 1
if (bit % 2 == 0):
print(" uint64_t w{0} = util::SafeLoadAs<uint64_t>(in);".format(k))
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
print(" in += 8;".format(k))
print(f" uint64_t w{k} = util::SafeLoadAs<uint64_t>(in);")
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
print(" in += 8;")
else:
print(" uint64_t w{0} = util::SafeLoadAs<uint32_t>(in);".format(k))
print(" w{0} = arrow::BitUtil::FromLittleEndian(w{0});".format(k))
print(" in += 4;".format(k))
print(f" uint64_t w{k} = util::SafeLoadAs<uint32_t>(in);")
print(f" w{k} = arrow::BitUtil::FromLittleEndian(w{k});")
print(" in += 4;")
for j in range(howmany(bit)):
firstword = j * bit // 64
secondword = (j * bit + bit - 1)//64
firstshift = (j*bit) % 64
firstshiftstr = " >> {0}".format(firstshift)
firstshiftstr = f" >> {firstshift}"
if(firstshift == 0):
firstshiftstr = "" # no need
if(firstword == secondword):
if(firstshift + bit == 64):
print(" out[{0}] = w{1}{2};".format(
j, firstword, firstshiftstr, firstshift))
print(f" out[{j}] = w{firstword}{firstshiftstr};")
else:
print(" out[{0}] = (w{1}{2}){3};".format(
j, firstword, firstshiftstr, maskstr))
print(f" out[{j}] = (w{firstword}{firstshiftstr}){maskstr};")
else:
secondshift = (64-firstshift)
print(" out[{0}] = ((w{1}{2}) | (w{3} << {4})){5};".format(
j, firstword, firstshiftstr, firstword+1, secondshift, maskstr))
print(f" out[{j}] = ((w{firstword}{firstshiftstr}) | "
f"(w{firstword+1} << {secondshift})){maskstr};")
print("")
print(" return in;")
print("}")
+2 -2
View File
@@ -37,8 +37,8 @@ def apply_template(template, data):
if __name__ == "__main__":
if len(sys.argv) != 4:
raise ValueError("Usage: {0} <template file> <data file> "
"<output file>".format(sys.argv[0]))
raise ValueError(f"Usage: {sys.argv[0]} <template file> <data file> "
"<output file>")
with open(sys.argv[1], "rb") as f:
template = f.read()
with open(sys.argv[2], "rb") as f:
+10 -10
View File
@@ -23,30 +23,30 @@ DEFAULT_THRESHOLD = 0.05
def items_per_seconds_fmt(value):
if value < 1000:
return "{} items/sec".format(value)
return f"{value} items/sec"
if value < 1000**2:
return "{:.3f}K items/sec".format(value / 1000)
return f"{value / 1000:.3f}K items/sec"
if value < 1000**3:
return "{:.3f}M items/sec".format(value / 1000**2)
return f"{value / 1000**2:.3f}M items/sec"
else:
return "{:.3f}G items/sec".format(value / 1000**3)
return f"{value / 1000**3:.3f}G items/sec"
def bytes_per_seconds_fmt(value):
if value < 1024:
return "{} bytes/sec".format(value)
return f"{value} bytes/sec"
if value < 1024**2:
return "{:.3f} KiB/sec".format(value / 1024)
return f"{value / 1024:.3f} KiB/sec"
if value < 1024**3:
return "{:.3f} MiB/sec".format(value / 1024**2)
return f"{value / 1024**2:.3f} MiB/sec"
if value < 1024**4:
return "{:.3f} GiB/sec".format(value / 1024**3)
return f"{value / 1024**3:.3f} GiB/sec"
else:
return "{:.3f} TiB/sec".format(value / 1024**4)
return f"{value / 1024**4:.3f} TiB/sec"
def change_fmt(value):
return "{:.3%}".format(value)
return f"{value:.3%}"
def formatter_for_unit(unit):
+2 -4
View File
@@ -43,7 +43,7 @@ class Benchmark:
return self.median
def __repr__(self):
return "Benchmark[name={},value={}]".format(self.name, self.value)
return f"Benchmark[name={self.name},value={self.value}]"
class BenchmarkSuite:
@@ -52,6 +52,4 @@ class BenchmarkSuite:
self.benchmarks = benchmarks
def __repr__(self):
return "BenchmarkSuite[name={}, benchmarks={}]".format(
self.name, self.benchmarks
)
return f"BenchmarkSuite[name={self.name}, benchmarks={self.benchmarks}]"
+2 -2
View File
@@ -45,7 +45,7 @@ class GoogleBenchmarkCommand(Command):
def list_benchmarks(self):
argv = ["--benchmark_list_tests"]
if self.benchmark_filter:
argv.append("--benchmark_filter={}".format(self.benchmark_filter))
argv.append(f"--benchmark_filter={self.benchmark_filter}")
result = self.run(*argv, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
return str.splitlines(result.stdout.decode("utf-8"))
@@ -166,7 +166,7 @@ class GoogleBenchmark(Benchmark):
counters)
def __repr__(self):
return "GoogleBenchmark[name={},runs={}]".format(self.names, self.runs)
return f"GoogleBenchmark[name={self.name},runs={self.runs}]"
@classmethod
def from_json(cls, payload):
+5 -8
View File
@@ -58,7 +58,7 @@ class JavaMicrobenchmarkHarnessCommand(Command):
def list_benchmarks(self):
argv = []
if self.benchmark_filter:
argv.append("-Dbenchmark.filter={}".format(self.benchmark_filter))
argv.append(f"-Dbenchmark.filter={self.benchmark_filter}")
result = self.build.list(
*argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -77,13 +77,11 @@ class JavaMicrobenchmarkHarnessCommand(Command):
def results(self, repetitions):
with NamedTemporaryFile(suffix=".json") as out:
argv = ["-Dbenchmark.runs={}".format(repetitions),
"-Dbenchmark.resultfile={}".format(out.name),
argv = [f"-Dbenchmark.runs={repetitions}",
f"-Dbenchmark.resultfile={out.name}",
"-Dbenchmark.resultformat=json"]
if self.benchmark_filter:
argv.append(
"-Dbenchmark.filter={}".format(self.benchmark_filter)
)
argv.append(f"-Dbenchmark.filter={self.benchmark_filter}")
self.build.benchmark(*argv, check=True)
return json.load(out)
@@ -187,8 +185,7 @@ class JavaMicrobenchmarkHarness(Benchmark):
counters)
def __repr__(self):
return "JavaMicrobenchmark[name={},runs={}]".format(
self.name, self.runs)
return f"JavaMicrobenchmark[name={self.name},runs={self.runs}]"
@classmethod
def from_json(cls, payload):
+7 -9
View File
@@ -69,7 +69,7 @@ class StaticBenchmarkRunner(BenchmarkRunner):
def list_benchmarks(self):
for suite in self._suites:
for benchmark in suite.benchmarks:
yield "{}.{}".format(suite.name, benchmark.name)
yield f"{suite.name}.{benchmark.name}"
@property
def suites(self):
@@ -102,7 +102,7 @@ class StaticBenchmarkRunner(BenchmarkRunner):
return BenchmarkRunnerCodec.decode(loaded, **kwargs)
def __repr__(self):
return "BenchmarkRunner[suites={}]".format(list(self.suites))
return f"BenchmarkRunner[suites={list(self.suites)}]"
class CppBenchmarkRunner(BenchmarkRunner):
@@ -165,7 +165,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
for suite_name, suite_bin in self.suites_binaries.items():
suite_cmd = GoogleBenchmarkCommand(suite_bin)
for benchmark_name in suite_cmd.list_benchmarks():
yield "{}.{}".format(suite_name, benchmark_name)
yield f"{suite_name}.{benchmark_name}"
@property
def suites(self):
@@ -176,7 +176,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
suite_and_binaries = self.suites_binaries
for suite_name in suite_and_binaries:
if not suite_matcher(suite_name):
logger.debug("Ignoring suite {}".format(suite_name))
logger.debug(f"Ignoring suite {suite_name}")
continue
suite_bin = suite_and_binaries[suite_name]
@@ -184,8 +184,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
# Filter may exclude all benchmarks
if not suite:
logger.debug("Suite {} executed but no results"
.format(suite_name))
logger.debug(f"Suite {suite_name} executed but no results")
continue
suite_found = True
@@ -274,7 +273,7 @@ class JavaBenchmarkRunner(BenchmarkRunner):
suite_cmd = JavaMicrobenchmarkHarnessCommand(self.build)
benchmark_names = suite_cmd.list_benchmarks()
for benchmark_name in benchmark_names:
yield "{}".format(benchmark_name)
yield f"{benchmark_name}"
@property
def suites(self):
@@ -284,8 +283,7 @@ class JavaBenchmarkRunner(BenchmarkRunner):
# Filter may exclude all benchmarks
if not suite:
logger.debug("Suite {} executed but no results"
.format(suite_name))
logger.debug(f"Suite {suite_name} executed but no results")
return
yield suite
+3 -3
View File
@@ -230,7 +230,7 @@ class CommentBot:
self.github = github.Github(**kwargs)
def parse_command(self, payload):
mention = '@{}'.format(self.name)
mention = f'@{self.name}'
comment = payload['comment']
if payload['sender']['login'] == self.name:
@@ -261,7 +261,7 @@ class CommentBot:
elif event == 'pull_request_review_comment':
return self.handle_review_comment(command, payload)
else:
raise ValueError("Unexpected event type {}".format(event))
raise ValueError(f"Unexpected event type {event}")
def handle_issue_comment(self, command, payload):
repo = self.github.get_repo(payload['repository']['id'], lazy=True)
@@ -355,7 +355,7 @@ def _clone_arrow_and_crossbow(dest, crossbow_repo, arrow_repo_url, pr_number):
git.checkout(local_branch, git_dir=arrow_path)
# 2. clone crossbow repository
crossbow_url = 'https://github.com/{}'.format(crossbow_repo)
crossbow_url = f'https://github.com/{crossbow_repo}'
git.clone(crossbow_url, str(queue_path))
# 3. initialize crossbow objects
+5 -6
View File
@@ -270,7 +270,7 @@ def decorate_lint_command(cmd):
Decorate the lint() command function to add individual per-check options.
"""
for check in lint_checks:
option = click.option("--{0}/--no-{0}".format(check.option_name),
option = click.option(f"--{check.option_name}/--no-{check.option_name}",
default=None, help=check.help)
cmd = option(cmd)
return cmd
@@ -414,7 +414,7 @@ def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras,
""" List benchmark suite.
"""
with tmpdir(preserve=preserve) as root:
logger.debug("Running benchmark {}".format(rev_or_path))
logger.debug(f"Running benchmark {rev_or_path}")
if language == "cpp":
conf = CppBenchmarkRunner.default_configuration(
@@ -495,7 +495,7 @@ def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras,
archery benchmark run --output=run.json
"""
with tmpdir(preserve=preserve) as root:
logger.debug("Running benchmark {}".format(rev_or_path))
logger.debug(f"Running benchmark {rev_or_path}")
if language == "cpp":
conf = CppBenchmarkRunner.default_configuration(
@@ -626,8 +626,7 @@ def benchmark_diff(ctx, src, preserve, output, language, cmake_extras,
archery --quiet benchmark diff WORKSPACE run.json > result.json
"""
with tmpdir(preserve=preserve) as root:
logger.debug("Comparing {} (contender) with {} (baseline)"
.format(contender, baseline))
logger.debug(f"Comparing {contender} (contender) with {baseline} (baseline)")
if language == "cpp":
conf = CppBenchmarkRunner.default_configuration(
@@ -704,7 +703,7 @@ def _format_comparisons_with_pandas(comparisons_json, no_counters,
def labelled(title, df):
if len(df) == 0:
return ''
title += ': ({})'.format(len(df))
title += f': ({len(df)})'
df_str = df.to_string(index=False)
bar = '-' * df_str.index('\n')
return '\n'.join([bar, title, bar, df_str])
+5 -6
View File
@@ -150,11 +150,10 @@ def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version,
queue.put(job, prefix=job_prefix)
if no_push:
click.echo('Branches and commits created but not pushed: `{}`'
.format(job.branch))
click.echo(f'Branches and commits created but not pushed: `{job.branch}`')
else:
queue.push()
click.echo('Pushed job identifier is: `{}`'.format(job.branch))
click.echo(f'Pushed job identifier is: `{job.branch}`')
@crossbow.command()
@@ -263,7 +262,7 @@ def render(obj, task, config_path, arrow_version, arrow_remote, arrow_branch,
for task_name, rendered_files in job.render_tasks().items():
for path, content in _flatten(rendered_files).items():
click.echo('#' * 80)
click.echo('### {:^72} ###'.format("/".join(path)))
click.echo(f"### {'/'.join(path):^72} ###")
click.echo('#' * 80)
click.echo(highlight(content))
@@ -534,8 +533,8 @@ def download_artifacts(obj, job_name, target_dir, dry_run, fetch,
else:
break
click.echo('Downloading {}\'s artifacts.'.format(job_name))
click.echo('Destination directory is {}'.format(target_dir))
click.echo(f'Downloading {job_name}\'s artifacts.')
click.echo(f'Destination directory is {target_dir}')
click.echo()
report = ConsoleReport(job, task_filters=task_filters)
+58 -64
View File
@@ -269,8 +269,8 @@ class Repo:
def origin(self):
remote = self.repo.remotes['origin']
if self.require_https and remote.url.startswith('git@github.com'):
raise CrossbowError("Change SSH origin URL to HTTPS to use "
"Crossbow: {}".format(remote.url))
raise CrossbowError(f"Change SSH origin URL to HTTPS to use "
f"Crossbow: {remote.url}")
return remote
def fetch(self, retry=3):
@@ -299,9 +299,9 @@ class Repo:
try:
self.origin.push(refs + self._updated_refs, callbacks=callbacks)
except pygit2.GitError:
raise RuntimeError('Failed to push updated references, '
'potentially because of credential issues: {}'
.format(self._updated_refs))
raise RuntimeError("Failed to push updated references, "
"potentially because of credential issues: "
f"{self._updated_refs}")
else:
self.updated_refs = []
@@ -329,9 +329,9 @@ class Repo:
return self.repo.remotes[self.branch.upstream.remote_name]
except (AttributeError, KeyError):
raise CrossbowError(
'Cannot determine git remote for the Arrow repository to '
'clone or push to, try to push the `{}` branch first to have '
'a remote tracking counterpart.'.format(self.branch.name)
"Cannot determine git remote for the Arrow repository to "
f"clone or push to, try to push the `{self.branch.name}` "
"branch first to have a remote tracking counterpart."
)
@property
@@ -422,7 +422,7 @@ class Repo:
branch = self.repo.create_branch(branch_name, commit)
# append to the pushable references
self._updated_refs.append('refs/heads/{}'.format(branch_name))
self._updated_refs.append(f"refs/heads/{branch_name}")
return branch
@@ -438,7 +438,7 @@ class Repo:
message)
# append to the pushable references
self._updated_refs.append('refs/tags/{}'.format(tag_name))
self._updated_refs.append(f"refs/tags/{tag_name}")
return self.repo[tag_id]
@@ -505,12 +505,11 @@ class Repo:
result = release.upload_asset(name=name, asset=fp,
content_type=mime)
except github3.exceptions.ResponseError as e:
logger.error('Attempt {} has failed with message: {}.'
.format(i + 1, str(e)))
logger.error('Error message {}'.format(e.msg))
logger.error('List of errors provided by GitHub:')
logger.error(f"Attempt {i + 1} has failed with message: {e}.")
logger.error(f"Error message {e.msg}")
logger.error("List of errors provided by GitHub:")
for err in e.errors:
logger.error(' - {}'.format(err))
logger.error(f" - {err}")
if e.code == 422:
# 422 Validation Failed, probably raised because
@@ -518,16 +517,15 @@ class Repo:
# reattempting the asset upload
for asset in release.assets():
if asset.name == name:
logger.info('Release asset {} already exists, '
'removing it...'.format(name))
logger.info(f"Release asset {name} already exists, "
"removing it...")
asset.delete()
logger.info('Asset {} removed.'.format(name))
logger.info(f"Asset {name} removed.")
break
except github3.exceptions.ConnectionError as e:
logger.error('Attempt {} has failed with message: {}.'
.format(i + 1, str(e)))
logger.error(f"Attempt {i + 1} has failed with message: {e}.")
else:
logger.info('Attempt {} has finished.'.format(i + 1))
logger.info(f"Attempt {i + 1} has finished.")
return result
time.sleep(retry_backoff)
@@ -536,14 +534,14 @@ class Repo:
def github_upload_asset_curl(self, release, path, name, mime):
upload_url, _ = release.upload_url.split('{?')
upload_url += '?name={}'.format(name)
upload_url += f"?name={name}"
command = [
'curl',
'--fail',
'-H', "Authorization: token {}".format(self.github_token),
'-H', "Content-Type: {}".format(mime),
'--data-binary', '@{}'.format(path),
'-H', f"Authorization: token {self.github_token}",
'-H', f"Content-Type: {mime}",
'--data-binary', f'@{path}',
upload_url
]
return subprocess.run(command, shell=False, check=True)
@@ -576,8 +574,8 @@ class Repo:
mime = mimetypes.guess_type(name)[0] or 'application/zip'
logger.info(
'Uploading asset `{}` with mimetype {} and size {}...'
.format(name, mime, size)
f"Uploading asset `{name}` with mimetype {mime} and size "
f"{size}..."
)
if method == 'requests':
@@ -588,7 +586,7 @@ class Repo:
mime=mime)
else:
raise CrossbowError(
'Unsupported upload method {}'.format(method)
f"Unsupported upload method {method}"
)
def github_pr(self, title, head=None, base=None, body=None,
@@ -645,12 +643,12 @@ class Queue(Repo):
def _next_job_id(self, prefix):
"""Auto increments the branch's identifier based on the prefix"""
latest_id = self._latest_prefix_id(prefix)
return '{}-{}'.format(prefix, latest_id + 1)
return f"{prefix}-{latest_id + 1}"
def _new_hex_id(self, prefix):
"""Append a new id to branch's identifier based on the prefix"""
hex_id = uuid.uuid4().hex[:10]
return '{}-{}'.format(prefix, hex_id)
return f"{prefix}-{hex_id}"
def latest_for_prefix(self, prefix):
prefix_date = self._prefix_contains_date(prefix)
@@ -667,13 +665,13 @@ class Queue(Repo):
raise RuntimeError(
f"No job has been submitted with prefix '{prefix}' yet"
)
job_name = '{}-{}'.format(prefix, latest_id)
job_name = f"{prefix}-{latest_id}"
return self.get(job_name)
def date_of(self, job):
# it'd be better to bound to the queue repository on deserialization
# and reorganize these methods to Job
branch_name = 'origin/{}'.format(job.branch)
branch_name = f'origin/{job.branch}'
branch = self.repo.branches[branch_name]
commit = self.repo[branch.target]
return date.fromtimestamp(commit.commit_time)
@@ -691,13 +689,13 @@ class Queue(Repo):
yield self.get(name)
def get(self, job_name):
branch_name = 'origin/{}'.format(job_name)
branch_name = f"origin/{job_name}"
branch = self.repo.branches[branch_name]
try:
content = self.file_contents(branch.target, 'job.yml')
except KeyError:
raise CrossbowError(
'No job is found with name: {}'.format(job_name)
f"No job is found with name: {job_name}"
)
buffer = StringIO(content.decode('utf-8'))
@@ -724,7 +722,7 @@ class Queue(Repo):
for task_name, task in job.tasks.items():
# adding CI's name to the end of the branch in order to use skip
# patterns on travis and circleci
task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
task.branch = f"{job.branch}-{task.ci}-{task_name}"
params = {
**job.params,
"arrow": job.target,
@@ -772,14 +770,14 @@ def get_version(root, **kwargs):
if 'dev' not in tag:
major += 1
return "{}.{}.{}.dev{}".format(major, minor, patch, version.distance or 0)
return f"{major}.{minor}.{patch}.dev{version.distance or 0}"
class Serializable:
@classmethod
def to_yaml(cls, representer, data):
tag = '!{}'.format(cls.__name__)
tag = f"!{cls.__name__}"
dct = {k: v for k, v in data.__dict__.items() if not k.startswith('_')}
return representer.represent_mapping(tag, dct)
@@ -943,9 +941,8 @@ class Task(Serializable):
params=params)
except jinja2.TemplateError as e:
raise RuntimeError(
'Failed to render template `{}` with {}: {}'.format(
self.template, e.__class__.__name__, str(e)
)
f"Failed to render template `{self.template}` with "
f"{e.__class__.__name__}: {e}"
)
tree = {**_default_tree, self.filename: rendered}
@@ -1092,8 +1089,8 @@ class TaskAssets(dict):
self[pattern] = github_assets[matches[0].group(0)]
else:
raise CrossbowError(
'Only a single asset should match pattern `{}`, there are '
'multiple ones: {}'.format(pattern, ', '.join(matches))
f"Only a single asset should match pattern `{pattern}`, "
f"there are multiple ones: {', '.join(matches)}"
)
def missing_patterns(self):
@@ -1239,12 +1236,12 @@ class Job(Serializable):
waited_for_minutes = (time.time() - started_at) / 60
if waited_for_minutes > poll_max_minutes:
msg = ('Exceeded the maximum amount of time waiting for job '
'to finish, waited for {} minutes.')
raise RuntimeError(msg.format(waited_for_minutes))
msg = (f'Exceeded the maximum amount of time waiting for job '
f'to finish, waited for {waited_for_minutes} minutes.')
raise RuntimeError(msg)
logger.info('Waiting {} minutes and then checking again'
.format(poll_interval_minutes))
logger.info(f'Waiting {poll_interval_minutes} minutes and then '
'checking again')
time.sleep(poll_interval_minutes * 60)
@@ -1278,9 +1275,8 @@ class Config(dict):
requested_groups = set(group_allowlist)
invalid_groups = requested_groups - valid_groups
if invalid_groups:
msg = 'Invalid group(s) {!r}. Must be one of {!r}'.format(
invalid_groups, valid_groups
)
msg = (f"Invalid group(s) {invalid_groups!r}. Must be one of "
f"{valid_groups!r}")
raise CrossbowError(msg)
# treat the task names as glob patterns to select tasks more easily
@@ -1291,7 +1287,7 @@ class Config(dict):
requested_tasks.update(matches)
else:
raise CrossbowError(
"Unable to match any tasks for `{}`".format(pattern)
f"Unable to match any tasks for `{pattern}`"
)
requested_group_tasks = set()
@@ -1309,7 +1305,7 @@ class Config(dict):
requested_group_tasks.update(matches)
else:
raise CrossbowError(
"Unable to match any tasks for `{}`".format(pattern)
f"Unable to match any tasks for `{pattern}`"
)
# remove any tasks that are negated with ~task-name
@@ -1320,7 +1316,7 @@ class Config(dict):
matches)
else:
raise CrossbowError(
"Unable to match any tasks for `{}`".format(pattern)
f"Unable to match any tasks for `{pattern}`"
)
requested_tasks = requested_tasks.union(requested_group_tasks)
@@ -1328,9 +1324,7 @@ class Config(dict):
# validate that the passed and matched tasks are defined in the config
invalid_tasks = requested_tasks - valid_tasks
if invalid_tasks:
msg = 'Invalid task(s) {!r}. Must be one of {!r}'.format(
invalid_tasks, valid_tasks
)
msg = f'Invalid task(s) {invalid_tasks!r}. Must be one of {valid_tasks!r}'
raise CrossbowError(msg)
return {
@@ -1346,9 +1340,9 @@ class Config(dict):
tasks = self.select(tasks=[pattern])
if not tasks:
raise CrossbowError(
"The pattern `{}` defined for task group `{}` is not "
"matching any of the tasks defined in the "
"configuration file.".format(pattern, group_name)
f"The pattern `{pattern}` defined for task group "
f"`{group_name}` is not matching any of the tasks "
"defined in the configuration file."
)
# validate that the tasks are constructible
@@ -1357,9 +1351,9 @@ class Config(dict):
Task(task_name, **task)
except Exception as e:
raise CrossbowError(
'Unable to construct a task object from the '
'definition of task `{}`. The original error message '
'is: `{}`'.format(task_name, str(e))
"Unable to construct a task object from the "
f"definition of task `{task_name}`. The original error "
f"message is: `{e}`"
)
# Get the default branch name from the repository
@@ -1393,8 +1387,8 @@ class Config(dict):
)
)
if not files:
raise CrossbowError('No files have been rendered for task `{}`'
.format(task_name))
raise CrossbowError("No files have been rendered for task "
f"`{task_name}`")
# configure yaml serializer
+8 -8
View File
@@ -62,10 +62,10 @@ class Report:
return url[:-4] if url.endswith('.git') else url
def url(self, query):
return '{}/branches/all?query={}'.format(self.repo_url, query)
return f'{self.repo_url}/branches/all?query={query}'
def branch_url(self, branch):
return '{}/tree/{}'.format(self.repo_url, branch)
return f'{self.repo_url}/tree/{branch}'
def task_url(self, task):
build_links = task.status().build_links
@@ -155,7 +155,7 @@ class ConsoleReport(Report):
line = self.HEADER.format(
state=state.upper(),
branch=branch,
content='uploaded {} / {}'.format(n_uploaded, n_expected)
content=f'uploaded {n_uploaded} / {n_expected}'
)
return click.style(line, fg=self.COLORS[state.lower()])
@@ -166,7 +166,7 @@ class ConsoleReport(Report):
content='Artifacts'
)
delimiter = '-' * len(header)
return '{}\n{}'.format(header, delimiter)
return f"{header}\n{delimiter}"
def artifact(self, state, pattern, asset):
if asset is None:
@@ -324,9 +324,9 @@ class CommentReport(Report):
url = 'https://github.com/{repo}/branches/all?query={branch}'
sha = self.job.target.head
msg = 'Revision: {}\n\n'.format(sha)
msg = f'Revision: {sha}\n\n'
msg += 'Submitted crossbow builds: [{repo} @ {branch}]'
msg += '({})\n'.format(url)
msg += f'({url})\n'
msg += '\n|Task|Status|\n|----|------|'
tasks = sorted(self.job.tasks.items(), key=operator.itemgetter(0))
@@ -342,8 +342,8 @@ class CommentReport(Report):
url=self.task_url(task)
)
except KeyError:
badge = 'unsupported CI service `{}`'.format(task.ci)
badge = f'unsupported CI service `{task.ci}`'
msg += '\n|{}|{}|'.format(key, badge)
msg += f'\n|{key}|{badge}|'
return msg.format(repo=self.crossbow_repo, branch=self.job.branch)
+4 -4
View File
@@ -76,7 +76,7 @@ def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli,
if not config_path.exists():
raise click.ClickException(
"Docker compose configuration cannot be found in directory {}, "
"try to pass the arrow source directory explicitly.".format(src)
f"try to pass the arrow source directory explicitly. {src}"
)
# take the Docker Compose parameters like PYTHON, PANDAS, UBUNTU from the
@@ -124,7 +124,7 @@ def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
except UndefinedImage as e:
raise click.ClickException(
"There is no service/image defined in docker-compose.yml with "
"name: {}".format(str(e))
f"name: {e}"
)
except RuntimeError as e:
raise click.ClickException(str(e))
@@ -157,7 +157,7 @@ def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
except UndefinedImage as e:
raise click.ClickException(
"There is no service/image defined in docker-compose.yml with "
"name: {}".format(str(e))
f"name: {e}"
)
except RuntimeError as e:
raise click.ClickException(str(e))
@@ -251,7 +251,7 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
except UndefinedImage as e:
raise click.ClickException(
"There is no service/image defined in docker-compose.yml with "
"name: {}".format(str(e))
f"name: {e}"
)
except RuntimeError as e:
raise click.ClickException(str(e))
+19 -24
View File
@@ -117,18 +117,18 @@ class ComposeConfig:
for name in self.with_gpus:
if name not in services:
errors.append(
'Service `{}` defined in `x-with-gpus` bot not in '
'`services`'.format(name)
f'Service `{name}` defined in `x-with-gpus` bot not in '
'`services`'
)
for name in nodes - services:
errors.append(
'Service `{}` is defined in `x-hierarchy` bot not in '
'`services`'.format(name)
f'Service `{name}` is defined in `x-hierarchy` bot not in '
'`services`'
)
for name in services - nodes:
errors.append(
'Service `{}` is defined in `services` but not in '
'`x-hierarchy`'.format(name)
f'Service `{name}` is defined in `services` but not in '
'`x-hierarchy`'
)
# trigger Docker Compose's own validation
@@ -147,9 +147,9 @@ class ComposeConfig:
errors += result.stderr.decode().splitlines()
if errors:
msg = '\n'.join([' - {}'.format(msg) for msg in errors])
msg = '\n'.join([f' - {msg}' for msg in errors])
raise ValueError(
'Found errors with docker-compose:\n{}'.format(msg)
f'Found errors with docker-compose:\n{msg}'
)
rendered_config = StringIO(result.stdout.decode())
@@ -230,9 +230,7 @@ class DockerCompose(Command):
result.check_returncode()
except subprocess.CalledProcessError as e:
raise RuntimeError(
"{} exited with non-zero exit code {}".format(
' '.join(e.cmd), e.returncode
)
f'{e.cmd} exited with non-zero exit code {e.returncode}'
)
def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True):
@@ -294,14 +292,12 @@ class DockerCompose(Command):
if self.config.using_buildx:
for k, v in service['build'].get('args', {}).items():
args.extend(['--build-arg', '{}={}'.format(k, v)])
args.extend(['--build-arg', f'{k}={v}'])
if use_cache:
cache_ref = '{}-cache'.format(service['image'])
cache_from = 'type=registry,ref={}'.format(cache_ref)
cache_to = (
'type=registry,ref={},mode=max'.format(cache_ref)
)
cache_ref = f'{service["image"]}-cache'
cache_from = f'type=registry,ref={cache_ref}'
cache_to = f'type=registry,ref={cache_ref},mode=max'
args.extend([
'--cache-from', cache_from,
'--cache-to', cache_to,
@@ -319,9 +315,9 @@ class DockerCompose(Command):
if self.config.debug and os.name != "nt":
args.append("--progress=plain")
for k, v in service['build'].get('args', {}).items():
args.extend(['--build-arg', '{}={}'.format(k, v)])
args.extend(['--build-arg', f'{k}={v}'])
for img in cache_from:
args.append('--cache-from="{}"'.format(img))
args.append(f'--cache-from="{img}"')
args.extend([
'-f', arrow_path(service['build']['dockerfile']),
'-t', service['image'],
@@ -358,13 +354,13 @@ class DockerCompose(Command):
# append env variables from the compose conf
for k, v in service.get('environment', {}).items():
if v is not None:
args.extend(['-e', '{}={}'.format(k, v)])
args.extend(['-e', f'{k}={v}'])
# append volumes from the compose conf
for v in service.get('volumes', []):
if not isinstance(v, str):
# if not the compact string volume definition
v = "{}:{}".format(v['source'], v['target'])
v = f'{v["source"]}:{v["target"]}'
args.extend(['-v', v])
# append capabilities from the compose conf
@@ -393,7 +389,7 @@ class DockerCompose(Command):
if env is not None:
for k, v in env.items():
args.extend(['-e', '{}={}'.format(k, v)])
args.extend(['-e', f'{k}={v}'])
if volumes is not None:
for volume in volumes:
@@ -443,8 +439,7 @@ class DockerCompose(Command):
self._execute_docker('login', '-u', user, '-p', password)
except subprocess.CalledProcessError:
# hide credentials
msg = ('Failed to push `{}`, check the passed credentials'
.format(service_name))
msg = f'Failed to push `{service_name}`, check the passed credentials'
raise RuntimeError(msg) from None
service = self.config.get(service_name)
@@ -199,7 +199,7 @@ def create_config(directory, yml_content, env_content=None):
if env_content is not None:
with env_path.open('w') as fp:
for k, v in env_content.items():
fp.write("{}={}\n".format(k, v))
fp.write(f"{k}={v}\n")
return config_path
+4 -8
View File
@@ -1592,8 +1592,7 @@ def generate_null_trivial_case(batch_sizes):
def generate_decimal32_case():
fields = [
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
bit_width=32)
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=32)
for i, precision in enumerate(range(3, 10))
]
@@ -1603,8 +1602,7 @@ def generate_decimal32_case():
def generate_decimal64_case():
fields = [
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
bit_width=64)
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=64)
for i, precision in enumerate(range(3, 19))
]
@@ -1614,8 +1612,7 @@ def generate_decimal64_case():
def generate_decimal128_case():
fields = [
DecimalField(name='f{}'.format(i), precision=precision, scale=2,
bit_width=128)
DecimalField(name=f'f{i}', precision=precision, scale=2, bit_width=128)
for i, precision in enumerate(range(3, 39))
]
@@ -1628,8 +1625,7 @@ def generate_decimal128_case():
def generate_decimal256_case():
fields = [
DecimalField(name='f{}'.format(i), precision=precision, scale=5,
bit_width=256)
DecimalField(name=f'f{i}', precision=precision, scale=5, bit_width=256)
for i, precision in enumerate(range(37, 70))
]
+6 -9
View File
@@ -77,8 +77,7 @@ class IntegrationRunner(object):
self.match = match
if self.match is not None:
print("-- Only running tests with {} in their name"
.format(self.match))
print(f"-- Only running tests with {self.match} in their name")
self.json_files = [json_file for json_file in self.json_files
if self.match in json_file.name]
@@ -115,7 +114,7 @@ class IntegrationRunner(object):
with group(f"Integration: Test: IPC: Gold: {consumer.name}"):
log('\n')
log('******************************************************')
log('Tests against golden files in {}'.format(gold_dir))
log(f'Tests against golden files in {gold_dir}')
log('******************************************************')
def run_gold(_, consumer, test_case: datagen.File):
@@ -280,8 +279,7 @@ class IntegrationRunner(object):
"""
with group(f"Integration: Test: IPC: {producer.name} -> {consumer.name}"):
log('##########################################################')
log('IPC: {0} producing, {1} consuming'
.format(producer.name, consumer.name))
log(f'IPC: {producer.name} producing, {consumer.name} consuming')
log('##########################################################')
case_runner = partial(self._run_ipc_test_case,
@@ -302,7 +300,7 @@ class IntegrationRunner(object):
json_path = test_case.path
log('=' * 70)
log('Testing file {0}'.format(json_path))
log(f'Testing file {json_path}')
if test_case.should_skip(producer.name, SKIP_IPC):
log(f'-- Skipping test because producer {producer.name} does '
@@ -405,8 +403,7 @@ class IntegrationRunner(object):
):
with group(f"Integration: Test: Flight: {producer.name} -> {consumer.name}"):
log('##########################################################')
log('Flight: {0} serving, {1} requesting'
.format(producer.name, consumer.name))
log(f'Flight: {producer.name} serving, {consumer.name} requesting')
log('##########################################################')
case_runner = partial(self._run_flight_test_case, producer, consumer)
@@ -423,7 +420,7 @@ class IntegrationRunner(object):
outcome = Outcome()
log('=' * 70)
log('Testing file {0}'.format(test_case.name))
log(f'Testing file {test_case.name}')
if test_case.should_skip(producer.name, SKIP_FLIGHT):
log(f'-- Skipping test because producer {producer.name} does '
@@ -120,9 +120,7 @@ class CppTester(Tester):
out, err = server.communicate()
raise RuntimeError(
"Flight-C++ server did not start properly, "
"stdout:\n{}\n\nstderr:\n{}\n".format(
output + out.decode(), err.decode()
)
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
)
port = int(output.split(":")[1])
yield port
@@ -233,10 +233,8 @@ class CSharpTester(Tester):
server.kill()
out, err = server.communicate()
raise RuntimeError(
'.NET Flight server did not start properly, '
'stdout: \n{}\n\nstderr:\n{}\n'.format(
output + out.decode(), err.decode()
)
".NET Flight server did not start properly, "
f"stdout: \n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
)
port = int(output.split(':')[-1])
yield port
+2 -4
View File
@@ -106,10 +106,8 @@ class GoTester(Tester):
server.kill()
out, err = server.communicate()
raise RuntimeError(
'Flight-Go server did not start properly, '
'stdout: \n{}\n\nstderr:\n{}\n'.format(
output + out.decode(), err.decode()
)
"Flight-Go server did not start properly, "
f"stdout: \n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
)
port = int(output.split(':')[1])
yield port
@@ -339,10 +339,8 @@ class JavaTester(Tester):
server.kill()
out, err = server.communicate()
raise RuntimeError(
'Flight-Java server did not start properly, '
'stdout:\n{}\n\nstderr:\n{}\n'.format(
output + out.decode(), err.decode()
)
"Flight-Java server did not start properly, "
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
)
port = int(output.split(':')[1])
yield port
@@ -105,10 +105,8 @@ class RustTester(Tester):
server.kill()
out, err = server.communicate()
raise RuntimeError(
'Flight-Rust server did not start properly, '
'stdout:\n{}\n\nstderr:\n{}\n'.format(
output + out.decode(), err.decode()
)
"Flight-Rust server did not start properly, "
f"stdout:\n{output + out.decode()}\n\nstderr:\n{err.decode()}\n"
)
port = int(output.split(':')[1])
yield port
+3 -3
View File
@@ -164,7 +164,7 @@ class CppConfiguration:
return self._cc
if self.with_fuzzing:
return "clang-{}".format(LLVM_VERSION)
return f"clang-{LLVM_VERSION}"
return None
@@ -174,7 +174,7 @@ class CppConfiguration:
return self._cxx
if self.with_fuzzing:
return "clang++-{}".format(LLVM_VERSION)
return f"clang++-{LLVM_VERSION}"
return None
@@ -277,7 +277,7 @@ class CppConfiguration:
@property
def definitions(self):
extras = list(self.cmake_extras) if self.cmake_extras else []
definitions = ["-D{}={}".format(d[0], d[1]) for d in self._gen_defs()]
definitions = [f"-D{d[0]}={d[1]}" for d in self._gen_defs()]
return definitions + extras
@property
+1 -1
View File
@@ -249,7 +249,7 @@ class NumpyDoc:
try:
obj = Docstring._load_obj(symbol)
except (ImportError, AttributeError):
print('{} is not available for import'.format(symbol))
print(f'{symbol} is not available for import')
else:
self.traverse(callback, obj, from_package=from_package)
+2 -2
View File
@@ -108,7 +108,7 @@ def release_changelog_regenerate(obj):
continue
release = Release(version, repo=repo,
issue_tracker=issue_tracker)
click.echo('Querying changelog for version: {}'.format(version))
click.echo(f'Querying changelog for version: {version}')
changelogs.append(release.changelog())
click.echo('Rendering new CHANGELOG.md file...')
@@ -139,4 +139,4 @@ def release_cherry_pick(obj, version, dry_run, recreate):
else:
click.echo(f'git checkout -b {release.branch} {release.base_branch}')
for commit in release.commits_to_pick():
click.echo('git cherry-pick {}'.format(commit.hexsha))
click.echo(f'git cherry-pick {commit.hexsha}')
+6 -7
View File
@@ -183,7 +183,7 @@ class CommitTitle:
matches = _TITLE_REGEX.match(headline)
if matches is None:
warnings.warn(
"Unable to parse commit message `{}`".format(headline)
f"Unable to parse commit message `{headline}`"
)
return CommitTitle(headline)
@@ -203,10 +203,10 @@ class CommitTitle:
def to_string(self, with_issue=True, with_components=True):
out = ""
if with_issue and self.issue:
out += "{}: ".format(self.issue)
out += f"{self.issue}: "
if with_components and self.components:
for component in self.components:
out += "[{}]".format(component)
out += f"[{component}]"
out += " "
out += self.summary
return out
@@ -225,13 +225,12 @@ class Commit:
return getattr(self._wrapped, attr)
def __repr__(self):
template = '<Commit sha={!r} issue={!r} components={!r} summary={!r}>'
return template.format(self.hexsha, self.issue, self.components,
self.summary)
return (f'<Commit sha={self.hexsha!r} issue={self.issue!r} '
f'components={self.components!r} summary={self.summary!r}>')
@property
def url(self):
return 'https://github.com/apache/arrow/commit/{}'.format(self.hexsha)
return f'https://github.com/apache/arrow/commit/{self.hexsha}'
@property
def title(self):
+1 -1
View File
@@ -50,7 +50,7 @@ def set_env_vars():
def github_url(path):
return 'https://api.github.com:443/{}'.format(path.strip('/'))
return f'https://api.github.com:443/{path.strip("/")}'
@group()
+1 -1
View File
@@ -70,7 +70,7 @@ class Cache:
"""
def download(path):
""" Tiny wrapper that download a file and save as key. """
logger.debug("Downloading {} as {}".format(url, path))
logger.debug(f"Downloading {url} as {path}")
conn = urlopen(url)
# Ensure the download is completed before writing to disks.
content = conn.read()
+8 -15
View File
@@ -79,7 +79,7 @@ class CMakeDefinition:
def arguments(self):
"""" Return the arguments to cmake invocation. """
arguments = [
"-G{}".format(self.generator),
f"-G{self.generator}",
] + self.definitions + [
self.source
]
@@ -99,13 +99,9 @@ class CMakeDefinition:
if os.path.exists(build_dir):
# Extra safety to ensure we're deleting a build folder.
if not CMakeBuild.is_build_dir(build_dir):
raise FileExistsError(
"{} is not a cmake build".format(build_dir)
)
raise FileExistsError(f"{build_dir} is not a cmake build")
if not force:
raise FileExistsError(
"{} exists use force=True".format(build_dir)
)
raise FileExistsError(f"{build_dir} exists use force=True")
rmtree(build_dir)
os.mkdir(build_dir)
@@ -116,7 +112,7 @@ class CMakeDefinition:
**kwargs)
def __repr__(self):
return "CMakeDefinition[source={}]".format(self.source)
return f"CMakeDefinition[source={self.source}]"
CMAKE_BUILD_TYPE_RE = re.compile("CMAKE_BUILD_TYPE:STRING=([a-zA-Z]+)")
@@ -194,7 +190,7 @@ class CMakeBuild(CMake):
be lost. Only build_type is recovered.
"""
if not CMakeBuild.is_build_dir(path):
raise ValueError("Not a valid CMakeBuild path: {}".format(path))
raise ValueError(f"Not a valid CMakeBuild path: {path}")
build_type = None
# Infer build_type by looking at CMakeCache.txt and looking for a magic
@@ -207,9 +203,6 @@ class CMakeBuild(CMake):
return CMakeBuild(path, build_type)
def __repr__(self):
return ("CMakeBuild["
"build = {},"
"build_type = {},"
"definition = {}]".format(self.build_dir,
self.build_type,
self.definition))
return (f"CMakeBuild[build = {self.build_dir},"
f"build_type = {self.build_type},"
f"definition = {self.definition}]")
+2 -2
View File
@@ -25,7 +25,7 @@ from .logger import logger, ctx
def default_bin(name, default):
assert default
env_name = "ARCHERY_{0}_BIN".format(default.upper())
env_name = f"ARCHERY_{default.upper()}_BIN"
return name if name else os.environ.get(env_name, default)
@@ -74,7 +74,7 @@ class Command:
if "check" not in kwargs:
kwargs["check"] = True
logger.debug("Executing `{}`".format(invocation))
logger.debug(f"Executing `{invocation}`")
return subprocess.run(invocation, **kwargs)
@property
+4 -6
View File
@@ -355,18 +355,16 @@ def python_numpydoc(symbols=None, allow_rules=None, disallow_rules=None):
qualname_with_signature = '.'.join([module, cython_signature])
click.echo(
click.style(
'-> {}'.format(qualname_with_signature),
f'-> {qualname_with_signature}',
fg='yellow'
)
)
for error in errors:
number_of_violations += 1
click.echo('{}: {}'.format(*error))
click.echo(f'{error[0]}: {error[1]}')
msg = 'Total number of docstring violations: {}'.format(
number_of_violations
)
msg = f'Total number of docstring violations: {number_of_violations}'
click.echo()
click.echo(click.style(msg, fg='red'))
@@ -392,7 +390,7 @@ def rat_linter(src, root):
violations = list(report.validate(exclusion=exclusion))
for violation in violations:
print("apache-rat license violation: {}".format(violation))
print(f"apache-rat license violation: {violation}")
yield LintResult(len(violations) == 0)
+5 -6
View File
@@ -85,7 +85,7 @@ class MavenDefinition:
# Extra safety to ensure we're deleting a build folder.
if not MavenBuild.is_build_dir(build_dir):
raise FileExistsError(
"{} is not a maven build".format(build_dir)
f"{build_dir} is not a maven build"
)
cmd_kwargs = cmd_kwargs if cmd_kwargs else {}
@@ -113,7 +113,7 @@ class MavenDefinition:
return arguments
def __repr__(self):
return "MavenDefinition[source={}]".format(self.source)
return f"MavenDefinition[source={self.source}]"
class MavenBuild(Maven):
@@ -193,12 +193,11 @@ class MavenBuild(Maven):
be lost.
"""
if not MavenBuild.is_build_dir(path):
raise ValueError("Not a valid MavenBuild path: {}".format(path))
raise ValueError(f"Not a valid MavenBuild path: {path}")
return MavenBuild(path, definition=None)
def __repr__(self):
return ("MavenBuild["
"build = {},"
"definition = {}]".format(self.build_dir,
self.definition))
f"build = {self.build_dir},"
f"definition = {self.definition}]")
+2 -2
View File
@@ -24,7 +24,7 @@ from .cache import Cache
from .command import capture_stdout
RAT_VERSION = 0.13
RAT_JAR_FILENAME = "apache-rat-{}.jar".format(RAT_VERSION)
RAT_JAR_FILENAME = f"apache-rat-{RAT_VERSION}.jar"
RAT_URL_ = "https://repo1.maven.org/maven2/org/apache/rat/apache-rat"
RAT_URL = "/".join([RAT_URL_, str(RAT_VERSION), RAT_JAR_FILENAME])
@@ -54,7 +54,7 @@ class RatReport:
self.tree = ElementTree.fromstring(xml)
def __repr__(self):
return "RatReport({})".format(self.xml)
return f"RatReport({self.xml})"
def validate(self, exclusion=None):
for r in self.tree.findall('resource'):
+1 -1
View File
@@ -32,7 +32,7 @@ class Report(metaclass=ABCMeta):
def __init__(self, **kwargs):
for field in self.fields:
if field not in kwargs:
raise ValueError('Missing keyword argument {}'.format(field))
raise ValueError(f'Missing keyword argument {field}')
self._data = kwargs
def __getattr__(self, key):
+5 -5
View File
@@ -64,7 +64,7 @@ class ArrowSources:
# validate by checking a specific path in the arrow source tree
if not (path / 'cpp' / 'CMakeLists.txt').exists():
raise InvalidArrowSource(
"No Arrow C++ sources found in {}.".format(path)
f"No Arrow C++ sources found in {path}."
)
self.path = path
@@ -116,7 +116,7 @@ class ArrowSources:
def archive(self, path, dereference=False, compressor=None, revision=None):
""" Saves a git archive at path. """
if not self.git_backed:
raise ValueError("{} is not backed by git".format(self))
raise ValueError(f"{self} is not backed by git")
rev = revision if revision else "HEAD"
archive = git.archive("--prefix=apache-arrow.tmp/", rev,
@@ -160,7 +160,7 @@ class ArrowSources:
Path to checkout the local clone.
"""
if not self.git_backed:
raise ValueError("{} is not backed by git".format(self))
raise ValueError(f"{self} is not backed by git")
if revision == ArrowSources.WORKSPACE:
return self, False
@@ -226,10 +226,10 @@ class ArrowSources:
except InvalidArrowSource:
pass
searched_paths = "\n".join([" - {}".format(p) for p in paths])
searched_paths = "\n".join([f" - {p}" for p in paths])
raise InvalidArrowSource(
"Unable to locate Arrow's source directory. "
"Searched paths are:\n{}".format(searched_paths)
f"Searched paths are:\n{searched_paths}"
)
def __repr__(self):
+10 -11
View File
@@ -235,13 +235,12 @@ def format_issue_output(issue_type, issue_id, status,
url = f'https://github.com/{ORG_NAME}/{PROJECT_NAME}/issues/{url_id}'
return """=== {} {} ===
Summary\t\t{}
Assignee\t{}
Components\t{}
Status\t\t{}
URL\t\t{}""".format(issue_type.upper(), issue_id, summary, assignee,
components, status, url)
return f"""=== {issue_type.upper()} {issue_id} ===
Summary\t\t{summary}
Assignee\t{assignee}
Components\t{components}
Status\t\t{status}
URL\t\t{url}"""
class GitHubAPI(object):
@@ -263,7 +262,7 @@ class GitHubAPI(object):
'(GitHub personal access token):')
headers = {
'Accept': 'application/vnd.github.v3+json',
'Authorization': 'token {0}'.format(token),
'Authorization': f'token {token}',
}
self.headers = headers
@@ -429,7 +428,7 @@ class PullRequest(object):
return GitHubIssue(self._github_api, github_id, self.cmd)
self.cmd.fail("PR title should be prefixed by a GitHub ID, like: "
"GH-XXX, but found {0}".format(self.title))
f"GH-XXX, but found {self.title}")
def merge(self):
"""
@@ -459,7 +458,7 @@ class PullRequest(object):
reverse=True)
for i, author in enumerate(distinct_authors):
print("Author {}: {}".format(i + 1, author))
print(f"Author {i + 1}: {author}")
if len(distinct_authors) > 1:
primary_author, distinct_other_authors = get_primary_author(
@@ -533,7 +532,7 @@ def get_primary_author(cmd, distinct_authors):
if author_pat.match(primary_author):
break
print('Bad author "{}", please try again'.format(primary_author))
print(f'Bad author "{primary_author}", please try again')
# When primary author is specified manually, de-dup it from
# author list and put it at the head of author list.
+2 -2
View File
@@ -103,7 +103,7 @@ class Downloader:
dest_path = os.path.join(dest_dir, filename)
print("Downloading {} to {}".format(path, dest_path))
print(f"Downloading {path} to {dest_path}")
url = f"{self.URL_ROOT}/{path}"
self._download_url(url, dest_path)
@@ -272,7 +272,7 @@ def download_rc_binaries(
repository=None,
tag=None,
):
version_string = "{}-rc{}".format(version, rc_number)
version_string = f'{version}-rc{rc_number}'
version_pattern = re.compile(r"\d+\.\d+\.\d+")
if target_package_type:
package_types = [target_package_type]
+1 -1
View File
@@ -382,7 +382,7 @@ html_context = {
# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#
html_title = u'Apache Arrow v{}'.format(version)
html_title = f"Apache Arrow v{version}"
# A shorter title for the navigation bar. Default is the same as html_title.
#
+2 -2
View File
@@ -37,7 +37,7 @@ class ParquetWriteBinary(object):
i in range(nuniques)], dtype='O')
values = unique_values[np.random.randint(0, nuniques, size=length)]
self.table = pa.table([pa.array(values) for i in range(num_cols)],
names=['f{}'.format(i) for i in range(num_cols)])
names=[f'f{i}' for i in range(num_cols)])
self.table_df = self.table.to_pandas()
def time_write_binary_table(self):
@@ -72,7 +72,7 @@ def generate_dict_table(num_cols, string_size, nunique, length,
random_order=random_order)
return pa.table([
data for i in range(num_cols)
], names=['f{}'.format(i) for i in range(num_cols)])
], names=[f'f{i}' for i in range(num_cols)])
class ParquetWriteDictionaries(object):
+3 -4
View File
@@ -99,8 +99,7 @@ class FlightServer(pyarrow.flight.FlightServerBase):
def do_action(self, context, action):
if action.type == "clear":
raise NotImplementedError(
"{} is not implemented.".format(action.type))
raise NotImplementedError(f"{action.type} is not implemented.")
elif action.type == "healthcheck":
pass
elif action.type == "shutdown":
@@ -109,7 +108,7 @@ class FlightServer(pyarrow.flight.FlightServerBase):
# request
threading.Thread(target=self._shutdown).start()
else:
raise KeyError("Unknown action {!r}".format(action.type))
raise KeyError(f"Unknown action {action.type!r}")
def _shutdown(self):
"""Shut down after a delay."""
@@ -141,7 +140,7 @@ def main():
tls_private_key = key_file.read()
tls_certificates.append((tls_cert_chain, tls_private_key))
location = "{}://{}:{}".format(scheme, args.host, args.port)
location = f"{scheme}://{args.host}:{args.port}"
server = FlightServer(args.host, location,
tls_certificates=tls_certificates,
+1 -1
View File
@@ -408,7 +408,7 @@ def get_library_dirs():
if not library_dir.startswith("-L"):
raise ValueError(
"pkg-config --libs-only-L returned unexpected "
"value {!r}".format(library_dir))
f"value {library_dir!r}")
append_library_dir(library_dir[2:])
if _sys.platform == 'win32':
+1 -1
View File
@@ -553,7 +553,7 @@ cdef class Declaration(_Weakrefable):
return frombytes(GetResultValue(DeclarationToString(self.decl)))
def __repr__(self):
return "<pyarrow.acero.Declaration>\n{0}".format(str(self))
return f"<pyarrow.acero.Declaration>\n{self}"
def to_table(self, bint use_threads=True):
"""
+11 -26
View File
@@ -63,14 +63,10 @@ def _pas():
def _forbid_instantiation(klass, subclasses_instead=True):
msg = '{} is an abstract class thus cannot be initialized.'.format(
klass.__name__
)
msg = f'{klass.__name__} is an abstract class thus cannot be initialized.'
if subclasses_instead:
subclasses = [cls.__name__ for cls in klass.__subclasses__]
msg += ' Use one of the subclasses instead: {}'.format(
', '.join(subclasses)
)
msg += f' Use one of the subclasses instead: {", ".join(subclasses)}'
raise TypeError(msg)
@@ -201,8 +197,7 @@ cdef class Kernel(_Weakrefable):
"""
def __init__(self):
raise TypeError("Do not call {}'s constructor directly"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
cdef class ScalarKernel(Kernel):
@@ -212,8 +207,7 @@ cdef class ScalarKernel(Kernel):
self.kernel = kernel
def __repr__(self):
return ("ScalarKernel<{}>"
.format(frombytes(self.kernel.signature.get().ToString())))
return f"ScalarKernel<{frombytes(self.kernel.signature.get().ToString())}>"
cdef class VectorKernel(Kernel):
@@ -223,8 +217,7 @@ cdef class VectorKernel(Kernel):
self.kernel = kernel
def __repr__(self):
return ("VectorKernel<{}>"
.format(frombytes(self.kernel.signature.get().ToString())))
return f"VectorKernel<{frombytes(self.kernel.signature.get().ToString())}>"
cdef class ScalarAggregateKernel(Kernel):
@@ -234,8 +227,7 @@ cdef class ScalarAggregateKernel(Kernel):
self.kernel = kernel
def __repr__(self):
return ("ScalarAggregateKernel<{}>"
.format(frombytes(self.kernel.signature.get().ToString())))
return f"ScalarAggregateKernel<{frombytes(self.kernel.signature.get().ToString())}>"
cdef class HashAggregateKernel(Kernel):
@@ -245,8 +237,7 @@ cdef class HashAggregateKernel(Kernel):
self.kernel = kernel
def __repr__(self):
return ("HashAggregateKernel<{}>"
.format(frombytes(self.kernel.signature.get().ToString())))
return f"HashAggregateKernel<{frombytes(self.kernel.signature.get().ToString())}>"
FunctionDoc = namedtuple(
@@ -298,17 +289,14 @@ cdef class Function(_Weakrefable):
}
def __init__(self):
raise TypeError("Do not call {}'s constructor directly"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
cdef void init(self, const shared_ptr[CFunction]& sp_func) except *:
self.sp_func = sp_func
self.base_func = sp_func.get()
def __repr__(self):
return ("arrow.compute.Function<name={}, kind={}, "
"arity={}, num_kernels={}>"
.format(self.name, self.kind, self.arity, self.num_kernels))
return f"arrow.compute.Function<name={self.name}, kind={self.kind}, arity={self.arity}, num_kernels={self.num_kernels}>"
def __reduce__(self):
# Reduction uses the global registry
@@ -2568,9 +2556,7 @@ cdef class Expression(_Weakrefable):
return frombytes(self.expr.ToString())
def __repr__(self):
return "<pyarrow.compute.{0} {1}>".format(
self.__class__.__name__, str(self)
)
return f"<pyarrow.compute.{self.__class__.__name__} {self}>"
@staticmethod
def from_substrait(object message not None):
@@ -2874,8 +2860,7 @@ cdef class UdfContext:
"""
def __init__(self):
raise TypeError("Do not call {}'s constructor directly"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly")
cdef void init(self, const CUdfContext &c_context):
self.c_context = c_context
+2 -3
View File
@@ -1150,9 +1150,8 @@ cdef class CSVStreamingReader(RecordBatchReader):
Schema schema
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
"use pyarrow.csv.open_csv() instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use pyarrow.csv.open_csv() instead.")
# Note about cancellation: we cannot create a SignalStopHandler
# by default here, as several CSVStreamingReader instances may be
+1 -2
View File
@@ -841,8 +841,7 @@ cdef class BufferWriter(NativeFile):
offset = offset + position
else:
with gil:
raise ValueError("Invalid value of whence: {0}"
.format(whence))
raise ValueError(f"Invalid value of whence: {whence}")
check_status(self.writer.Seek(offset))
return self.tell()
+7 -9
View File
@@ -1128,7 +1128,7 @@ cdef class FileSystemDataset(Dataset):
elif not isinstance(root_partition, Expression):
raise TypeError(
"Argument 'root_partition' has incorrect type (expected "
"Expression, got {0})".format(type(root_partition))
f"Expression, got {type(root_partition)})"
)
for fragment in fragments:
@@ -1220,8 +1220,8 @@ cdef class FileSystemDataset(Dataset):
]:
if not isinstance(arg, class_):
raise TypeError(
"Argument '{0}' has incorrect type (expected {1}, "
"got {2})".format(name, class_.__name__, type(arg))
f"Argument '{name}' has incorrect type (expected {class_.__name__}, "
f"got {type(arg)})"
)
partitions = partitions or [_true] * len(paths)
@@ -1988,9 +1988,7 @@ cdef class FileFragment(Fragment):
)
if partition:
partition = f" partition=[{partition}]"
return "<pyarrow.dataset.{0}{1} path={2}{3}>".format(
self.__class__.__name__, typ, self.path, partition
)
return f"<pyarrow.dataset.{self.__class__.__name__}{typ} path={self.path}{partition}>"
def __reduce__(self):
buffer = self.buffer
@@ -3385,7 +3383,7 @@ cdef class FileSystemDatasetFactory(DatasetFactory):
)
else:
raise TypeError('Must pass either paths or a FileSelector, but '
'passed {}'.format(type(paths_or_selector)))
f'passed {type(paths_or_selector)}')
self.init(GetResultValue(result))
@@ -3528,7 +3526,7 @@ cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
if not isinstance(expr, Expression):
raise TypeError(
"Expected an Expression for a 'column' dictionary "
"value, got {} instead".format(type(expr))
f"value, got {type(expr)} instead"
)
c_exprs.push_back((<Expression> expr).unwrap())
@@ -3540,7 +3538,7 @@ cdef void _populate_builder(const shared_ptr[CScannerBuilder]& ptr,
else:
raise ValueError(
"Expected a list or a dict for 'columns', "
"got {} instead.".format(type(columns))
f"got {type(columns)} instead."
)
check_status(builder.BatchSize(batch_size))
+2 -4
View File
@@ -336,7 +336,7 @@ class RowGroupInfo:
}
def __repr__(self):
return "RowGroupInfo({})".format(self.id)
return f"RowGroupInfo({self.id})"
def __eq__(self, other):
if isinstance(other, int):
@@ -671,9 +671,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions):
self._set_arrow_properties()
def __repr__(self):
return "<pyarrow.dataset.ParquetFileWriteOptions {0}>".format(
" ".join([f"{key}={value}" for key, value in self._properties.items()])
)
return f"<pyarrow.dataset.ParquetFileWriteOptions {self._properties}>"
cdef set _PARQUET_READ_OPTIONS = {
+28 -41
View File
@@ -88,8 +88,7 @@ _FLIGHT_SERVER_ERROR_REGEX = re.compile(
def _munge_grpc_python_error(message):
m = _FLIGHT_SERVER_ERROR_REGEX.match(message)
if m:
return ('Flight RPC failed with Python exception \"{}: {}\"'
.format(m.group(2), m.group(1)))
return f'Flight RPC failed with Python exception "{m.group(2)}: {m.group(1)}"'
else:
return message
@@ -130,8 +129,7 @@ cdef class FlightCallOptions(_Weakrefable):
self.options.write_options = c_write_options.c_options
if read_options is not None:
if not isinstance(read_options, IpcReadOptions):
raise TypeError("expected IpcReadOptions, got {}"
.format(type(read_options)))
raise TypeError(f"expected IpcReadOptions, got {type(read_options)}")
self.options.read_options = read_options.c_options
if headers is not None:
self.options.headers = headers
@@ -142,8 +140,7 @@ cdef class FlightCallOptions(_Weakrefable):
return &DEFAULT_CALL_OPTIONS
elif isinstance(obj, FlightCallOptions):
return &((<FlightCallOptions> obj).options)
raise TypeError("Expected a FlightCallOptions object, not "
"'{}'".format(type(obj)))
raise TypeError(f"Expected a FlightCallOptions object, not '{type(obj)}'")
_CertKeyPair = collections.namedtuple('_CertKeyPair', ['cert', 'key'])
@@ -182,7 +179,7 @@ cdef class FlightError(Exception):
self.extra_info = tobytes(extra_info)
cdef CStatus to_status(self):
message = tobytes("Flight error: {}".format(str(self)))
message = tobytes(f"Flight error: {self}")
return CStatus_UnknownError(message)
@@ -280,8 +277,7 @@ cdef class Action(_Weakrefable):
@staticmethod
cdef CAction unwrap(action) except *:
if not isinstance(action, Action):
raise TypeError("Must provide Action, not '{}'".format(
type(action)))
raise TypeError(f"Must provide Action, not '{type(action)}'")
return (<Action> action).action
def serialize(self):
@@ -493,10 +489,9 @@ cdef class FlightDescriptor(_Weakrefable):
CFlightDescriptor descriptor
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
"`pyarrow.flight.FlightDescriptor.for_{path,command}` "
"function instead."
.format(self.__class__.__name__))
"function instead.")
@staticmethod
def for_path(*path):
@@ -552,8 +547,8 @@ cdef class FlightDescriptor(_Weakrefable):
@staticmethod
cdef CFlightDescriptor unwrap(descriptor) except *:
if not isinstance(descriptor, FlightDescriptor):
raise TypeError("Must provide a FlightDescriptor, not '{}'".format(
type(descriptor)))
raise TypeError(
f"Must provide a FlightDescriptor, not '{type(descriptor)}'")
return (<FlightDescriptor> descriptor).descriptor
def serialize(self):
@@ -693,8 +688,7 @@ cdef class Location(_Weakrefable):
CLocation.Parse(tobytes(location)).Value(&c_location))
return c_location
elif not isinstance(location, Location):
raise TypeError("Must provide a Location, not '{}'".format(
type(location)))
raise TypeError(f"Must provide a Location, not '{type(location)}'")
return (<Location> location).location
@@ -733,7 +727,7 @@ cdef class FlightEndpoint(_Weakrefable):
self.endpoint.ticket.ticket = tobytes(ticket)
else:
raise TypeError("Argument ticket must be a Ticket instance, string or bytes, "
"not '{}'".format(type(ticket)))
f"not '{type(ticket)}'")
for location in locations:
if isinstance(location, Location):
@@ -744,7 +738,7 @@ cdef class FlightEndpoint(_Weakrefable):
CLocation.Parse(tobytes(location)).Value(&c_location))
else:
raise TypeError("Argument locations must contain Location instances, strings or bytes, "
"not '{}'".format(type(location)))
f"not '{type(location)}'")
self.endpoint.locations.push_back(c_location)
if expiration_time is not None:
@@ -753,11 +747,11 @@ cdef class FlightEndpoint(_Weakrefable):
expiration_time.cast(timestamp("ns")).value)
else:
raise TypeError("Argument expiration_time must be a TimestampScalar, "
"not '{}'".format(type(expiration_time)))
f"not '{type(expiration_time)}'")
if not isinstance(app_metadata, (str, bytes)):
raise TypeError("Argument app_metadata must be a string or bytes, "
"not '{}'".format(type(app_metadata)))
f"not '{type(app_metadata)}'")
self.endpoint.app_metadata = tobytes(app_metadata)
@property
@@ -919,8 +913,8 @@ cdef class FlightInfo(_Weakrefable):
if isinstance(endpoint, FlightEndpoint):
c_endpoints.push_back((<FlightEndpoint> endpoint).endpoint)
else:
raise TypeError('Endpoint {} is not instance of'
' FlightEndpoint'.format(endpoint))
raise TypeError(
f'Endpoint {endpoint} is not instance of FlightEndpoint')
check_flight_status(CreateFlightInfo(c_schema,
descriptor.descriptor,
@@ -1048,8 +1042,7 @@ cdef class FlightStreamChunk(_Weakrefable):
return iter((self.data, self.app_metadata))
def __repr__(self):
return "<FlightStreamChunk with data: {} with metadata: {}>".format(
self.chunk.data != NULL, self.chunk.app_metadata != NULL)
return f"<FlightStreamChunk with data: {self.chunk.data != NULL} with metadata: {self.chunk.app_metadata != NULL}>"
cdef class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin):
@@ -1527,8 +1520,7 @@ cdef class FlightClient(_Weakrefable):
if not isinstance(auth_handler, ClientAuthHandler):
raise TypeError(
"FlightClient.authenticate takes a ClientAuthHandler, "
"not '{}'".format(type(auth_handler)))
f"FlightClient.authenticate takes a ClientAuthHandler, not '{type(auth_handler)}'")
handler.reset((<ClientAuthHandler> auth_handler).to_handler())
with nogil:
check_flight_status(
@@ -1831,7 +1823,7 @@ cdef class RecordBatchStream(FlightDataStream):
if (not isinstance(data_source, RecordBatchReader) and
not isinstance(data_source, lib.Table)):
raise TypeError("Expected RecordBatchReader or Table, "
"but got: {}".format(type(data_source)))
f"but got: {type(data_source)}")
self.data_source = data_source
self.write_options = _get_options(options).c_options
@@ -1844,8 +1836,8 @@ cdef class RecordBatchStream(FlightDataStream):
table = (<Table> self.data_source).table
reader.reset(new TableBatchReader(deref(table)))
else:
raise RuntimeError("Can't construct RecordBatchStream "
"from type {}".format(type(self.data_source)))
raise RuntimeError(
f"Can't construct RecordBatchStream from type {type(self.data_source)}")
return new CRecordBatchStream(reader, self.write_options)
@@ -2127,8 +2119,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
raise ValueError("Got a FlightDataStream whose schema "
"does not match the declared schema of this "
"GeneratorStream. "
"Got: {}\nExpected: {}".format(
substream_schema, stream_schema))
f"Got: {substream_schema}\nExpected: {stream_schema}")
stream.current_stream.reset(
new CPyFlightDataStream(result, move(data_stream)))
# Loop around and try again
@@ -2139,8 +2130,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
raise ValueError("Got a RecordBatch whose schema does not "
"match the declared schema of this "
"GeneratorStream. "
"Got: {}\nExpected: {}".format(batch.schema,
stream_schema))
f"Got: {batch.schema}\nExpected: {stream_schema}")
check_flight_status(GetRecordBatchPayload(
deref(batch.batch),
stream.c_options,
@@ -2152,7 +2142,7 @@ cdef CStatus _data_stream_next(void* self, CFlightPayload* payload) except *:
raise TypeError("GeneratorStream must be initialized with "
"an iterator of FlightDataStream, Table, "
"RecordBatch, or RecordBatchStreamReader objects, "
"not {}.".format(type(result)))
f"not {type(result)}.")
# Don't loop around
return CStatus_OK()
# Ran out of attempts (the RPC handler kept yielding empty tables/readers)
@@ -2174,8 +2164,7 @@ cdef CStatus _list_flights(void* self, const CServerCallContext& context,
for info in result:
if not isinstance(info, FlightInfo):
raise TypeError("FlightServerBase.list_flights must return "
"FlightInfo instances, but got {}".format(
type(info)))
f"FlightInfo instances, but got {type(info)}")
flights.push_back(deref((<FlightInfo> info).info.get()))
listing.reset(new CSimpleFlightListing(flights))
except FlightError as flight_error:
@@ -2199,8 +2188,7 @@ cdef CStatus _get_flight_info(void* self, const CServerCallContext& context,
return (<FlightError> flight_error).to_status()
if not isinstance(result, FlightInfo):
raise TypeError("FlightServerBase.get_flight_info must return "
"a FlightInfo instance, but got {}".format(
type(result)))
f"a FlightInfo instance, but got {type(result)}")
info.reset(new CFlightInfo(deref((<FlightInfo> result).info.get())))
return CStatus_OK()
@@ -2216,8 +2204,7 @@ cdef CStatus _get_schema(void* self, const CServerCallContext& context,
py_descriptor)
if not isinstance(result, SchemaResult):
raise TypeError("FlightServerBase.get_schema_info must return "
"a SchemaResult instance, but got {}".format(
type(result)))
f"a SchemaResult instance, but got {type(result)}")
info.reset(new CSchemaResult(deref((<SchemaResult> result).result.get())))
return CStatus_OK()
@@ -2943,7 +2930,7 @@ cdef class FlightServerBase(_Weakrefable):
if auth_handler:
if not isinstance(auth_handler, ServerAuthHandler):
raise TypeError("auth_handler must be a ServerAuthHandler, "
"not a '{}'".format(type(auth_handler)))
f"not a '{type(auth_handler)}'")
c_options.get().auth_handler.reset(
(<ServerAuthHandler> auth_handler).to_handler())
+7 -10
View File
@@ -180,8 +180,7 @@ cdef class FileInfo(_Weakrefable):
@staticmethod
cdef CFileInfo unwrap_safe(obj):
if not isinstance(obj, FileInfo):
raise TypeError("Expected FileInfo instance, got {0}"
.format(type(obj)))
raise TypeError(f"Expected FileInfo instance, got {type(obj)}")
return (<FileInfo> obj).unwrap()
def __repr__(self):
@@ -406,8 +405,7 @@ cdef class FileSelector(_Weakrefable):
self.selector.recursive = recursive
def __repr__(self):
return ("<FileSelector base_dir={0.base_dir!r} "
"recursive={0.recursive}>".format(self))
return f"<FileSelector base_dir={self.base_dir!r} recursive={self.recursive}>"
cdef class FileSystem(_Weakrefable):
@@ -449,7 +447,7 @@ cdef class FileSystem(_Weakrefable):
--------
Create a new FileSystem subclass from a URI:
>>> uri = 'file:///{}/pyarrow-fs-example.dat'.format(local_path)
>>> uri = f'file:///{local_path}/pyarrow-fs-example.dat'
>>> local_new, path_new = fs.FileSystem.from_uri(uri)
>>> local_new
<pyarrow._fs.LocalFileSystem object at ...
@@ -564,7 +562,7 @@ cdef class FileSystem(_Weakrefable):
--------
>>> local
<pyarrow._fs.LocalFileSystem object at ...>
>>> local.get_file_info("/{}/pyarrow-fs-example.dat".format(local_path))
>>> local.get_file_info(f"/{local_path}/pyarrow-fs-example.dat")
<FileInfo for '/.../pyarrow-fs-example.dat': type=FileType.File, size=4>
"""
cdef:
@@ -1192,8 +1190,7 @@ cdef class SubTreeFileSystem(FileSystem):
self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
def __repr__(self):
return ("SubTreeFileSystem(base_path={}, base_fs={}"
.format(self.base_path, self.base_fs))
return f"SubTreeFileSystem(base_path={self.base_path}, base_fs={self.base_fs})"
def __reduce__(self):
return SubTreeFileSystem, (
@@ -1260,8 +1257,8 @@ cdef class PyFileSystem(FileSystem):
shared_ptr[CPyFileSystem] wrapped
if not isinstance(handler, FileSystemHandler):
raise TypeError("Expected a FileSystemHandler instance, got {0}"
.format(type(handler)))
raise TypeError(
f"Expected a FileSystemHandler instance, got {type(handler)}")
vtable.get_type_name = _cb_get_type_name
vtable.equals = _cb_equals
+1 -1
View File
@@ -74,7 +74,7 @@ cdef class HadoopFileSystem(FileSystem):
if not host.startswith(('hdfs://', 'viewfs://')) and host != "default":
# TODO(kszucs): do more sanitization
host = 'hdfs://{}'.format(host)
host = f'hdfs://{host}'
options.ConfigureEndPoint(tobytes(host), int(port))
options.ConfigureReplication(replication)
+2 -2
View File
@@ -216,8 +216,8 @@ cdef class ParseOptions(_Weakrefable):
v = CUnexpectedFieldBehavior_InferType
else:
raise ValueError(
"Unexpected value `{}` for `unexpected_field_behavior`, pass "
"either `ignore`, `error` or `infer`.".format(value)
f"Unexpected value `{value}` for `unexpected_field_behavior`, pass "
f"either `ignore`, `error` or `infer`."
)
self.options.unexpected_field_behavior = v
+64 -111
View File
@@ -59,25 +59,16 @@ cdef class Statistics(_Weakrefable):
pass
def __repr__(self):
return """{}
has_min_max: {}
min: {}
max: {}
null_count: {}
distinct_count: {}
num_values: {}
physical_type: {}
logical_type: {}
converted_type (legacy): {}""".format(object.__repr__(self),
self.has_min_max,
self.min,
self.max,
self.null_count,
self.distinct_count,
self.num_values,
self.physical_type,
str(self.logical_type),
self.converted_type)
return f"""{object.__repr__(self)}
has_min_max: {self.has_min_max}
min: {self.min}
max: {self.max}
null_count: {self.null_count}
distinct_count: {self.distinct_count}
num_values: {self.num_values}
physical_type: {self.physical_type}
logical_type: {self.logical_type}
converted_type (legacy): {self.converted_type}"""
def to_dict(self):
"""
@@ -234,7 +225,7 @@ cdef class ParquetLogicalType(_Weakrefable):
self.type = type
def __repr__(self):
return "{}\n {}".format(object.__repr__(self), str(self))
return f"{object.__repr__(self)}\n {self}"
def __str__(self):
return frombytes(self.type.get().ToString(), safe=True)
@@ -455,39 +446,24 @@ cdef class ColumnChunkMetaData(_Weakrefable):
def __repr__(self):
statistics = indent(repr(self.statistics), 4 * ' ')
geo_statistics = indent(repr(self.geo_statistics), 4 * ' ')
return """{0}
file_offset: {1}
file_path: {2}
physical_type: {3}
num_values: {4}
path_in_schema: {5}
is_stats_set: {6}
return f"""{object.__repr__(self)}
file_offset: {self.file_offset}
file_path: {self.file_path}
physical_type: {self.physical_type}
num_values: {self.num_values}
path_in_schema: {self.path_in_schema}
is_stats_set: {self.is_stats_set}
statistics:
{7}
{statistics}
geo_statistics:
{8}
compression: {9}
encodings: {10}
has_dictionary_page: {11}
dictionary_page_offset: {12}
data_page_offset: {13}
total_compressed_size: {14}
total_uncompressed_size: {15}""".format(object.__repr__(self),
self.file_offset,
self.file_path,
self.physical_type,
self.num_values,
self.path_in_schema,
self.is_stats_set,
statistics,
geo_statistics,
self.compression,
self.encodings,
self.has_dictionary_page,
self.dictionary_page_offset,
self.data_page_offset,
self.total_compressed_size,
self.total_uncompressed_size)
{geo_statistics}
compression: {self.compression}
encodings: {self.encodings}
has_dictionary_page: {self.has_dictionary_page}
dictionary_page_offset: {self.dictionary_page_offset}
data_page_offset: {self.data_page_offset}
total_compressed_size: {self.total_compressed_size}
total_uncompressed_size: {self.total_uncompressed_size}"""
def to_dict(self):
"""
@@ -791,16 +767,15 @@ cdef class SortingColumn:
elif descending == "ascending":
descending = False
else:
raise ValueError("Invalid sort key direction: {0}"
.format(descending))
raise ValueError(f"Invalid sort key direction: {descending}")
else:
raise ValueError("Invalid sort key: {0}".format(sort_key))
raise ValueError(f"Invalid sort key: {sort_key}")
try:
column_index = col_map[name]
except KeyError:
raise ValueError("Sort key name '{0}' not found in schema:\n{1}"
.format(name, schema))
raise ValueError(
f"Sort key name '{name}' not found in schema:\n{schema}")
sorting_columns.append(
cls(column_index, descending=descending, nulls_first=nulls_first)
@@ -851,9 +826,8 @@ cdef class SortingColumn:
return tuple(sort_keys), null_placement
def __repr__(self):
return """{}(column_index={}, descending={}, nulls_first={})""".format(
self.__class__.__name__,
self.column_index, self.descending, self.nulls_first)
return f"{self.__class__.__name__}(column_index={self.column_index}, " \
f"descending={self.descending}, nulls_first={self.nulls_first})"
def __eq__(self, SortingColumn other):
return (self.column_index == other.column_index and
@@ -943,21 +917,17 @@ cdef class RowGroupMetaData(_Weakrefable):
Metadata for column within this chunk.
"""
if i < 0 or i >= self.num_columns:
raise IndexError('{0} out of bounds'.format(i))
raise IndexError(f'{i} out of bounds')
cdef ColumnChunkMetaData chunk = ColumnChunkMetaData.__new__(ColumnChunkMetaData)
chunk.init(self, i)
return chunk
def __repr__(self):
return """{0}
num_columns: {1}
num_rows: {2}
total_byte_size: {3}
sorting_columns: {4}""".format(object.__repr__(self),
self.num_columns,
self.num_rows,
self.total_byte_size,
self.sorting_columns)
return f"""{object.__repr__(self)}
num_columns: {self.num_columns}
num_rows: {self.num_rows}
total_byte_size: {self.total_byte_size}
sorting_columns: {self.sorting_columns}"""
def to_dict(self):
"""
@@ -1047,17 +1017,13 @@ cdef class FileMetaData(_Weakrefable):
self.serialized_size))
def __repr__(self):
return """{0}
created_by: {1}
num_columns: {2}
num_rows: {3}
num_row_groups: {4}
format_version: {5}
serialized_size: {6}""".format(object.__repr__(self),
self.created_by, self.num_columns,
self.num_rows, self.num_row_groups,
self.format_version,
self.serialized_size)
return f"""{object.__repr__(self)}
created_by: {self.created_by}
num_columns: {self.num_columns}
num_rows: {self.num_rows}
num_row_groups: {self.num_row_groups}
format_version: {self.format_version}
serialized_size: {self.serialized_size}"""
def to_dict(self):
"""
@@ -1145,8 +1111,7 @@ cdef class FileMetaData(_Weakrefable):
elif version == ParquetVersion_V2_6:
return '2.6'
else:
warnings.warn('Unrecognized file version, assuming 2.6: {}'
.format(version))
warnings.warn(f'Unrecognized file version, assuming 2.6: {version}')
return '2.6'
@property
@@ -1255,9 +1220,7 @@ cdef class ParquetSchema(_Weakrefable):
self.schema = container._metadata.schema()
def __repr__(self):
return "{0}\n{1}".format(
object.__repr__(self),
frombytes(self.schema.ToString(), safe=True))
return f"{object.__repr__(self)}\n{frombytes(self.schema.ToString(), safe=True)}"
def __reduce__(self):
return ParquetSchema, (self.parent,)
@@ -1329,7 +1292,7 @@ cdef class ParquetSchema(_Weakrefable):
column_schema : ColumnSchema
"""
if i < 0 or i >= len(self):
raise IndexError('{0} out of bounds'.format(i))
raise IndexError(f'{i} out of bounds')
return ColumnSchema(self, i)
@@ -1374,25 +1337,18 @@ cdef class ColumnSchema(_Weakrefable):
physical_type = self.physical_type
converted_type = self.converted_type
if converted_type == 'DECIMAL':
converted_type = 'DECIMAL({0}, {1})'.format(self.precision,
self.scale)
converted_type = f'DECIMAL({self.precision}, {self.scale})'
elif physical_type == 'FIXED_LEN_BYTE_ARRAY':
converted_type = ('FIXED_LEN_BYTE_ARRAY(length={0})'
.format(self.length))
converted_type = f'FIXED_LEN_BYTE_ARRAY(length={self.length})'
return """<ParquetColumnSchema>
name: {0}
path: {1}
max_definition_level: {2}
max_repetition_level: {3}
physical_type: {4}
logical_type: {5}
converted_type (legacy): {6}""".format(self.name, self.path,
self.max_definition_level,
self.max_repetition_level,
physical_type,
str(self.logical_type),
converted_type)
return f"""<ParquetColumnSchema>
name: {self.name}
path: {self.path}
max_definition_level: {self.max_definition_level}
max_repetition_level: {self.max_repetition_level}
physical_type: {physical_type}
logical_type: {self.logical_type}
converted_type (legacy): {converted_type}"""
@property
def name(self):
@@ -2031,8 +1987,8 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
elif data_page_version == "2.0":
props.data_page_version(ParquetDataPageVersion_V2)
else:
raise ValueError("Unsupported Parquet data page version: {0}"
.format(data_page_version))
raise ValueError(
f"Unsupported Parquet data page version: {data_page_version}")
# version
@@ -2044,8 +2000,7 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
elif version == "2.6":
props.version(ParquetVersion_V2_6)
else:
raise ValueError("Unsupported Parquet format version: {0}"
.format(version))
raise ValueError(f"Unsupported Parquet format version: {version}")
# compression
@@ -2220,8 +2175,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
elif coerce_timestamps == 'us':
arrow_props.coerce_timestamps(TimeUnit_MICRO)
elif coerce_timestamps is not None:
raise ValueError('Invalid value for coerce_timestamps: {0}'
.format(coerce_timestamps))
raise ValueError(f'Invalid value for coerce_timestamps: {coerce_timestamps}')
# allow_truncated_timestamps
@@ -2243,8 +2197,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
warnings.warn("V1 parquet writer engine is a no-op. Use V2.")
arrow_props.set_engine_version(ArrowWriterEngineVersion.V1)
elif writer_engine_version != "V2":
raise ValueError("Unsupported Writer Engine Version: {0}"
.format(writer_engine_version))
raise ValueError(f"Unsupported Writer Engine Version: {writer_engine_version}")
arrow_properties = arrow_props.build()
+3 -4
View File
@@ -44,7 +44,7 @@ cdef cipher_to_name(ParquetCipher cipher):
elif ParquetCipher_AES_GCM_CTR_V1 == cipher:
return 'AES_GCM_CTR_V1'
else:
raise ValueError('Invalid cipher value: {0}'.format(cipher))
raise ValueError(f'Invalid cipher value: {cipher}')
cdef class EncryptionConfiguration(_Weakrefable):
"""Configuration of the encryption, such as which columns to encrypt"""
@@ -100,7 +100,7 @@ cdef class EncryptionConfiguration(_Weakrefable):
# to the string defined by the spec
# 'key1: col1 , col2; key2: col3 , col4'
column_keys = "; ".join(
["{}: {}".format(k, ", ".join(v)) for k, v in value.items()])
[f"{k}: {', '.join(v)}" for k, v in value.items()])
self.configuration.get().column_keys = tobytes(column_keys)
@property
@@ -340,8 +340,7 @@ cdef void _cb_create_kms_client(
result = handler(connection_config)
if not isinstance(result, KmsClient):
raise TypeError(
"callable must return KmsClient instances, but got {}".format(
type(result)))
f"callable must return KmsClient instances, but got {type(result)}")
out[0] = (<KmsClient> result).unwrap()
+2 -2
View File
@@ -307,8 +307,8 @@ def _perform_join_asof(left_operand, left_on, left_by,
columns_collisions = set(left_operand.schema.names) & set(right_columns)
if columns_collisions:
raise ValueError(
"Columns {} present in both tables. AsofJoin does not support "
"column collisions.".format(columns_collisions),
f"Columns {columns_collisions} present in both tables. "
"AsofJoin does not support column collisions."
)
# Add the join node to the execplan
+22 -23
View File
@@ -340,11 +340,10 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
if value_type is not None:
warnings.warn(
"The dtype of the 'categories' of the passed "
"categorical values ({0}) does not match the "
"specified type ({1}). For now ignoring the specified "
f"categorical values ({values.categories.dtype}) does not match the "
f"specified type ({value_type}). For now ignoring the specified "
"type, but in the future this mismatch will raise a "
"TypeError".format(
values.categories.dtype, value_type),
"TypeError",
FutureWarning, stacklevel=2)
dictionary = array(
values.categories.values, memory_pool=memory_pool)
@@ -1028,9 +1027,9 @@ cdef class Array(_PandasConvertible):
"""
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use one of "
"the `pyarrow.Array.from_*` functions instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor "
"directly, use one of the `pyarrow.Array.from_*` "
"functions instead.")
cdef void init(self, const shared_ptr[CArray]& sp_array) except *:
self.sp_array = sp_array
@@ -1272,18 +1271,18 @@ cdef class Array(_PandasConvertible):
if type.num_fields != len(children):
raise ValueError("Type's expected number of children "
"({0}) did not match the passed number "
"({1}).".format(type.num_fields, len(children)))
f"({type.num_fields}) did not match the passed number "
f"({len(children)})")
if type.has_variadic_buffers:
if type.num_buffers > len(buffers):
raise ValueError("Type's expected number of buffers is at least "
"{0}, but the passed number is "
"{1}.".format(type.num_buffers, len(buffers)))
f"{type.num_buffers}, but the passed number is "
f"{len(buffers)}.")
elif type.num_buffers != len(buffers):
raise ValueError("Type's expected number of buffers "
"({0}) did not match the passed number "
"({1}).".format(type.num_buffers, len(buffers)))
f"({type.num_buffers}) did not match the passed number "
f"({len(buffers)}).")
for buf in buffers:
# None will produce a null buffer pointer
@@ -1355,7 +1354,7 @@ cdef class Array(_PandasConvertible):
def __repr__(self):
type_format = object.__repr__(self)
return '{0}\n{1}'.format(type_format, str(self))
return f'{type_format}\n{self}'
def to_string(self, *, int indent=2, int top_level_indent=0, int window=10,
int container_window=2, c_bool skip_new_lines=False):
@@ -3713,7 +3712,7 @@ cdef class UnionArray(Array):
result = (<CUnionArray*> self.ap).field(pos)
if result != NULL:
return pyarrow_wrap_array(result)
raise KeyError("UnionArray does not have child {}".format(pos))
raise KeyError(f"UnionArray does not have child {pos}")
@property
def type_codes(self):
@@ -4382,8 +4381,8 @@ cdef class RunEndEncodedArray(Array):
if type.num_fields != len(children):
raise ValueError("RunEndEncodedType's expected number of children "
"({0}) did not match the passed number "
"({1}).".format(type.num_fields, len(children)))
f"({type.num_fields}) did not match the passed number "
f"({len(children)})")
# buffers are validated as if we needed to pass them to C++, but
# _make_from_arrays will take care of filling in the expected
@@ -4395,13 +4394,13 @@ cdef class RunEndEncodedArray(Array):
"bitmap, buffers[0] is not None")
if type.num_buffers != len(buffers):
raise ValueError("RunEndEncodedType's expected number of buffers "
"({0}) did not match the passed number "
"({1}).".format(type.num_buffers, len(buffers)))
f"({type.num_buffers}) did not match the passed number "
f"({len(buffers)}).")
# null_count is also validated as if we needed it
if null_count != -1 and null_count != 0:
raise ValueError("RunEndEncodedType's expected null_count (0) "
"did not match passed number ({0})".format(null_count))
f"did not match passed number ({null_count})")
return RunEndEncodedArray._from_arrays(type, False, length, children[0],
children[1], offset)
@@ -4484,8 +4483,8 @@ cdef class ExtensionArray(Array):
shared_ptr[CExtensionArray] ext_array
if storage.type != typ.storage_type:
raise TypeError("Incompatible storage type {0} "
"for extension type {1}".format(storage.type, typ))
raise TypeError(f"Incompatible storage type {storage.type} "
f"for extension type {typ}")
ext_array = make_shared[CExtensionArray](typ.sp_type, storage.sp_array)
cdef Array result = pyarrow_wrap_array(<shared_ptr[CArray]> ext_array)
@@ -4963,7 +4962,7 @@ def concat_arrays(arrays, MemoryPool memory_pool=None):
for array in arrays:
if not isinstance(array, Array):
raise TypeError("Iterable should contain Array objects, "
"got {0} instead".format(type(array)))
f"got {type(array)} instead")
c_arrays.push_back(pyarrow_unwrap_array(array))
with nogil:
+14 -15
View File
@@ -142,8 +142,7 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
summary = cpp_doc.summary
if not summary:
arg_str = "arguments" if func.arity > 1 else "argument"
summary = ("Call compute function {!r} with the given {}"
.format(func.name, arg_str))
summary = f"Call compute function {func.name!r} with the given {arg_str}"
doc_pieces.append(f"{summary}.\n\n")
@@ -183,11 +182,11 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
f"does not have a docstring", RuntimeWarning)
options_sig = inspect.signature(options_class)
for p in options_sig.parameters.values():
doc_pieces.append(dedent("""\
{0} : optional
Parameter for {1} constructor. Either `options`
or `{0}` can be passed, but not both at the same time.
""".format(p.name, options_class.__name__)))
doc_pieces.append(dedent(f"""\
{p.name} : optional
Parameter for {options_class.__name__} constructor. Either `options`
or `{p.name}` can be passed, but not both at the same time.
"""))
doc_pieces.append(dedent(f"""\
options : pyarrow.compute.{options_class.__name__}, optional
Alternative way of passing options.
@@ -200,7 +199,8 @@ def _decorate_compute_function(wrapper, exposed_name, func, options_class):
# 4. Custom addition (e.g. examples)
if doc_addition is not None:
doc_pieces.append("\n{}\n".format(dedent(doc_addition).strip("\n")))
stripped = dedent(doc_addition).strip('\n')
doc_pieces.append(f"\n{stripped}\n")
wrapper.__doc__ = "".join(doc_pieces)
return wrapper
@@ -213,8 +213,8 @@ def _get_options_class(func):
try:
return globals()[class_name]
except KeyError:
warnings.warn("Python binding for {} not exposed"
.format(class_name), RuntimeWarning)
warnings.warn(f"Python binding for {class_name} not exposed",
RuntimeWarning)
return None
@@ -222,9 +222,8 @@ def _handle_options(name, options_class, options, args, kwargs):
if args or kwargs:
if options is not None:
raise TypeError(
"Function {!r} called with both an 'options' argument "
"and additional arguments"
.format(name))
f"Function {name!r} called with both an 'options' argument "
f"and additional arguments")
return options_class(*args, **kwargs)
if options is not None:
@@ -233,8 +232,8 @@ def _handle_options(name, options_class, options, args, kwargs):
elif isinstance(options, options_class):
return options
raise TypeError(
"Function {!r} expected a {} parameter, got {}"
.format(name, options_class, type(options)))
f"Function {name!r} expected a {options_class} parameter, "
f"got {type(options)}")
return None
+1 -1
View File
@@ -233,7 +233,7 @@ def pytest_ignore_collect(collection_path, config):
# handle cuda, flight, etc
for group in doctest_groups:
if 'pyarrow/{}'.format(group) in str(collection_path):
if f'pyarrow/{group}' in str(collection_path):
if not defaults[group]:
return True
+16 -21
View File
@@ -114,7 +114,7 @@ def __getattr__(name):
raise ImportError(_parquet_msg)
raise AttributeError(
"module 'pyarrow.dataset' has no attribute '{0}'".format(name)
f"module 'pyarrow.dataset' has no attribute '{name}'"
)
@@ -234,8 +234,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
return DirectoryPartitioning.discover(field_names)
else:
raise ValueError(
"Expected list of field names, got {}".format(
type(field_names)))
f"Expected list of field names, got {type(field_names)}")
else:
raise ValueError(
"For the default directory flavor, need to specify "
@@ -253,8 +252,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
return FilenamePartitioning.discover(field_names)
else:
raise ValueError(
"Expected list of field names, got {}".format(
type(field_names)))
f"Expected list of field names, got {type(field_names)}")
else:
raise ValueError(
"For the filename flavor, need to specify "
@@ -269,8 +267,7 @@ def partitioning(schema=None, field_names=None, flavor=None,
return HivePartitioning(schema, dictionaries)
else:
raise ValueError(
"Expected Schema for 'schema', got {}".format(
type(schema)))
f"Expected Schema for 'schema', got {type(schema)}")
else:
return HivePartitioning.discover()
else:
@@ -292,8 +289,8 @@ def _ensure_partitioning(scheme):
elif isinstance(scheme, (Partitioning, PartitioningFactory)):
pass
else:
raise ValueError("Expected Partitioning or PartitioningFactory, got {}"
.format(type(scheme)))
raise ValueError(
f"Expected Partitioning or PartitioningFactory, got {type(scheme)}")
return scheme
@@ -317,7 +314,7 @@ def _ensure_format(obj):
elif obj == "json":
return JsonFileFormat()
else:
raise ValueError("format '{}' is not supported".format(obj))
raise ValueError(f"format '{obj}' is not supported")
def _ensure_multiple_sources(paths, filesystem=None):
@@ -382,16 +379,15 @@ def _ensure_multiple_sources(paths, filesystem=None):
raise FileNotFoundError(info.path)
elif file_type == FileType.Directory:
raise IsADirectoryError(
'Path {} points to a directory, but only file paths are '
f'Path {info.path} points to a directory, but only file paths are '
'supported. To construct a nested or union dataset pass '
'a list of dataset objects instead.'.format(info.path)
'a list of dataset objects instead.'
)
else:
raise IOError(
'Path {} exists but its type is unknown (could be a '
f'Path {info.path} exists but its type is unknown (could be a '
'special file such as a Unix socket or character device, '
'or Windows NUL / CON / ...)'.format(info.path)
)
'or Windows NUL / CON / ...)')
return filesystem, paths
@@ -802,18 +798,18 @@ RecordBatch or Table, iterable of RecordBatch, RecordBatchReader, or URI
return _in_memory_dataset(source, **kwargs)
else:
unique_types = set(type(elem).__name__ for elem in source)
type_names = ', '.join('{}'.format(t) for t in unique_types)
type_names = ', '.join(f'{t}' for t in unique_types)
raise TypeError(
'Expected a list of path-like or dataset objects, or a list '
'of batches or tables. The given list contains the following '
'types: {}'.format(type_names)
f'types: {type_names}'
)
elif isinstance(source, (pa.RecordBatch, pa.Table)):
return _in_memory_dataset(source, **kwargs)
else:
raise TypeError(
'Expected a path-like, list of path-likes or a list of Datasets '
'instead of the given type: {}'.format(type(source).__name__)
f'instead of the given type: {type(source).__name__}'
)
@@ -987,9 +983,8 @@ Table/RecordBatch, or iterable of RecordBatch
file_options = format.make_write_options()
if format != file_options.format:
raise TypeError("Supplied FileWriteOptions have format {}, "
"which doesn't match supplied FileFormat {}".format(
format, file_options))
raise TypeError(f"Supplied FileWriteOptions have format {format}, "
f"which doesn't match supplied FileFormat {file_options}")
if basename_template is None:
basename_template = "part-{i}." + format.default_extname
+3 -4
View File
@@ -73,7 +73,7 @@ cdef class Device(_Weakrefable):
return self.device.get().Equals(deref((<Device>other).device.get()))
def __repr__(self):
return "<pyarrow.Device: {}>".format(frombytes(self.device.get().ToString()))
return f"<pyarrow.Device: {frombytes(self.device.get().ToString())}>"
@property
def type_name(self):
@@ -137,9 +137,8 @@ cdef class MemoryManager(_Weakrefable):
return self.memory_manager
def __repr__(self):
return "<pyarrow.MemoryManager device: {}>".format(
frombytes(self.memory_manager.get().device().get().ToString())
)
device_str = frombytes(self.memory_manager.get().device().get().ToString())
return f"<pyarrow.MemoryManager device: {device_str}>"
@property
def device(self):
+11 -14
View File
@@ -70,10 +70,8 @@ class FeatherDataset:
def validate_schemas(self, piece, table):
if not self.schema.equals(table.schema):
raise ValueError('Schema in {!s} was different. \n'
'{!s}\n\nvs\n\n{!s}'
.format(piece, self.schema,
table.schema))
raise ValueError(f'Schema in {piece} was different. \n'
f'{self.schema}\n\nvs\n\n{table.schema}')
def read_pandas(self, columns=None, use_threads=True):
"""
@@ -100,14 +98,15 @@ def check_chunked_overflow(name, col):
return
if col.type in (ext.binary(), ext.string()):
raise ValueError("Column '{}' exceeds 2GB maximum capacity of "
raise ValueError(f"Column '{name}' exceeds 2GB maximum capacity of "
"a Feather binary column. This restriction may be "
"lifted in the future".format(name))
"lifted in the future")
else:
# TODO(wesm): Not sure when else this might be reached
raise ValueError("Column '{}' of type {} was chunked on conversion "
"to Arrow and cannot be currently written to "
"Feather format".format(name, str(col.type)))
raise ValueError(
f"Column '{name}' of type {col.type} was chunked on conversion to Arrow "
"and cannot be currently written to Feather format"
)
_FEATHER_SUPPORTED_CODECS = {'lz4', 'zstd', 'uncompressed'}
@@ -180,9 +179,8 @@ def write_feather(df, dest, compression=None, compression_level=None,
compression = 'lz4'
elif (compression is not None and
compression not in _FEATHER_SUPPORTED_CODECS):
raise ValueError('compression="{}" not supported, must be '
'one of {}'.format(compression,
_FEATHER_SUPPORTED_CODECS))
raise ValueError(f'compression="{compression}" not supported, must be '
f'one of {_FEATHER_SUPPORTED_CODECS}')
try:
_feather.write_feather(table, dest, compression=compression,
@@ -268,8 +266,7 @@ def read_table(source, columns=None, memory_map=False, use_threads=True):
else:
column_type_names = [t.__name__ for t in column_types]
raise TypeError("Columns must be indices or names. "
"Got columns {} of types {}"
.format(columns, column_type_names))
f"Got columns {columns} of types {column_type_names}")
# Feather v1 already respects the column selection
if reader.version < 3:
+10 -12
View File
@@ -74,11 +74,11 @@ def __getattr__(name):
if name in _not_imported:
raise ImportError(
"The pyarrow installation is not built with support for "
"'{0}'".format(name)
f"'{name}'"
)
raise AttributeError(
"module 'pyarrow.fs' has no attribute '{0}'".format(name)
f"module 'pyarrow.fs' has no attribute '{name}'"
)
@@ -93,10 +93,9 @@ def _filesystem_from_str(uri):
if prefix_info.type != FileType.Directory:
raise ValueError(
"The path component of the filesystem URI must point to a "
"directory but it has a type: `{}`. The path component "
"is `{}` and the given filesystem URI is `{}`".format(
prefix_info.type.name, prefix_info.path, uri
)
f"directory but it has a type: `{prefix_info.type.name}`. The path "
f"component is `{prefix_info.path}` and the given filesystem URI is "
f"`{uri}`"
)
filesystem = SubTreeFileSystem(prefix, filesystem)
return filesystem
@@ -126,9 +125,8 @@ def _ensure_filesystem(filesystem, *, use_mmap=False):
return PyFileSystem(FSSpecHandler(filesystem))
raise TypeError(
"Unrecognized filesystem: {}. `filesystem` argument must be a "
"FileSystem instance or a valid file system URI'".format(
type(filesystem))
f"Unrecognized filesystem: {type(filesystem)}. `filesystem` argument must be a "
"FileSystem instance or a valid file system URI"
)
@@ -234,7 +232,7 @@ def copy_files(source, destination,
Copy one file from S3 bucket to a local directory:
>>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
... "file:///{}/index_copy.ndjson".format(local_path))
... f"file:///{local_path}/index_copy.ndjson")
>>> fs.LocalFileSystem().get_file_info(str(local_path)+
... '/index_copy.ndjson')
@@ -243,7 +241,7 @@ def copy_files(source, destination,
Copy file using a FileSystem object:
>>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
... "file:///{}/index_copy.ndjson".format(local_path),
... f"file:///{local_path}/index_copy.ndjson",
... source_filesystem=fs.S3FileSystem())
"""
source_fs, source_path = _resolve_filesystem_and_path(
@@ -297,7 +295,7 @@ class FSSpecHandler(FileSystemHandler):
protocol = self.fs.protocol
if isinstance(protocol, list):
protocol = protocol[0]
return "fsspec+{0}".format(protocol)
return f"fsspec+{protocol}"
def normalize_path(self, path):
return path
+15 -19
View File
@@ -80,9 +80,8 @@ cdef class Node(_Weakrefable):
shared_ptr[CNode] node
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use the "
"TreeExprBuilder API directly"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use the "
"TreeExprBuilder API directly")
@staticmethod
cdef create(shared_ptr[CNode] node):
@@ -95,7 +94,7 @@ cdef class Node(_Weakrefable):
def __repr__(self):
type_format = object.__repr__(self)
return '{0}\n{1}'.format(type_format, str(self))
return f"{type_format}\n{self}"
def return_type(self):
return pyarrow_wrap_data_type(self.node.get().return_type())
@@ -113,7 +112,7 @@ cdef class Expression(_Weakrefable):
def __repr__(self):
type_format = object.__repr__(self)
return '{0}\n{1}'.format(type_format, str(self))
return f"{type_format}\n{self}"
def root(self):
return Node.create(self.expression.get().root())
@@ -127,9 +126,8 @@ cdef class Condition(_Weakrefable):
shared_ptr[CCondition] condition
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use the "
"TreeExprBuilder API instead"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use the "
"TreeExprBuilder API instead")
@staticmethod
cdef create(shared_ptr[CCondition] condition):
@@ -142,7 +140,7 @@ cdef class Condition(_Weakrefable):
def __repr__(self):
type_format = object.__repr__(self)
return '{0}\n{1}'.format(type_format, str(self))
return f"{type_format}\n{self}"
def root(self):
return Node.create(self.condition.get().root())
@@ -156,8 +154,8 @@ cdef class SelectionVector(_Weakrefable):
shared_ptr[CSelectionVector] selection_vector
def __init__(self):
raise TypeError("Do not call {}'s constructor directly."
.format(self.__class__.__name__))
raise TypeError(
f"Do not call {self.__class__.__name__}'s constructor directly.")
@staticmethod
cdef create(shared_ptr[CSelectionVector] selection_vector):
@@ -176,9 +174,8 @@ cdef class Projector(_Weakrefable):
MemoryPool pool
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
"make_projector instead"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
"make_projector instead")
@staticmethod
cdef create(shared_ptr[CProjector] projector, MemoryPool pool):
@@ -226,9 +223,8 @@ cdef class Filter(_Weakrefable):
shared_ptr[CFilter] filter
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
"make_filter instead"
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
"make_filter instead")
@staticmethod
cdef create(shared_ptr[CFilter] filter):
@@ -712,8 +708,8 @@ cdef class FunctionSignature(_Weakrefable):
shared_ptr[CFunctionSignature] signature
def __init__(self):
raise TypeError("Do not call {}'s constructor directly."
.format(self.__class__.__name__))
raise TypeError(
f"Do not call {self.__class__.__name__}'s constructor directly.")
@staticmethod
cdef create(shared_ptr[CFunctionSignature] signature):
+1 -1
View File
@@ -81,7 +81,7 @@ cdef inline CSelectionVector_Mode _ensure_selection_mode(str name) except *:
elif uppercase == 'UINT64':
return CSelectionVector_Mode_UINT64
else:
raise ValueError('Invalid value for Selection Mode: {!r}'.format(name))
raise ValueError(f'Invalid value for Selection Mode: {name!r}')
cdef inline str _selection_mode_name(CSelectionVector_Mode ctype):
if ctype == CSelectionVector_Mode_NONE:
+1 -1
View File
@@ -308,7 +308,7 @@ class _PyArrowColumn:
kind = DtypeKind.DATETIME
ts = dtype.unit[0]
tz = dtype.tz if dtype.tz else ""
f_string = "ts{ts}:{tz}".format(ts=ts, tz=tz)
f_string = f"ts{ts}:{tz}"
return kind, bit_width, f_string, Endianness.NATIVE
elif pa.types.is_dictionary(dtype):
kind = DtypeKind.CATEGORICAL
+13 -18
View File
@@ -335,8 +335,7 @@ cdef class NativeFile(_Weakrefable):
offset = offset + position
else:
with gil:
raise ValueError("Invalid value of whence: {0}"
.format(whence))
raise ValueError(f"Invalid value of whence: {whence}")
check_status(handle.get().Seek(offset))
return self.tell()
@@ -702,8 +701,7 @@ cdef class NativeFile(_Weakrefable):
# the passed buffer, so it's hard for us to avoid doubling the memory
buf = <uint8_t*> malloc(buffer_size)
if buf == NULL:
raise MemoryError("Failed to allocate {0} bytes"
.format(buffer_size))
raise MemoryError(f"Failed to allocate {buffer_size} bytes")
writer_thread.start()
@@ -770,8 +768,7 @@ cdef class NativeFile(_Weakrefable):
# the passed buffer, so it's hard for us to avoid doubling the memory
buf = <uint8_t*> malloc(buffer_size)
if buf == NULL:
raise MemoryError("Failed to allocate {0} bytes"
.format(buffer_size))
raise MemoryError(f"Failed to allocate {buffer_size} bytes")
cdef int64_t total_bytes = 0
cdef int32_t c_buffer_size = buffer_size
@@ -951,7 +948,7 @@ cdef class PythonFile(NativeFile):
elif inferred_mode.startswith('r'):
kind = 'r'
else:
raise ValueError('Invalid file mode: {0}'.format(mode))
raise ValueError(f'Invalid file mode: {mode}')
# If mode was given, check it matches the given file
if mode is not None:
@@ -1087,7 +1084,7 @@ cdef class MemoryMappedFile(NativeFile):
self.is_readable = True
self.is_writable = True
else:
raise ValueError('Invalid file mode: {0}'.format(mode))
raise ValueError(f'Invalid file mode: {mode}')
with nogil:
handle = GetResultValue(CMemoryMappedFile.Open(c_path, c_mode))
@@ -1149,8 +1146,7 @@ def memory_map(path, mode='r'):
cdef _check_is_file(path):
if os.path.isdir(path):
raise IOError("Expected file path, but {0} is a directory"
.format(path))
raise IOError(f"Expected file path, but {path} is a directory")
def create_memory_map(path, size):
@@ -1247,7 +1243,7 @@ cdef class OSFile(NativeFile):
elif mode in ('a', 'ab'):
self._open_writable(c_path, append=True)
else:
raise ValueError('Invalid file mode: {0}'.format(mode))
raise ValueError(f'Invalid file mode: {mode}')
cdef _open_readable(self, c_string path, CMemoryPool* pool):
cdef shared_ptr[ReadableFile] handle
@@ -2212,8 +2208,7 @@ cdef get_writer(object source, shared_ptr[COutputStream]* writer):
nf = source
writer[0] = nf.get_output_stream()
else:
raise TypeError('Unable to write to object of type: {0}'
.format(type(source)))
raise TypeError(f'Unable to write to object of type: {type(source)}')
# ---------------------------------------------------------------------
@@ -2248,7 +2243,7 @@ cdef CCompressionType _ensure_compression(str name) except *:
elif uppercase == 'ZSTD':
return CCompressionType_ZSTD
else:
raise ValueError('Invalid value for compression: {!r}'.format(name))
raise ValueError(f'Invalid value for compression: {name!r}')
cdef class CacheOptions(_Weakrefable):
@@ -2810,8 +2805,8 @@ def input_stream(source, compression='detect', buffer_size=None):
hasattr(source, 'closed')):
stream = PythonFile(source, 'r')
else:
raise TypeError("pa.input_stream() called with instance of '{}'"
.format(source.__class__))
raise TypeError(
f"pa.input_stream() called with instance of '{source.__class__}'")
if compression == 'detect':
# detect for OSFile too
@@ -2902,8 +2897,8 @@ def output_stream(source, compression='detect', buffer_size=None):
hasattr(source, 'closed')):
stream = PythonFile(source, 'w')
else:
raise TypeError("pa.output_stream() called with instance of '{}'"
.format(source.__class__))
raise TypeError(
f"pa.output_stream() called with instance of '{source.__class__}'")
if compression == 'detect':
compression = _detect_compression(source_path)
+13 -15
View File
@@ -301,9 +301,8 @@ cdef class Message(_Weakrefable):
pass
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
"`pyarrow.ipc.read_message` function instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
"`pyarrow.ipc.read_message` function instead.")
@property
def type(self):
@@ -393,9 +392,9 @@ cdef class Message(_Weakrefable):
body_len = 0 if body is None else body.size
return """pyarrow.Message
type: {0}
metadata length: {1}
body length: {2}""".format(self.type, metadata_len, body_len)
type: {self.type}
metadata length: {metadata_len}
body length: {body_len}"""
cdef class MessageReader(_Weakrefable):
@@ -410,9 +409,9 @@ cdef class MessageReader(_Weakrefable):
pass
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use "
"`pyarrow.ipc.MessageReader.open_stream` function "
"instead.".format(self.__class__.__name__))
"instead.")
@staticmethod
def open_stream(source):
@@ -660,9 +659,8 @@ cdef class RecordBatchReader(_Weakrefable):
# cdef block is in lib.pxd
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
"use one of the RecordBatchReader.from_* functions instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use one of the RecordBatchReader.from_* functions instead.")
def __iter__(self):
return self
@@ -1122,7 +1120,7 @@ cdef class _RecordBatchFileReader(_Weakrefable):
cdef shared_ptr[CRecordBatch] batch
if i < 0 or i >= self.num_record_batches:
raise ValueError('Batch number {0} out of range'.format(i))
raise ValueError(f'Batch number {i} out of range')
with nogil:
batch = GetResultValue(self.reader.get().ReadRecordBatch(i))
@@ -1152,7 +1150,7 @@ cdef class _RecordBatchFileReader(_Weakrefable):
CRecordBatchWithMetadata batch_with_metadata
if i < 0 or i >= self.num_record_batches:
raise ValueError('Batch number {0} out of range'.format(i))
raise ValueError(f'Batch number {i} out of range')
with nogil:
batch_with_metadata = GetResultValue(
@@ -1274,8 +1272,8 @@ cdef NativeFile as_native_file(source):
source = BufferReader(source)
if not isinstance(source, NativeFile):
raise ValueError('Unable to read message from object with type: {0}'
.format(type(source)))
raise ValueError(
f'Unable to read message from object with type: {type(source)}')
return source
+12 -16
View File
@@ -81,9 +81,9 @@ metadata : dict | pyarrow.KeyValueMetadata, optional
class RecordBatchStreamWriter(lib._RecordBatchStreamWriter):
__doc__ = """Writer for the Arrow streaming binary format
__doc__ = f"""Writer for the Arrow streaming binary format
{}""".format(_ipc_writer_class_doc)
{_ipc_writer_class_doc}"""
def __init__(self, sink, schema, *, options=None):
options = _get_legacy_format_default(options)
@@ -118,10 +118,9 @@ class RecordBatchFileReader(lib._RecordBatchFileReader):
class RecordBatchFileWriter(lib._RecordBatchFileWriter):
__doc__ = """Writer to create the Arrow binary file format
__doc__ = f"""Writer to create the Arrow binary file format
{}
""".format(_ipc_file_writer_class_doc)
{_ipc_file_writer_class_doc}"""
def __init__(self, sink, schema, *, options=None, metadata=None):
options = _get_legacy_format_default(options)
@@ -131,8 +130,7 @@ class RecordBatchFileWriter(lib._RecordBatchFileWriter):
def _get_legacy_format_default(options):
if options:
if not isinstance(options, IpcWriteOptions):
raise TypeError("expected IpcWriteOptions, got {}"
.format(type(options)))
raise TypeError(f"expected IpcWriteOptions, got {type(options)}")
return options
metadata_version = MetadataVersion.V5
@@ -146,9 +144,7 @@ def _get_legacy_format_default(options):
def _ensure_default_ipc_read_options(options):
if options and not isinstance(options, IpcReadOptions):
raise TypeError(
"expected IpcReadOptions, got {}".format(type(options))
)
raise TypeError(f"expected IpcReadOptions, got {type(options)}")
return options or IpcReadOptions()
@@ -157,16 +153,16 @@ def new_stream(sink, schema, *, options=None):
options=options)
new_stream.__doc__ = """\
new_stream.__doc__ = f"""\
Create an Arrow columnar IPC stream writer instance
{}
{_ipc_writer_class_doc}
Returns
-------
writer : RecordBatchStreamWriter
A writer for the given sink
""".format(_ipc_writer_class_doc)
"""
def open_stream(source, *, options=None, memory_pool=None):
@@ -196,16 +192,16 @@ def new_file(sink, schema, *, options=None, metadata=None):
return RecordBatchFileWriter(sink, schema, options=options, metadata=metadata)
new_file.__doc__ = """\
new_file.__doc__ = f"""\
Create an Arrow columnar IPC file writer instance
{}
{_ipc_file_writer_class_doc}
Returns
-------
writer : RecordBatchFileWriter
A writer for the given sink
""".format(_ipc_file_writer_class_doc)
"""
def open_file(source, footer_offset=None, *, options=None, memory_pool=None):
+3 -3
View File
@@ -239,7 +239,7 @@ def field(jvm_field):
typ = pa.decimal128(jvm_type.getPrecision(), jvm_type.getScale())
else:
raise NotImplementedError(
"Unsupported JVM type: {}".format(type_str))
f"Unsupported JVM type: {type_str}")
else:
# TODO: The following JVM types are not implemented:
# Struct, List, FixedSizeList, Union, Dictionary
@@ -295,8 +295,8 @@ def array(jvm_array):
if jvm_array.getField().getType().isComplex():
minor_type_str = jvm_array.getMinorType().toString()
raise NotImplementedError(
"Cannot convert JVM Arrow array of type {},"
" complex types not yet implemented.".format(minor_type_str))
f"Cannot convert JVM Arrow array of type {minor_type_str}, "
"complex types not yet implemented.")
dtype = field(jvm_array.getField()).type
buffers = [jvm_buffer(buf)
for buf in list(jvm_array.getBuffers(False))]
+6 -9
View File
@@ -29,9 +29,8 @@ cdef class MemoryPool(_Weakrefable):
"""
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
"use pyarrow.*_memory_pool instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use pyarrow.*_memory_pool instead.")
cdef void init(self, CMemoryPool* pool):
self.pool = pool
@@ -126,9 +125,8 @@ cdef class LoggingMemoryPool(MemoryPool):
unique_ptr[CLoggingMemoryPool] logging_pool
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
"use pyarrow.logging_memory_pool instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use pyarrow.logging_memory_pool instead.")
cdef class ProxyMemoryPool(MemoryPool):
@@ -141,9 +139,8 @@ cdef class ProxyMemoryPool(MemoryPool):
unique_ptr[CProxyMemoryPool] proxy_pool
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
"use pyarrow.proxy_memory_pool instead."
.format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use pyarrow.proxy_memory_pool instead.")
def default_memory_pool():
+6 -6
View File
@@ -222,7 +222,7 @@ bloom_filter_fpp : double, default 0.05
class ORCWriter:
__doc__ = """
__doc__ = f"""
Writer interface for a single ORC file
Parameters
@@ -231,8 +231,8 @@ where : str or pyarrow.io.NativeFile
Writable target. For passing Python file objects or byte buffers,
see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream
or pyarrow.io.FixedSizeBufferWriter.
{}
""".format(_orc_writer_args_docs)
{_orc_writer_args_docs}
"""
is_open = False
@@ -369,7 +369,7 @@ def write_table(table, where, *,
writer.write(table)
write_table.__doc__ = """
write_table.__doc__ = f"""
Write a table into an ORC file.
Parameters
@@ -380,5 +380,5 @@ where : str or pyarrow.io.NativeFile
Writable target. For passing Python file objects or byte buffers,
see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream
or pyarrow.io.FixedSizeBufferWriter.
{}
""".format(_orc_writer_args_docs)
{_orc_writer_args_docs}
"""
+6 -4
View File
@@ -67,14 +67,16 @@ cdef class _PandasAPIShim(object):
self._have_pandas = False
if raise_:
raise ImportError(
"pyarrow requires pandas 1.0.0 or above, pandas {} is "
"installed".format(self._version)
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
"installed"
)
else:
warnings.warn(
"pyarrow requires pandas 1.0.0 or above, pandas {} is "
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
"installed. Therefore, pandas-specific integration is not "
"used.".format(self._version), stacklevel=2)
"used.",
stacklevel=2
)
return
self._is_v1 = self._loose_version < Version('2.0.0')
+18 -21
View File
@@ -81,7 +81,7 @@ def get_logical_type(arrow_type):
if isinstance(arrow_type, pa.lib.DictionaryType):
return 'categorical'
elif isinstance(arrow_type, pa.lib.ListType):
return 'list[{}]'.format(get_logical_type(arrow_type.value_type))
return f'list[{get_logical_type(arrow_type.value_type)}]'
elif isinstance(arrow_type, pa.lib.TimestampType):
return 'datetimetz' if arrow_type.tz is not None else 'datetime'
elif pa.types.is_decimal(arrow_type):
@@ -180,9 +180,8 @@ def get_column_metadata(column, name, arrow_type, field_name):
and not isinstance(name, str)
):
raise TypeError(
'Column name must be a string. Got column {} of type {}'.format(
name, type(name).__name__
)
f"Column name must be a string. Got column {name} of type "
f"{type(name).__name__}"
)
assert isinstance(field_name, str), str(type(field_name))
@@ -365,7 +364,7 @@ def _index_level_name(index, i, column_names):
if index.name is not None and index.name not in column_names:
return _column_name_to_strings(index.name)
else:
return '__index_level_{:d}__'.format(i)
return f'__index_level_{i:d}__'
def _get_columns_to_convert(df, schema, preserve_index, columns):
@@ -373,7 +372,7 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
if not df.columns.is_unique:
raise ValueError(
'Duplicate column names found: {}'.format(list(df.columns))
f'Duplicate column names found: {list(df.columns)}'
)
if schema is not None:
@@ -396,7 +395,7 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
if _pandas_api.is_sparse(col):
raise TypeError(
"Sparse pandas data (column {}) not supported.".format(name))
f"Sparse pandas data (column {name}) not supported.")
columns_to_convert.append(col)
convert_fields.append(None)
@@ -458,27 +457,27 @@ def _get_columns_to_convert_given_schema(df, schema, preserve_index):
except (KeyError, IndexError):
# name not found as index level
raise KeyError(
"name '{}' present in the specified schema is not found "
"in the columns or index".format(name))
f"name '{name}' present in the specified schema is not found "
"in the columns or index")
if preserve_index is False:
raise ValueError(
"name '{}' present in the specified schema corresponds "
f"name '{name}' present in the specified schema corresponds "
"to the index, but 'preserve_index=False' was "
"specified".format(name))
"specified")
elif (preserve_index is None and
isinstance(col, _pandas_api.pd.RangeIndex)):
raise ValueError(
"name '{}' is present in the schema, but it is a "
f"name '{name}' is present in the schema, but it is a "
"RangeIndex which will not be converted as a column "
"in the Table, but saved as metadata-only not in "
"columns. Specify 'preserve_index=True' to force it "
"being added as a column, or remove it from the "
"specified schema".format(name))
"specified schema")
is_index = True
if _pandas_api.is_sparse(col):
raise TypeError(
"Sparse pandas data (column {}) not supported.".format(name))
f"Sparse pandas data (column {name}) not supported.")
field = schema.field(name)
columns_to_convert.append(col)
@@ -621,13 +620,12 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None,
except (pa.ArrowInvalid,
pa.ArrowNotImplementedError,
pa.ArrowTypeError) as e:
e.args += ("Conversion failed for column {!s} with type {!s}"
.format(col.name, col.dtype),)
e.args += (
f"Conversion failed for column {col.name} with type {col.dtype}",)
raise e
if not field_nullable and result.null_count > 0:
raise ValueError("Field {} was non-nullable but pandas column "
"had {} null values".format(str(field),
result.null_count))
raise ValueError(f"Field {field} was non-nullable but pandas column "
f"had {result.null_count} null values")
return result
def _can_definitely_zero_copy(arr):
@@ -1004,8 +1002,7 @@ def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None)
# Possibly the result of munged metadata
continue
else:
raise ValueError("Unrecognized index kind: {}"
.format(descr['kind']))
raise ValueError(f"Unrecognized index kind: {descr['kind']}")
index_arrays.append(index_level)
index_names.append(index_name)
+18 -19
View File
@@ -181,9 +181,7 @@ def filters_to_expression(filters):
elif op == 'not in':
return ~field.isin(val)
else:
raise ValueError(
'"{0}" is not a valid operator in predicates.'.format(
(col, op, val)))
raise ValueError(f'"{col}" is not a valid operator in predicates.')
disjunction_members = []
@@ -949,14 +947,14 @@ and write the RecordBatch into the Parquet file:
class ParquetWriter:
__doc__ = """
__doc__ = f"""
Class for incrementally building a Parquet file for Arrow tables.
Parameters
----------
where : path or file-like object
schema : pyarrow.Schema
{}
{_parquet_writer_arg_docs}
writer_engine_version : unused
**options : dict
If options contains a key `metadata_collector` then the
@@ -966,8 +964,8 @@ writer_engine_version : unused
Examples
--------
{}
""".format(_parquet_writer_arg_docs, _parquet_writer_example_doc)
{_parquet_writer_example_doc}
"""
def __init__(self, where, schema, filesystem=None,
flavor=None,
@@ -1112,9 +1110,10 @@ Examples
assert self.is_open
if not table.schema.equals(self.schema, check_metadata=False):
msg = ('Table schema does not match schema used to create file: '
'\ntable:\n{!s} vs. \nfile:\n{!s}'
.format(table.schema, self.schema))
msg = (
"Table schema does not match schema used to create file: \n"
f"table:\n{table.schema!s} vs. \nfile:\n{self.schema!s}"
)
raise ValueError(msg)
self.writer.write_table(table, row_group_size=row_group_size)
@@ -1215,7 +1214,7 @@ create a ParquetDataset object with filter:
class ParquetDataset:
__doc__ = """
__doc__ = f"""
Encapsulates details of reading a complete Parquet dataset possibly
consisting of multiple files and partitions in subdirectories.
@@ -1236,8 +1235,8 @@ filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], defaul
exploited to avoid loading files at all if they contain no matching rows.
Within-file level filtering and different partitioning schemes are supported.
{1}
{0}
{_DNF_filter_doc}
{_read_docstring_common}
ignore_prefixes : list, optional
Files matching any of these prefixes will be ignored by the
discovery process.
@@ -1276,8 +1275,8 @@ arrow_extensions_enabled : bool, default False
Examples
--------
{2}
""".format(_read_docstring_common, _DNF_filter_doc, _parquet_dataset_example)
{_parquet_dataset_example}
"""
def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None,
read_dictionary=None, memory_map=False, buffer_size=None,
@@ -1966,7 +1965,7 @@ Defining column encoding per-column:
... use_dictionary=False)
"""
write_table.__doc__ = """
write_table.__doc__ = f"""
Write a Table to Parquet format.
Parameters
@@ -1977,14 +1976,14 @@ row_group_size : int
Maximum number of rows in each written row group. If None, the
row group size will be the minimum of the Table size and
1024 * 1024.
{}
{_parquet_writer_arg_docs}
**kwargs : optional
Additional options for ParquetWriter
Examples
--------
{}
""".format(_parquet_writer_arg_docs, _write_table_example)
{_write_table_example}
"""
def write_to_dataset(table, root_path, partition_cols=None,
+11 -18
View File
@@ -26,8 +26,8 @@ cdef class Scalar(_Weakrefable):
"""
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use "
"pa.scalar() instead.".format(self.__class__.__name__))
raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, "
"use pa.scalar() instead.")
cdef void init(self, const shared_ptr[CScalar]& wrapped):
self.wrapped = wrapped
@@ -117,9 +117,7 @@ cdef class Scalar(_Weakrefable):
check_status(self.wrapped.get().Validate())
def __repr__(self):
return '<pyarrow.{}: {!r}>'.format(
self.__class__.__name__, self.as_py()
)
return f'<pyarrow.{self.__class__.__name__}: {self.as_py()!r}>'
def __str__(self):
return str(self.as_py())
@@ -611,10 +609,10 @@ def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
# otherwise safely truncate to microsecond resolution datetime
if value % 1000 != 0:
raise ValueError(
"Nanosecond resolution temporal type {} is not safely "
f"Nanosecond resolution temporal type {value} is not safely "
"convertible to microseconds to convert to datetime.datetime. "
"Install pandas to return as Timestamp with nanosecond "
"support or access the .value attribute.".format(value)
"support or access the .value attribute."
)
delta = datetime.timedelta(microseconds=value // 1000)
@@ -735,9 +733,7 @@ cdef class TimestampScalar(Scalar):
type_format = str(_pc().strftime(self, format="%Y-%m-%dT%H:%M:%S%z"))
else:
type_format = str(_pc().strftime(self))
return '<pyarrow.{}: {!r}>'.format(
self.__class__.__name__, type_format
)
return f'<pyarrow.{self.__class__.__name__}: {type_format!r}>'
cdef class DurationScalar(Scalar):
@@ -783,10 +779,10 @@ cdef class DurationScalar(Scalar):
# otherwise safely truncate to microsecond resolution timedelta
if sp.value % 1000 != 0:
raise ValueError(
"Nanosecond duration {} is not safely convertible to "
f"Nanosecond duration {sp.value} is not safely convertible to "
"microseconds to convert to datetime.timedelta. Install "
"pandas to return as Timedelta with nanosecond support or "
"access the .value attribute.".format(sp.value)
"access the .value attribute."
)
return datetime.timedelta(microseconds=sp.value // 1000)
@@ -1049,9 +1045,7 @@ cdef class StructScalar(Scalar, collections.abc.Mapping):
return None
def __repr__(self):
return '<pyarrow.{}: {!r}>'.format(
self.__class__.__name__, self._as_py_tuple()
)
return f'<pyarrow.{self.__class__.__name__}: {self._as_py_tuple()!r}>'
def __str__(self):
return str(self._as_py_tuple())
@@ -1356,9 +1350,8 @@ cdef class ExtensionScalar(Scalar):
storage = None
elif isinstance(value, Scalar):
if value.type != typ.storage_type:
raise TypeError("Incompatible storage type {0} "
"for extension type {1}"
.format(value.type, typ))
raise TypeError(f"Incompatible storage type {value.type} "
f"for extension type {typ}")
storage = value
else:
storage = scalar(value, typ.storage_type)
+23 -28
View File
@@ -113,7 +113,7 @@ cdef class ChunkedArray(_PandasConvertible):
def __repr__(self):
type_format = object.__repr__(self)
return '{0}\n{1}'.format(type_format, str(self))
return f"{type_format}\n{self}"
def to_string(self, *, int indent=0, int window=5, int container_window=2,
c_bool skip_new_lines=False):
@@ -1568,8 +1568,8 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
schema.reset(new CSchema(c_fields, c_meta))
return arrays
else:
raise ValueError('Length of names ({}) does not match '
'length of arrays ({})'.format(len(names), K))
raise ValueError(f'Length of names ({len(names)}) does not match '
f'length of arrays ({K})')
c_fields.resize(K)
@@ -1578,8 +1578,8 @@ cdef _schema_from_arrays(arrays, names, metadata, shared_ptr[CSchema]* schema):
'Table or RecordBatch.')
if len(names) != K:
raise ValueError('Length of names ({}) does not match '
'length of arrays ({})'.format(len(names), K))
raise ValueError(f'Length of names ({len(names)}) does not match '
f'length of arrays ({K})')
converted_arrays = []
for i in range(K):
@@ -1726,11 +1726,10 @@ cdef class _Tabular(_PandasConvertible):
field_indices = self.schema.get_all_field_indices(i)
if len(field_indices) == 0:
raise KeyError("Field \"{}\" does not exist in schema"
.format(i))
raise KeyError(f'Field "{i}" does not exist in schema')
elif len(field_indices) > 1:
raise KeyError("Field \"{}\" exists {} times in schema"
.format(i, len(field_indices)))
raise KeyError(
f'Field "{i}" exists {len(field_indices)} times in schema')
else:
return field_indices[0]
elif isinstance(i, int):
@@ -2368,15 +2367,13 @@ cdef class _Tabular(_PandasConvertible):
show_field_metadata=show_metadata,
show_schema_metadata=show_metadata
)
title = 'pyarrow.{}\n{}'.format(type(self).__name__, schema_as_string)
title = f'pyarrow.{type(self).__name__}\n{schema_as_string}'
pieces = [title]
if preview_cols:
pieces.append('----')
for i in range(min(self.num_columns, preview_cols)):
pieces.append('{}: {}'.format(
self.field(i).name,
self.column(i).to_string(indent=0, skip_new_lines=True)
))
pieces.append(
f'{self.field(i).name}: {self.column(i).to_string(indent=0, skip_new_lines=True)}')
if preview_cols < self.num_columns:
pieces.append('...')
return '\n'.join(pieces)
@@ -2436,7 +2433,7 @@ cdef class _Tabular(_PandasConvertible):
for col in columns:
idx = self.schema.get_field_index(col)
if idx == -1:
raise KeyError("Column {!r} not found".format(col))
raise KeyError(f"Column {col!r} not found")
indices.append(idx)
indices.sort()
@@ -3072,7 +3069,7 @@ cdef class RecordBatch(_Tabular):
indices = self.schema.get_all_field_indices(name)
if not indices:
raise KeyError("Column {!r} not found".format(name))
raise KeyError(f"Column {name!r} not found")
for index in indices:
idx_to_new_name[index] = new_name
@@ -3347,14 +3344,13 @@ cdef class RecordBatch(_Tabular):
list newcols = []
if self.schema.names != target_schema.names:
raise ValueError("Target schema's field names are not matching "
"the record batch's field names: {!r}, {!r}"
.format(self.schema.names, target_schema.names))
raise ValueError(f"Target schema's field names are not matching "
f"the record batch's field names: {self.schema.names!r}, {target_schema.names!r}")
for column, field in zip(self.itercolumns(), target_schema):
if not field.nullable and column.null_count > 0:
raise ValueError("Casting field {!r} with null values to non-nullable"
.format(field.name))
raise ValueError(
f"Casting field {field.name!r} with null values to non-nullable")
casted = column.cast(field.type, safe=safe, options=options)
newcols.append(casted)
@@ -3550,7 +3546,7 @@ cdef class RecordBatch(_Tabular):
for arr in converted_arrays:
if len(arr) != num_rows:
raise ValueError('Arrays were not all the same length: '
'{0} vs {1}'.format(len(arr), num_rows))
f'{len(arr)} vs {num_rows}')
c_arrays.push_back(arr.sp_array)
result = pyarrow_wrap_batch(CRecordBatch.Make(c_schema, num_rows,
@@ -4714,14 +4710,13 @@ cdef class Table(_Tabular):
list newcols = []
if self.schema.names != target_schema.names:
raise ValueError("Target schema's field names are not matching "
"the table's field names: {!r}, {!r}"
.format(self.schema.names, target_schema.names))
raise ValueError(f"Target schema's field names are not matching "
f"the table's field names: {self.schema.names!r}, {target_schema.names!r}")
for column, field in zip(self.itercolumns(), target_schema):
if not field.nullable and column.null_count > 0:
raise ValueError("Casting field {!r} with null values to non-nullable"
.format(field.name))
raise ValueError(
f"Casting field {field.name!r} with null values to non-nullable")
casted = column.cast(field.type, safe=safe, options=options)
newcols.append(casted)
@@ -5558,7 +5553,7 @@ cdef class Table(_Tabular):
indices = self.schema.get_all_field_indices(name)
if not indices:
raise KeyError("Column {!r} not found".format(name))
raise KeyError(f"Column {name!r} not found")
for index in indices:
idx_to_new_name[index] = new_name
+25 -25
View File
@@ -53,10 +53,10 @@ cdef class Tensor(_Weakrefable):
return _struct.pack(f"{len(values)}n", *values)
def __repr__(self):
return """<pyarrow.Tensor>
type: {0.type}
shape: {0.shape}
strides: {0.strides}""".format(self)
return f"""<pyarrow.Tensor>
type: {self.type}
shape: {self.shape}
strides: {self.strides}"""
@staticmethod
def from_numpy(obj, dim_names=None):
@@ -321,8 +321,8 @@ cdef class SparseCOOTensor(_Weakrefable):
def __repr__(self):
return """<pyarrow.SparseCOOTensor>
type: {0.type}
shape: {0.shape}""".format(self)
type: {self.type}
shape: {self.shape}"""
@classmethod
def from_dense_numpy(cls, obj, dim_names=None):
@@ -394,7 +394,7 @@ shape: {0.shape}""".format(self)
import scipy.sparse
if not isinstance(obj, scipy.sparse.coo_matrix):
raise TypeError(
"Expected scipy.sparse.coo_matrix, got {}".format(type(obj)))
f"Expected scipy.sparse.coo_matrix, got {type(obj)}")
cdef shared_ptr[CSparseCOOTensor] csparse_tensor
cdef vector[int64_t] c_shape
@@ -440,7 +440,7 @@ shape: {0.shape}""".format(self)
import sparse
if not isinstance(obj, sparse.COO):
raise TypeError(
"Expected sparse.COO, got {}".format(type(obj)))
f"Expected sparse.COO, got {type(obj)}")
cdef shared_ptr[CSparseCOOTensor] csparse_tensor
cdef vector[int64_t] c_shape
@@ -624,9 +624,9 @@ cdef class SparseCSRMatrix(_Weakrefable):
self.type = pyarrow_wrap_data_type(self.stp.type())
def __repr__(self):
return """<pyarrow.SparseCSRMatrix>
type: {0.type}
shape: {0.shape}""".format(self)
return f"""<pyarrow.SparseCSRMatrix>
type: {self.type}
shape: {self.shape}"""
@classmethod
def from_dense_numpy(cls, obj, dim_names=None):
@@ -705,7 +705,7 @@ shape: {0.shape}""".format(self)
import scipy.sparse
if not isinstance(obj, scipy.sparse.csr_matrix):
raise TypeError(
"Expected scipy.sparse.csr_matrix, got {}".format(type(obj)))
f"Expected scipy.sparse.csr_matrix, got {type(obj)}")
cdef shared_ptr[CSparseCSRMatrix] csparse_tensor
cdef vector[int64_t] c_shape
@@ -865,9 +865,9 @@ cdef class SparseCSCMatrix(_Weakrefable):
self.type = pyarrow_wrap_data_type(self.stp.type())
def __repr__(self):
return """<pyarrow.SparseCSCMatrix>
type: {0.type}
shape: {0.shape}""".format(self)
return f"""<pyarrow.SparseCSCMatrix>
type: {self.type}
shape: {self.shape}"""
@classmethod
def from_dense_numpy(cls, obj, dim_names=None):
@@ -946,7 +946,7 @@ shape: {0.shape}""".format(self)
import scipy.sparse
if not isinstance(obj, scipy.sparse.csc_matrix):
raise TypeError(
"Expected scipy.sparse.csc_matrix, got {}".format(type(obj)))
f"Expected scipy.sparse.csc_matrix, got {type(obj)}")
cdef shared_ptr[CSparseCSCMatrix] csparse_tensor
cdef vector[int64_t] c_shape
@@ -1115,9 +1115,9 @@ cdef class SparseCSFTensor(_Weakrefable):
self.type = pyarrow_wrap_data_type(self.stp.type())
def __repr__(self):
return """<pyarrow.SparseCSFTensor>
type: {0.type}
shape: {0.shape}""".format(self)
return f"""<pyarrow.SparseCSFTensor>
type: {self.type}
shape: {self.shape}"""
@classmethod
def from_dense_numpy(cls, obj, dim_names=None):
@@ -1183,14 +1183,14 @@ shape: {0.shape}""".format(self)
# Enforce preconditions for SparseCSFTensor indices
if not (isinstance(indptr, (list, tuple)) and
isinstance(indices, (list, tuple))):
raise TypeError("Expected list or tuple, got {}, {}"
.format(type(indptr), type(indices)))
raise TypeError(
f"Expected list or tuple, got {type(indptr)}, {type(indices)}")
if len(indptr) != len(shape) - 1:
raise ValueError("Expected list of {ndim} np.arrays for "
"SparseCSFTensor.indptr".format(ndim=len(shape)))
raise ValueError(f"Expected list of {len(shape)} np.arrays for "
"SparseCSFTensor.indptr")
if len(indices) != len(shape):
raise ValueError("Expected list of {ndim} np.arrays for "
"SparseCSFTensor.indices".format(ndim=len(shape)))
raise ValueError(f"Expected list of {len(shape)} np.arrays for "
"SparseCSFTensor.indices")
if any([x.ndim != 1 for x in indptr]):
raise ValueError("Expected a list of 1-dimensional arrays for "
"SparseCSFTensor.indptr")
+10 -11
View File
@@ -83,17 +83,16 @@ def pytest_addoption(parser):
elif value in {'0', 'false', 'off', 'no', 'n'}:
return False
else:
raise ValueError('{}={} is not parsable as boolean'
.format(name.upper(), value))
raise ValueError(f'{name.upper()}={value} is not parsable as boolean')
for group in groups:
default = bool_env('PYARROW_TEST_{}'.format(group), defaults[group])
parser.addoption('--enable-{}'.format(group),
default = bool_env(f'PYARROW_TEST_{group}', defaults[group])
parser.addoption(f'--enable-{group}',
action='store_true', default=default,
help=('Enable the {} test group'.format(group)))
parser.addoption('--disable-{}'.format(group),
help=(f'Enable the {group} test group'))
parser.addoption(f'--disable-{group}',
action='store_true', default=False,
help=('Disable the {} test group'.format(group)))
help=(f'Disable the {group} test group'))
class PyArrowConfig:
@@ -107,7 +106,7 @@ class PyArrowConfig:
def requires(self, group):
if not self.is_enabled[group]:
pytest.skip('{} NOT enabled'.format(group))
pytest.skip(f'{group} NOT enabled')
def pytest_configure(config):
@@ -119,8 +118,8 @@ def pytest_configure(config):
"markers", mark,
)
enable_flag = '--enable-{}'.format(mark)
disable_flag = '--disable-{}'.format(mark)
enable_flag = f'--enable-{mark}'
disable_flag = f'--disable-{mark}'
is_enabled = (config.getoption(enable_flag) and not
config.getoption(disable_flag))
@@ -217,7 +216,7 @@ def s3_server(s3_connection, tmpdir_factory):
tmpdir = tmpdir_factory.getbasetemp()
host, port, access_key, secret_key = s3_connection
address = '{}:{}'.format(host, port)
address = f'{host}:{port}'
env = os.environ.copy()
env.update({
'MINIO_ACCESS_KEY': access_key,
+1 -1
View File
@@ -157,7 +157,7 @@ def dataframe_with_lists(include_index=False, parquet_compatible=False):
]
for value_type, data in temporal_pairs:
field_name = '{}_list'.format(value_type)
field_name = f'{value_type}_list'
field_type = pa.list_(value_type)
field = pa.field(field_name, field_type)
fields.append(field)
+5 -6
View File
@@ -49,7 +49,7 @@ def s3_bucket(s3_server):
host, port, access_key, secret_key = s3_server['connection']
s3_client = boto3.client(
's3',
endpoint_url='http://{}:{}'.format(host, port),
endpoint_url=f'http://{host}:{port}',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=botocore.client.Config(signature_version='s3v4'),
@@ -75,11 +75,11 @@ def s3_example_s3fs(s3_server, s3_bucket):
key=access_key,
secret=secret_key,
client_kwargs={
'endpoint_url': 'http://{}:{}'.format(host, port)
'endpoint_url': f'http://{host}:{port}'
}
)
test_path = '{}/{}'.format(s3_bucket, guid())
test_path = f'{s3_bucket}/{guid()}'
fs.mkdir(test_path)
yield fs, test_path
@@ -95,9 +95,8 @@ def s3_example_fs(s3_server):
host, port, access_key, secret_key = s3_server['connection']
uri = (
"s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
"&allow_bucket_creation=True"
.format(access_key, secret_key, host, port)
f"s3://{access_key}:{secret_key}@mybucket/data.parquet?scheme=http"
f"&endpoint_override={host}:{port}&allow_bucket_creation=True"
)
fs, path = FileSystem.from_uri(uri)
@@ -183,7 +183,7 @@ def test_dictionary_array_automatically_read():
# Make a large dictionary, a little over 4MB of data
dict_length = 4000
dict_values = pa.array([('x' * 1000 + '_{}'.format(i))
dict_values = pa.array([('x' * 1000 + f'_{i}')
for i in range(dict_length)])
num_chunks = 10
@@ -220,8 +220,7 @@ def test_decimal_roundtrip(tempdir):
util.randdecimal(precision, scale)
for _ in range(num_values)
]
column_name = ('dec_precision_{:d}_scale_{:d}'
.format(precision, scale))
column_name = f'dec_precision_{precision}_scale_{scale}'
columns[column_name] = random_decimal_values
expected = pd.DataFrame(columns)
@@ -255,7 +254,7 @@ def test_decimal_roundtrip_negative_scale(tempdir):
@pytest.mark.parametrize('dtype', [int, float])
def test_single_pylist_column_roundtrip(tempdir, dtype,):
filename = tempdir / 'single_{}_column.parquet'.format(dtype.__name__)
filename = tempdir / f'single_{dtype.__name__}_column.parquet'
data = [pa.array(list(map(dtype, range(5))))]
table = pa.Table.from_arrays(data, names=['a'])
_write_table(table, filename)
+10 -10
View File
@@ -566,7 +566,7 @@ def _generate_partition_directories(fs, base_dir, partition_spec, df):
level_dir = pathsep.join([
str(base_dir),
'{}={}'.format(name, value)
f'{name}={value}'
])
fs.create_dir(level_dir)
@@ -648,7 +648,7 @@ def test_read_multiple_files(tempdir):
# Hack so that we don't have a dtype cast in v1 files
df['uint32'] = df['uint32'].astype(np.int64)
path = dirpath / '{}.parquet'.format(i)
path = dirpath / f'{i}.parquet'
table = pa.Table.from_pandas(df)
_write_table(table, path)
@@ -683,7 +683,7 @@ def test_read_multiple_files(tempdir):
# Test failure modes with non-uniform metadata
bad_apple = _test_dataframe(size, seed=i).iloc[:, :4]
bad_apple_path = tempdir / '{}.parquet'.format(guid())
bad_apple_path = tempdir / f'{guid()}.parquet'
t = pa.Table.from_pandas(bad_apple)
_write_table(t, bad_apple_path)
@@ -720,7 +720,7 @@ def test_dataset_read_pandas(tempdir):
df.index = np.arange(i * size, (i + 1) * size)
df.index.name = 'index'
path = dirpath / '{}.parquet'.format(i)
path = dirpath / f'{i}.parquet'
table = pa.Table.from_pandas(df)
_write_table(table, path)
@@ -749,7 +749,7 @@ def test_dataset_memory_map(tempdir):
dirpath.mkdir()
df = _test_dataframe(10, seed=0)
path = dirpath / '{}.parquet'.format(0)
path = dirpath / '0.parquet'
table = pa.Table.from_pandas(df)
_write_table(table, path, version='2.6')
@@ -764,7 +764,7 @@ def test_dataset_enable_buffered_stream(tempdir):
dirpath.mkdir()
df = _test_dataframe(10, seed=0)
path = dirpath / '{}.parquet'.format(0)
path = dirpath / '0.parquet'
table = pa.Table.from_pandas(df)
_write_table(table, path, version='2.6')
@@ -784,7 +784,7 @@ def test_dataset_enable_pre_buffer(tempdir):
dirpath.mkdir()
df = _test_dataframe(10, seed=0)
path = dirpath / '{}.parquet'.format(0)
path = dirpath / '0.parquet'
table = pa.Table.from_pandas(df)
_write_table(table, path, version='2.6')
@@ -801,7 +801,7 @@ def _make_example_multifile_dataset(base_path, nfiles=10, file_nrows=5):
paths = []
for i in range(nfiles):
df = _test_dataframe(file_nrows, seed=i)
path = base_path / '{}.parquet'.format(i)
path = base_path / f'{i}.parquet'
test_data.append(_write_table(df, path))
paths.append(path)
@@ -823,7 +823,7 @@ def test_ignore_private_directories(tempdir, dir_prefix):
file_nrows=5)
# private directory
(dirpath / '{}staging'.format(dir_prefix)).mkdir()
(dirpath / f'{dir_prefix}staging').mkdir()
dataset = pq.ParquetDataset(dirpath)
@@ -873,7 +873,7 @@ def test_ignore_hidden_files_underscore(tempdir):
def test_ignore_no_private_directories_in_base_path(tempdir, dir_prefix):
# ARROW-8427 - don't ignore explicitly listed files if parent directory
# is a private directory
dirpath = tempdir / "{0}data".format(dir_prefix) / guid()
dirpath = tempdir / f'{dir_prefix}data' / guid()
dirpath.mkdir(parents=True)
paths = _make_example_multifile_dataset(dirpath, nfiles=10,
@@ -267,7 +267,7 @@ def test_statistics_convert_logical_types(tempdir):
for i, (min_val, max_val, typ) in enumerate(cases):
t = pa.Table.from_arrays([pa.array([min_val, max_val], type=typ)],
['col'])
path = str(tempdir / ('example{}.parquet'.format(i)))
path = str(tempdir / f'example{i}.parquet')
pq.write_table(t, path, version='2.6')
pf = pq.ParquetFile(path)
stats = pf.metadata.row_group(0).column(0).statistics
+1 -1
View File
@@ -588,7 +588,7 @@ def test_dataset_read_pandas_common_metadata(
np.arange(i * size, (i + 1) * size, dtype="int64"), name='index'
)
path = dirpath / '{}.parquet'.format(i)
path = dirpath / f'{i}.parquet'
table = pa.Table.from_pandas(df, preserve_index=preserve_index)
+2 -2
View File
@@ -246,7 +246,7 @@ def test_to_numpy_writable():
@pytest.mark.parametrize('tz', [None, "UTC"])
def test_to_numpy_datetime64(unit, tz):
arr = pa.array([1, 2, 3], pa.timestamp(unit, tz=tz))
expected = np.array([1, 2, 3], dtype="datetime64[{}]".format(unit))
expected = np.array([1, 2, 3], dtype=f"datetime64[{unit}]")
np_arr = arr.to_numpy()
np.testing.assert_array_equal(np_arr, expected)
@@ -255,7 +255,7 @@ def test_to_numpy_datetime64(unit, tz):
@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
def test_to_numpy_timedelta64(unit):
arr = pa.array([1, 2, 3], pa.duration(unit))
expected = np.array([1, 2, 3], dtype="timedelta64[{}]".format(unit))
expected = np.array([1, 2, 3], dtype=f"timedelta64[{unit}]")
np_arr = arr.to_numpy()
np.testing.assert_array_equal(np_arr, expected)
+2 -2
View File
@@ -559,7 +559,7 @@ class BaseTestCSV(abc.ABC):
if self.use_threads:
row_info = ""
else:
row_info = "Row #{}: ".format(row)
row_info = f"Row #{row}: "
return msg_format.format(row_info, *args)
csv, _ = make_random_csv(4, 100, write_names=True)
@@ -1423,7 +1423,7 @@ class BaseCSVTableRead(BaseTestCSV):
except AttributeError:
clock = time.time
num_columns = 10000
col_names = ["K{}".format(i) for i in range(num_columns)]
col_names = [f"K{i}" for i in range(num_columns)]
csv = make_empty_csv(col_names)
t1 = clock()
convert_options = ConvertOptions(
+4 -2
View File
@@ -747,8 +747,10 @@ def make_table():
dictionary=a2)
arrays = [a0, a1, a2, a3, a4, a5]
schema = pa.schema([('f{}'.format(i), arr.type)
for i, arr in enumerate(arrays)])
schema = pa.schema([
(f'f{i}', arr.type)
for i, arr in enumerate(arrays)
])
batch = pa.record_batch(arrays, schema=schema)
table = pa.Table.from_batches([batch])
return table
+4 -5
View File
@@ -123,7 +123,7 @@ def test_cython_api(tmpdir):
# Check the extension module is loadable from a subprocess without
# pyarrow imported first.
code = """if 1:
code = f"""if 1:
import sys
import os
@@ -131,17 +131,16 @@ def test_cython_api(tmpdir):
# Add dll directory was added on python 3.8
# and is required in order to find extra DLLs
# only for win32
for dir in {library_dirs}:
for dir in {pa.get_library_dirs()}:
os.add_dll_directory(dir)
except AttributeError:
pass
mod = __import__({mod_name!r})
mod = __import__('pyarrow_cython_example')
arr = mod.make_null_array(5)
assert mod.get_array_length(arr) == 5
assert arr.null_count == 5
""".format(mod_name='pyarrow_cython_example',
library_dirs=pa.get_library_dirs())
"""
path_var = None
if sys.platform == 'win32':
+24 -25
View File
@@ -122,7 +122,7 @@ def mockfs():
]
for i, directory in enumerate(directories):
path = '{}/file{}.parquet'.format(directory, i)
path = f'{directory}/file{i}.parquet'
mockfs.create_dir(directory)
with mockfs.open_output_stream(path) as out:
data = [
@@ -198,15 +198,15 @@ def multisourcefs(request):
mockfs.create_dir('plain')
n = len(df_a)
for i, chunk in enumerate([df_a.iloc[i:i+n//10] for i in range(0, n, n//10)]):
path = 'plain/chunk-{}.parquet'.format(i)
path = f'plain/chunk-{i}.parquet'
with mockfs.open_output_stream(path) as out:
pq.write_table(_table_from_pandas(chunk), out)
# create one with schema partitioning by weekday and color
mockfs.create_dir('schema')
for part, chunk in df_b.groupby([df_b.date.dt.dayofweek, df_b.color]):
folder = 'schema/{}/{}'.format(*part)
path = '{}/chunk.parquet'.format(folder)
folder = f'schema/{part[0]}/{part[1]}'
path = f'{folder}/chunk.parquet'
mockfs.create_dir(folder)
with mockfs.open_output_stream(path) as out:
pq.write_table(_table_from_pandas(chunk), out)
@@ -214,8 +214,8 @@ def multisourcefs(request):
# create one with hive partitioning by year and month
mockfs.create_dir('hive')
for part, chunk in df_c.groupby([df_c.date.dt.year, df_c.date.dt.month]):
folder = 'hive/year={}/month={}'.format(*part)
path = '{}/chunk.parquet'.format(folder)
folder = f'hive/year={part[0]}/month={part[1]}'
path = f'{folder}/chunk.parquet'
mockfs.create_dir(folder)
with mockfs.open_output_stream(path) as out:
pq.write_table(_table_from_pandas(chunk), out)
@@ -223,8 +223,8 @@ def multisourcefs(request):
# create one with hive partitioning by color
mockfs.create_dir('hive_color')
for part, chunk in df_d.groupby("color"):
folder = 'hive_color/color={}'.format(part)
path = '{}/chunk.parquet'.format(folder)
folder = f'hive_color/color={part}'
path = f'{folder}/chunk.parquet'
mockfs.create_dir(folder)
with mockfs.open_output_stream(path) as out:
pq.write_table(_table_from_pandas(chunk), out)
@@ -1801,8 +1801,8 @@ def test_fragments_repr(tempdir, dataset):
fragment = list(dataset.get_fragments())[0]
assert (
repr(fragment) ==
"<pyarrow.dataset.ParquetFileFragment path={}>".format(
dataset.filesystem.normalize_path(str(path)))
"<pyarrow.dataset.ParquetFileFragment path="
f"{dataset.filesystem.normalize_path(str(path))}>"
)
# non-parquet format
@@ -1812,8 +1812,8 @@ def test_fragments_repr(tempdir, dataset):
fragment = list(dataset.get_fragments())[0]
assert (
repr(fragment) ==
"<pyarrow.dataset.FileFragment type=ipc path={}>".format(
dataset.filesystem.normalize_path(str(path)))
"<pyarrow.dataset.FileFragment type=ipc path="
f"{dataset.filesystem.normalize_path(str(path))}>"
)
@@ -2569,7 +2569,7 @@ def _create_partitioned_dataset(basedir):
path.mkdir()
for i in range(3):
part = path / "part={}".format(i)
part = path / f"part={i}"
part.mkdir()
pq.write_table(table.slice(3*i, 3), part / "test.parquet")
@@ -2781,9 +2781,8 @@ def s3_example_simple(s3_server):
host, port, access_key, secret_key = s3_server['connection']
uri = (
"s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
"&allow_bucket_creation=True"
.format(access_key, secret_key, host, port)
f"s3://{access_key}:{secret_key}@mybucket/data.parquet?scheme=http"
f"&endpoint_override={host}:{port}&allow_bucket_creation=True"
)
fs, path = FileSystem.from_uri(uri)
@@ -2833,7 +2832,7 @@ def test_open_dataset_from_uri_s3_fsspec(s3_example_simple):
key=access_key,
secret=secret_key,
client_kwargs={
'endpoint_url': 'http://{}:{}'.format(host, port)
'endpoint_url': f'http://{host}:{port}'
}
)
@@ -2855,10 +2854,10 @@ def test_open_dataset_from_s3_with_filesystem_uri(s3_server):
host, port, access_key, secret_key = s3_server['connection']
bucket = 'theirbucket'
path = 'nested/folder/data.parquet'
uri = "s3://{}:{}@{}/{}?scheme=http&endpoint_override={}:{}"\
"&allow_bucket_creation=true".format(
access_key, secret_key, bucket, path, host, port
)
uri = (
f"s3://{access_key}:{secret_key}@{bucket}/{path}?scheme=http"
f"&endpoint_override={host}:{port}&allow_bucket_creation=true"
)
fs, path = FileSystem.from_uri(uri)
assert path == 'theirbucket/nested/folder/data.parquet'
@@ -3361,7 +3360,7 @@ def test_csv_format(tempdir, dataset_reader):
])
def test_csv_format_compressed(tempdir, compression, dataset_reader):
if not pyarrow.Codec.is_available(compression):
pytest.skip("{} support is not built".format(compression))
pytest.skip(f"{compression} support is not built")
table = pa.table({'a': pa.array([1, 2, 3], type="int64"),
'b': pa.array([.1, .2, .3], type="float64")})
filesystem = fs.LocalFileSystem()
@@ -4778,7 +4777,7 @@ def test_write_dataset_parquet(tempdir):
format = ds.ParquetFileFormat()
opts = format.make_write_options(version=version)
assert "<pyarrow.dataset.ParquetFileWriteOptions" in repr(opts)
base_dir = tempdir / 'parquet_dataset_version{0}'.format(version)
base_dir = tempdir / f'parquet_dataset_version{version}'
ds.write_dataset(table, base_dir, format=format, file_options=opts)
meta = pq.read_metadata(base_dir / "part-0.parquet")
expected_version = "1.0" if version == "1.0" else "2.6"
@@ -5001,7 +5000,7 @@ def test_write_dataset_s3_put_only(s3_server):
fs = S3FileSystem(
access_key='test_dataset_limited_user',
secret_key='limited123',
endpoint_override='{}:{}'.format(host, port),
endpoint_override=f'{host}:{port}',
scheme='http'
)
@@ -5049,7 +5048,7 @@ def test_write_dataset_s3_put_only(s3_server):
fs = S3FileSystem(
access_key='limited',
secret_key='limited123',
endpoint_override='{}:{}'.format(host, port),
endpoint_override=f'{host}:{port}',
scheme='http',
allow_bucket_creation=True,
)
+1 -1
View File
@@ -864,7 +864,7 @@ class PeriodType(pa.ExtensionType):
return self._freq
def __arrow_ext_serialize__(self):
return "freq={}".format(self.freq).encode()
return f"freq={self.freq}".encode()
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
+1 -1
View File
@@ -443,7 +443,7 @@ def test_all_null_category(version):
@pytest.mark.pandas
def test_multithreaded_read(version):
data = {'c{}'.format(i): [''] * 10
data = {f'c{i}': [''] * 10
for i in range(100)}
df = pd.DataFrame(data)
_check_pandas_roundtrip(df, use_threads=True, version=version)
+4 -4
View File
@@ -83,9 +83,9 @@ def read_flight_resource(path):
return f.read()
except FileNotFoundError:
raise RuntimeError(
"Test resource {} not found; did you initialize the "
"test resource submodule?\n{}".format(root / path,
traceback.format_exc()))
f"Test resource {root / path} not found; did you initialize the "
f"test resource submodule?\n{traceback.format_exc()}"
)
def example_tls_certs():
@@ -431,7 +431,7 @@ class ExchangeFlightServer(FlightServerBase):
return self.exchange_transform(context, reader, writer)
else:
raise pa.ArrowInvalid(
"Unknown command: {}".format(descriptor.command))
f"Unknown command: {descriptor.command}")
def exchange_do_get(self, context, reader, writer):
"""Emulate DoGet with DoExchange."""
+10 -11
View File
@@ -128,13 +128,13 @@ class DummyHandler(FileSystemHandler):
def open_input_stream(self, path):
if "notfound" in path:
raise FileNotFoundError(path)
data = "{0}:input_stream".format(path).encode('utf8')
data = f"{path}:input_stream".encode('utf8')
return pa.BufferReader(data)
def open_input_file(self, path):
if "notfound" in path:
raise FileNotFoundError(path)
data = "{0}:input_file".format(path).encode('utf8')
data = f"{path}:input_file".encode('utf8')
return pa.BufferReader(data)
def open_output_stream(self, path, metadata):
@@ -249,7 +249,7 @@ def s3fs(request, s3_server):
fs = S3FileSystem(
access_key=access_key,
secret_key=secret_key,
endpoint_override='{}:{}'.format(host, port),
endpoint_override=f'{host}:{port}',
scheme='http',
allow_bucket_creation=True,
allow_bucket_deletion=True
@@ -382,7 +382,7 @@ def py_fsspec_s3fs(request, s3_server):
fs = s3fs.S3FileSystem(
key=access_key,
secret=secret_key,
client_kwargs=dict(endpoint_url='http://{}:{}'.format(host, port))
client_kwargs=dict(endpoint_url=f'http://{host}:{port}')
)
fs = PyFileSystem(FSSpecHandler(fs))
fs.create_dir(bucket)
@@ -523,7 +523,7 @@ def test_s3fs_limited_permissions_create_bucket(s3_server):
fs = S3FileSystem(
access_key='test_fs_limited_user',
secret_key='limited123',
endpoint_override='{}:{}'.format(host, port),
endpoint_override=f'{host}:{port}',
scheme='http'
)
fs.create_dir('existing-bucket/test')
@@ -766,7 +766,7 @@ def test_get_file_info_with_selector(fs, pathfn):
info.path.rstrip("/").endswith(dir_b)):
assert info.type == FileType.Directory
else:
raise ValueError('unexpected path {}'.format(info.path))
raise ValueError(f'unexpected path {info.path}')
check_mtime_or_absent(info)
# non-recursive selector -> not selecting the nested file_c
@@ -1510,7 +1510,7 @@ def test_hdfs_options(hdfs_connection, pickle_module):
host, port, 'me', replication + 1, buffer_size, default_block_size
))
hdfs5 = HadoopFileSystem(host, port)
hdfs6 = HadoopFileSystem.from_uri('hdfs://{}:{}'.format(host, port))
hdfs6 = HadoopFileSystem.from_uri(f'hdfs://{host}:{port}')
hdfs7 = HadoopFileSystem(host, port, user='localuser')
hdfs8 = HadoopFileSystem(host, port, user='localuser',
kerb_ticket="cache_path")
@@ -1550,7 +1550,7 @@ def test_hdfs_options(hdfs_connection, pickle_module):
assert hdfs.get_file_info(FileSelector('/'))
hdfs = HadoopFileSystem.from_uri(
"hdfs://{}:{}/?user={}".format(host, port, user)
f"hdfs://{host}:{port}/?user={user}"
)
assert hdfs.get_file_info(FileSelector('/'))
@@ -1596,9 +1596,8 @@ def test_filesystem_from_uri_s3(s3_server):
host, port, access_key, secret_key = s3_server['connection']
uri = "s3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}"\
"&allow_bucket_creation=True" \
.format(access_key, secret_key, host, port)
uri = f"s3://{access_key}:{secret_key}@mybucket/foo/bar?scheme=http&" \
f"endpoint_override={host}:{port}&allow_bucket_creation=True"
fs, path = FileSystem.from_uri(uri)
assert isinstance(fs, S3FileSystem)
+4 -4
View File
@@ -822,7 +822,7 @@ def test_cache_options_pickling(pickle_module):
])
def test_compress_decompress(compression):
if not Codec.is_available(compression):
pytest.skip("{} support is not built".format(compression))
pytest.skip(f"{compression} support is not built")
INPUT_SIZE = 10000
test_data = (np.random.randint(0, 255, size=INPUT_SIZE)
@@ -863,7 +863,7 @@ def test_compress_decompress(compression):
])
def test_compression_level(compression):
if not Codec.is_available(compression):
pytest.skip("{} support is not built".format(compression))
pytest.skip(f"{compression} support is not built")
codec = Codec(compression)
if codec.name == "snappy":
@@ -1755,7 +1755,7 @@ def test_unknown_compression_raises():
])
def test_compressed_roundtrip(compression):
if not Codec.is_available(compression):
pytest.skip("{} support is not built".format(compression))
pytest.skip(f"{compression} support is not built")
data = b"some test data\n" * 10 + b"eof\n"
raw = pa.BufferOutputStream()
@@ -1776,7 +1776,7 @@ def test_compressed_roundtrip(compression):
)
def test_compressed_recordbatch_stream(compression):
if not Codec.is_available(compression):
pytest.skip("{} support is not built".format(compression))
pytest.skip(f"{compression} support is not built")
# ARROW-4836: roundtrip a RecordBatch through a compressed stream
table = pa.Table.from_arrays([pa.array([1, 2, 3, 4, 5])], ['a'])
+4 -4
View File
@@ -45,7 +45,7 @@ def root_allocator():
}).text
jar_path = os.path.join(
arrow_dir, 'java', 'tools', 'target',
'arrow-tools-{}-jar-with-dependencies.jar'.format(version))
f'arrow-tools-{version}-jar-with-dependencies.jar')
jar_path = os.getenv("ARROW_TOOLS_JAR", jar_path)
kwargs = {}
# This will be the default behaviour in jpype 0.8+
@@ -224,7 +224,7 @@ def test_jvm_types(root_allocator, pa_type, jvm_spec, nullable):
])
def test_jvm_array(root_allocator, pa_type, py_data, jvm_type):
# Create vector
cls = "org.apache.arrow.vector.{}".format(jvm_type)
cls = f"org.apache.arrow.vector.{jvm_type}"
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
jvm_vector.allocateNew(len(py_data))
for i, val in enumerate(py_data):
@@ -241,7 +241,7 @@ def test_jvm_array(root_allocator, pa_type, py_data, jvm_type):
def test_jvm_array_empty(root_allocator):
cls = "org.apache.arrow.vector.{}".format('IntVector')
cls = f"org.apache.arrow.vector.{'IntVector'}"
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
jvm_vector.allocateNew()
jvm_array = pa_jvm.array(jvm_vector)
@@ -362,7 +362,7 @@ def test_jvm_array_empty(root_allocator):
def test_jvm_record_batch(root_allocator, pa_type, py_data, jvm_type,
jvm_spec):
# Create vector
cls = "org.apache.arrow.vector.{}".format(jvm_type)
cls = f"org.apache.arrow.vector.{jvm_type}"
jvm_vector = jpype.JClass(cls)("vector", root_allocator)
jvm_vector.allocateNew(len(py_data))
for i, val in enumerate(py_data):
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff. Show more