mirrors/ClickHouse
0
0 Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2026-10-28 16:18:23 +08:00
Code Activity

Merge pull request #122952 from ClickHouse/remove-unused-columns-single-pass

Browse Source 
Refactor remove-unused-columns optimization
This commit is contained in:
Nikolai Kochetov authored and GitHub committed 2026-10-07 12:44:04 +00:00
2 parents 1899759dcd e28358b5bf
commit e95690ab45
42 files changed
+1951 -1062
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/Interpreters/ActionsDAG.cpp
+42 -60
View File
@@ -1415,9 +1415,15 @@ void ActionsDAG::foldFilterPredicateThroughMaterialize(const std::string & filte
{
if (filter_column_name.empty())
return;
const Node * filter_node = tryFindInOutputs(filter_column_name);
if (!filter_node)
return;
const auto it = std::ranges::find_if(outputs, [&](const Node * output) { return output->result_name == filter_column_name; });
if (it != outputs.end())
foldFilterPredicateThroughMaterialize(static_cast<size_t>(it - outputs.begin()));
}
void ActionsDAG::foldFilterPredicateThroughMaterialize(size_t filter_output_position)
{
const Node * filter_node = outputs.at(filter_output_position);
/// A prior optimizer pass may already have folded this filter. Replacing an
/// existing const output with another const output makes the pass report a
@@ -1435,15 +1441,8 @@ void ActionsDAG::foldFilterPredicateThroughMaterialize(const std::string & filte
/// `removeUnusedActions` prunes the now-orphan subtree later
const Node & new_const = addColumn(
std::move(folded->column), filter_node->result_type,
std::string(filter_column_name), folded->deterministic, folded->masked_secret);
for (auto & out : outputs)
{
if (out == filter_node)
{
out = &new_const;
break;
}
}
filter_node->result_name, folded->deterministic, folded->masked_secret);
outputs[filter_output_position] = &new_const;
}
void ActionsDAG::deduplicateSubtrees()
@@ -1853,22 +1852,6 @@ ActionsDAG::MatchedInputPositions ActionsDAG::matchInputPositionsToHeader(const
return matchInputNodesToHeader(inputs, header);
}
ActionsDAG::SplitOutputPositions ActionsDAG::splitOutputPositions(const std::vector<size_t> & output_positions) const
{
SplitOutputPositions result;
const size_t num_dag_outputs = outputs.size();
for (size_t pos : output_positions)
{
if (pos < num_dag_outputs)
result.dag_indices.push_back(pos);
else
result.passthrough_indices.push_back(pos - num_dag_outputs);
}
return result;
}
ColumnsWithTypeAndName ActionsDAG::evaluatePartialResult(
IntermediateExecutionResult & node_to_column,
const NodeRawConstPtrs & outputs,
@@ -2223,50 +2206,49 @@ bool ActionsDAG::tryRestoreColumn(const std::string & column_name)
bool ActionsDAG::removeUnusedResult(const std::string & column_name)
{
/// Find column in output nodes and remove.
const Node * col = nullptr;
auto output_it = std::ranges::find_if(outputs, [&](const Node * node) { return node->result_name == column_name; });
if (output_it == outputs.end())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found result {} in ActionsDAG\n{}", column_name, dumpDAG());
/// The nodes the result is computed by, each the only child of the one before, down to an input or a column.
std::vector<const Node *> chain;
for (const Node * node = *output_it;; node = node->children.front())
{
auto it = outputs.begin();
for (; it != outputs.end(); ++it)
if ((*it)->result_name == column_name)
break;
if (node->children.size() > 1)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Result {} is not a chain of nodes down to an input or a column: {} has {} children\n{}",
column_name, node->result_name, node->children.size(), dumpDAG());
if (it == outputs.end())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not found result {} in ActionsDAG\n{}", column_name, dumpDAG());
col = *it;
outputs.erase(it);
chain.push_back(node);
if (node->children.empty())
break;
}
/// Check if column is in input.
auto it = inputs.begin();
for (; it != inputs.end(); ++it)
if (*it == col)
break;
outputs.erase(output_it);
/// Check column has no dependent.
std::unordered_map<const Node *, size_t> uses;
for (const auto & node : nodes)
for (const auto * child : node.children)
if (col == child)
return false;
++uses[child];
for (const auto * output : outputs)
++uses[output];
/// Do not remove input if it was mentioned in output nodes several times.
for (const auto * output_node : outputs)
if (col == output_node)
/// Remove the chain from the top while nothing else uses a node: another node, or an output of the same name or of
/// another one.
for (const auto * node : chain)
{
if (uses[node] != 0)
return false;
/// Remove from nodes and inputs.
for (auto jt = nodes.begin(); jt != nodes.end(); ++jt)
{
if (&(*jt) == col)
{
nodes.erase(jt);
break;
}
if (!node->children.empty())
--uses[node->children.front()];
if (node->type == ActionType::INPUT)
std::erase(inputs, node);
nodes.remove_if([&](const Node & candidate) { return &candidate == node; });
}
if (it != inputs.end())
inputs.erase(it);
return true;
}
src/Interpreters/ActionsDAG.h
+6 -15
View File
@@ -235,9 +235,10 @@ public:
/// If column is not in outputs, try to find it in nodes and insert back into outputs.
bool tryRestoreColumn(const std::string & column_name);
/// Find column in result. Remove it from outputs.
/// If columns is in inputs and has no dependent nodes, remove it from inputs too.
/// Return true if column was removed from inputs.
/// Removes the output `column_name`, and the chain of nodes it is computed by - the output node, its only child, and
/// so on, down to an input or a column - up to the first node that another node or output still uses.
/// Returns true if the whole chain is gone, so that the DAG no longer reads the input at its end.
/// Throws if a node on the chain has more than one child.
bool removeUnusedResult(const std::string & column_name);
/// Remove node with <node_name> from outputs.
@@ -282,6 +283,8 @@ public:
/// is safe to re-emit as a single Const COLUMN at the filter root - other outputs and
/// representation-observing parents elsewhere in the DAG are never touched
void foldFilterPredicateThroughMaterialize(const std::string & filter_column_name);
/// The same for the filter at `filter_output_position` of the outputs.
void foldFilterPredicateThroughMaterialize(size_t filter_output_position);
/// Collapse structurally equivalent subtrees (aliased duplicates, equal constants, functions with identical arguments)
/// outputs preserve their names via aliases when needed, dead nodes are pruned
@@ -374,18 +377,6 @@ public:
/// Same as above, but with an explicit list of input nodes instead of using the DAG's inputs.
static MatchedInputPositions matchInputNodesToHeader(const NodeRawConstPtrs & input_nodes, const Block & header);
/// Split output positions into DAG output indices and pass-through indices.
/// The output header is structured as [DAG outputs..., pass-through inputs...].
/// Positions below getOutputs().size() are DAG output indices;
/// positions at or above are pass-through indices (with the DAG output count subtracted),
/// can be used to index into the list of pass-through inputs from matchInputPositionsToHeader.
struct SplitOutputPositions
{
std::vector<size_t> dag_indices;
std::vector<size_t> passthrough_indices;
};
SplitOutputPositions splitOutputPositions(const std::vector<size_t> & output_positions) const;
using IntermediateExecutionResult = std::unordered_map<const Node *, ColumnWithTypeAndName>;
static ColumnsWithTypeAndName evaluatePartialResult(
IntermediateExecutionResult & node_to_column,
src/Processors/QueryPlan/ExpressionStep.cpp
+132 -79
View File
@@ -1,4 +1,5 @@
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Processors/QueryPlan/QueryPlanFormat.h>
#include <Processors/QueryPlan/Serialization.h>
#include <Processors/QueryPlan/QueryPlanStepRegistry.h>
@@ -11,6 +12,9 @@
#include <Common/JSONBuilder.h>
#include <Interpreters/ActionsDAG.h>
#include <span>
#include <unordered_set>
namespace DB
{
@@ -142,113 +146,162 @@ bool ExpressionStep::canRemoveUnusedColumns() const
return true;
}
ExpressionStep::RemoveUnusedColumnsResult ExpressionStep::removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs)
ActionsDAG::NodeRawConstPtrs ExpressionStep::UnneededColumnsPlan::neededDAGOutputs(const ActionsDAG::NodeRawConstPtrs & outputs) const
{
ActionsDAG::NodeRawConstPtrs needed;
needed.reserve(outputs.size() - unneeded_dag_position_count);
size_t next_unneeded = 0;
for (size_t position = 0; position < outputs.size(); ++position)
{
if (next_unneeded < unneeded_dag_position_count && unneeded_output_positions[next_unneeded] == position)
++next_unneeded;
else
needed.push_back(outputs[position]);
}
return needed;
}
std::vector<size_t> ExpressionStep::UnneededColumnsPlan::unneededInputPositions() const
{
std::vector<size_t> positions;
for (size_t position = 0; position < input_columns.size(); ++position)
{
const auto column = input_columns[position];
if (column == InputColumnUsage::ReadDropped || column == InputColumnUsage::PassesThroughDropped)
positions.push_back(position);
}
return positions;
}
ExpressionStep::UnneededColumnsPlan
ExpressionStep::analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
if (output_header == nullptr)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Output header is not set in ExpressionStep");
/// When extra columns were absorbed from a child step that cannot reduce its output,
/// prevent input removal to avoid re-creating the mismatch on subsequent optimization passes.
if (prevent_input_removal)
remove_inputs = false;
UnneededColumnsPlan plan;
plan.unneeded_output_positions = unneeded_output_positions;
const auto required_output_count = required_output_positions.size();
const auto & input_header = input_headers.front();
const auto actions_dag_input_count_before = actions_dag.getInputs().size();
/// The output header is structured as:
/// [DAG output 0, ..., DAG output N-1, pass-through input 0, pass-through input 1, ...]
/// Split required positions into DAG output indices and pass-through input indices.
auto [required_dag_indices, required_passthrough_indices]
= actions_dag.splitOutputPositions(required_output_positions);
/// so the positions below the number of DAG outputs are the DAG outputs nobody needs, and the rest
/// name pass-through columns, counting from that number. The positions are sorted, so the first
/// group is a prefix of them and the second is the remaining suffix.
chassert(std::ranges::is_sorted(unneeded_output_positions));
const auto dag_output_count = actions_dag.getOutputs().size();
const auto first_passthrough
= std::ranges::lower_bound(unneeded_output_positions, dag_output_count) - unneeded_output_positions.begin();
/// Build the list of pass-through input columns (input header columns not consumed by DAG inputs).
auto passthrough_input_header_positions = actions_dag.matchInputPositionsToHeader(*input_header).passthrough;
plan.unneeded_dag_position_count = first_passthrough;
/// Determine which pass-through inputs are required by the caller.
std::set<size_t> required_passthrough_header_positions;
for (size_t pt_idx : required_passthrough_indices)
const auto unneeded_passthrough_positions
= std::span{unneeded_output_positions}.subspan(first_passthrough);
/// What removeUnusedActions would keep once the outputs are pruned.
///
/// It also folds constants before it collects the nodes to keep, and folding clears the children of
/// a folded node, so those children are dropped. Stop at such a node to see the same.
auto roots = plan.neededDAGOutputs(actions_dag.getOutputs());
for (const auto & node : actions_dag.getNodes())
if (node.type == ActionsDAG::ActionType::ARRAY_JOIN)
roots.push_back(&node);
const auto is_folded_constant = [](const ActionsDAG::Node * node)
{
if (pt_idx >= passthrough_input_header_positions.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Required output position {} is out of range for pass-through inputs", pt_idx);
required_passthrough_header_positions.insert(passthrough_input_header_positions[pt_idx]);
}
return node->column && !node->children.empty();
};
const auto num_passthrough = passthrough_input_header_positions.size();
const auto has_to_remove_any_pass_through = num_passthrough > required_passthrough_indices.size();
const auto surviving_nodes = findReachableNodes(roots, is_folded_constant);
/// Keep only the required DAG output nodes.
auto & dag_outputs = actions_dag.getOutputs();
ActionsDAG::NodeRawConstPtrs new_dag_outputs;
new_dag_outputs.reserve(required_dag_indices.size());
for (size_t idx : required_dag_indices)
new_dag_outputs.push_back(dag_outputs[idx]);
dag_outputs = std::move(new_dag_outputs);
/// One entry per column of the input header: the input reading it, or nothing when it passes by.
const auto header_columns = mapHeaderColumnsToInputs(actions_dag.getInputs(), *input_header);
plan.input_columns.resize(header_columns.size());
auto updated_actions = actions_dag.removeUnusedActions(remove_inputs);
/// The caller's pass-through indices ascend, and so do the pass-through columns, so one walk over
/// the header pairs them up.
size_t passthrough_index = 0;
size_t next_unneeded_passthrough = 0;
/// If we cannot remove inputs but need to remove pass-through outputs,
/// convert unrequired pass-through inputs into DAG inputs so they stop being pass-throughs.
const auto has_to_add_input_to_actions = !remove_inputs && has_to_remove_any_pass_through;
if (has_to_add_input_to_actions)
/// A dropped column the child keeps is consumed by a new input, and inputs are paired with header columns by name,
/// in order. So a pass-through column after a needed one of the same name cannot be consumed, and is kept.
std::unordered_set<std::string_view> needed_passthrough_names;
std::vector<size_t> kept_unneeded_positions;
for (size_t position = 0; position < header_columns.size(); ++position)
{
for (size_t pt_pos : passthrough_input_header_positions)
if (!header_columns.passesThrough(position))
{
if (!required_passthrough_header_positions.contains(pt_pos))
{
const auto & col = input_header->getByPosition(pt_pos);
actions_dag.addInput(col.name, col.type);
}
const auto * input = actions_dag.getInputs()[header_columns.read_by[position]];
const bool is_needed = surviving_nodes.contains(input);
plan.input_columns[position] = is_needed ? InputColumnUsage::ReadNeeded : InputColumnUsage::ReadDropped;
continue;
}
updated_actions = true;
const auto output_position = dag_output_count + passthrough_index;
bool is_unneeded = next_unneeded_passthrough < unneeded_passthrough_positions.size()
&& unneeded_passthrough_positions[next_unneeded_passthrough] == output_position;
if (is_unneeded)
++next_unneeded_passthrough;
const auto & name = input_header->getByPosition(position).name;
if (is_unneeded && needed_passthrough_names.contains(name))
{
is_unneeded = false;
kept_unneeded_positions.push_back(output_position);
}
if (!is_unneeded)
needed_passthrough_names.insert(name);
plan.input_columns[position] = is_unneeded ? InputColumnUsage::PassesThroughDropped : InputColumnUsage::PassesThroughNeeded;
++passthrough_index;
}
if (!updated_actions && output_header->columns() == required_output_count)
return {};
if (next_unneeded_passthrough != unneeded_passthrough_positions.size())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Unneeded output position {} is out of range for the output header",
unneeded_passthrough_positions[next_unneeded_passthrough]);
if (actions_dag.getInputs().size() > getInputHeaders().at(0)->columns())
throw Exception(ErrorCodes::LOGICAL_ERROR, "There cannot be more inputs in the DAG than columns in the input header");
/// Both lists are sorted, and the kept positions are pass-through ones, so they are not in the prefix of DAG outputs.
std::erase_if(plan.unneeded_output_positions, [&](size_t position) { return std::ranges::binary_search(kept_unneeded_positions, position); });
const auto actions_dag_has_less_inputs = actions_dag.getInputs().size() < actions_dag_input_count_before;
const auto update_inputs = remove_inputs && (actions_dag_has_less_inputs || has_to_remove_any_pass_through);
if (update_inputs)
{
/// Build the set of required input header positions: DAG inputs that survived pruning + required pass-throughs.
auto matched_positions = actions_dag.matchInputPositionsToHeader(*input_header).matched;
std::set<size_t> required_input_positions(matched_positions.begin(), matched_positions.end());
/// Add required pass-through input positions.
for (size_t pt_pos : required_passthrough_header_positions)
required_input_positions.insert(pt_pos);
/// Build the result vector and update the input header.
std::vector<size_t> result_positions(required_input_positions.begin(), required_input_positions.end());
Block new_input_header{};
for (size_t pos : result_positions)
new_input_header.insert(input_header->getByPosition(pos));
SharedHeader new_shared_input_header = std::make_shared<const Block>(std::move(new_input_header));
updateInputHeader(std::move(new_shared_input_header), 0);
return {true, {std::move(result_positions)}, required_output_positions};
}
updateOutputHeader();
/// Outputs changed but inputs didn't.
return {true, {}, required_output_positions};
return plan;
}
bool ExpressionStep::canRemoveColumnsFromOutput() const
ExpressionStep::UnneededInputPositions ExpressionStep::getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
if (output_header == nullptr)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Output header is not set in ExpressionStep");
return canRemoveUnusedColumns();
return {analyzeUnneededColumns(unneeded_output_positions).unneededInputPositions()};
}
ExpressionStep::RemoveUnusedColumnsResult
ExpressionStep::removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs)
{
const auto plan = analyzeUnneededColumns(unneeded_output_positions);
const auto & pruned = inputs.at(0);
const auto input_header = input_headers.front();
actions_dag.getOutputs() = plan.neededDAGOutputs(actions_dag.getOutputs());
RemoveUnusedColumnsResult result;
result.dropped_output_positions = plan.unneeded_output_positions;
const bool dag_changed = alignInputsWithPrunedChild(actions_dag, plan.input_columns, *input_header, pruned);
result.step_changed = !result.dropped_output_positions.empty() || dag_changed
|| !blocksHaveEqualStructure(*input_header, *pruned.header);
if (result.step_changed)
updateInputHeader(pruned.header, 0);
return result;
}
QueryPlanStepPtr ExpressionStep::clone() const
{
return std::make_unique<ExpressionStep>(*this);
src/Processors/QueryPlan/ExpressionStep.h
+24 -9
View File
@@ -18,7 +18,6 @@ public:
ExpressionStep(const ExpressionStep & other)
: ITransformingStep(other)
, actions_dag(other.actions_dag.clone())
, prevent_input_removal(other.prevent_input_removal)
{}
String getName() const override { return "Expression"; }
@@ -45,20 +44,36 @@ public:
bool supportsDataflowStatisticsCollection() const override { return true; }
bool canRemoveUnusedColumns() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs) override;
bool canRemoveColumnsFromOutput() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs) override;
/// Prevent future input removal by removeUnusedColumns.
/// Used when extra columns were absorbed from a child step that cannot reduce its output
/// (e.g., ReadFromMergeTree with FINAL must keep sort key columns).
void setPreventInputRemoval() { prevent_input_removal = true; }
bool isInputRemovalPrevented() const { return prevent_input_removal; }
UnneededInputPositions getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const override;
private:
void updateOutputHeader() override;
/// Everything removeUnusedColumns needs to know, computed without touching the step.
/// Shared by removeUnusedColumns and getUnneededColumns so their answers cannot differ.
struct UnneededColumnsPlan
{
/// One entry per column of the input header, in header order.
std::vector<InputColumnUsage> input_columns;
/// The positions nobody needs that the step drops, which are the outputs that go away. A pass-through
/// column after a needed one of the same name is kept, even if nobody needs it.
std::vector<size_t> unneeded_output_positions;
/// How many of them index the DAG's outputs. The output header holds the DAG outputs first and the
/// pass-through columns after, so the unneeded DAG outputs are a prefix of `unneeded_output_positions`.
size_t unneeded_dag_position_count = 0;
/// The DAG outputs that remain, in their order.
ActionsDAG::NodeRawConstPtrs neededDAGOutputs(const ActionsDAG::NodeRawConstPtrs & outputs) const;
/// What the step does not need of its child: the columns it neither reads nor passes on.
std::vector<size_t> unneededInputPositions() const;
};
UnneededColumnsPlan analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const;
ActionsDAG actions_dag;
bool prevent_input_removal = false;
};
}
src/Processors/QueryPlan/FilterStep.cpp
+198 -133
View File
@@ -1,10 +1,10 @@
#include <Processors/QueryPlan/FilterStep.h>
#include <algorithm>
#include <limits>
#include <optional>
#include <ranges>
#include <set>
#include <stack>
#include <unordered_set>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeNullable.h>
@@ -13,6 +13,7 @@
#include <IO/Operators.h>
#include <Interpreters/ActionsDAG.h>
#include <Interpreters/ExpressionActions.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Processors/QueryPlan/QueryPlanFormat.h>
#include <Processors/QueryPlan/QueryPlanStepRegistry.h>
#include <Processors/QueryPlan/Serialization.h>
@@ -48,20 +49,18 @@ static ITransformingStep::Traits getTraits()
};
}
FilterDAGOutputPruningResult pruneFilterDAGOutputsByPosition(
ActionsDAG & dag,
FilterStep::UnneededColumnsPlan FilterStep::analyzeUnneededColumns(
const ActionsDAG & dag,
const String & filter_column_name,
bool & remove_filter_column,
bool remove_filter_column,
const Block & input_header,
const std::vector<size_t> & required_output_positions,
bool remove_inputs)
const std::vector<size_t> & unneeded_output_positions)
{
FilterDAGOutputPruningResult result;
UnneededColumnsPlan plan;
plan.remove_filter_column = remove_filter_column;
const bool was_remove_filter_column = remove_filter_column;
const auto & old_outputs = dag.getOutputs();
const size_t old_dag_outputs_size = old_outputs.size();
const auto actions_dag_input_count_before = dag.getInputs().size();
/// The pre-erase output header (from ActionsDAG::updateHeader) is:
/// [DAG output 0, ..., DAG output N-1, pass-through input 0, ...]
@@ -86,107 +85,184 @@ FilterDAGOutputPruningResult pruneFilterDAGOutputsByPosition(
filter_column_name,
fmt::join(dag.getNames(), ", "));
/// Map positions from the final (post-erase) header to the pre-erase header.
/// remove_filter_column is captured by value because the mapping depends on the original value of the flag, not on
/// whether the filter column is still present at the time of mapping.
auto map_to_pre_erase_pos = [filter_col_pre_erase_pos, was_remove_filter_column](size_t pos) -> size_t
{
if (!was_remove_filter_column)
return pos;
return pos >= filter_col_pre_erase_pos ? pos + 1 : pos;
};
/// Map positions from post-erase to pre-erase layout, then split into DAG vs pass-through.
/// Map positions from the final (post-erase) header to the pre-erase header. The mapping depends on
/// the incoming value of the flag, not on the value the pruning settles on. An erased filter column is not
/// in the final header, so no position maps to it.
std::vector<size_t> pre_erase_positions;
pre_erase_positions.reserve(required_output_positions.size());
for (size_t pos : required_output_positions)
pre_erase_positions.push_back(map_to_pre_erase_pos(pos));
pre_erase_positions.reserve(unneeded_output_positions.size());
for (size_t pos : unneeded_output_positions)
pre_erase_positions.push_back(remove_filter_column && pos >= filter_col_pre_erase_pos ? pos + 1 : pos);
auto [required_dag_indices, required_passthrough_indices] = dag.splitOutputPositions(pre_erase_positions);
/// The positions ascend, so the unneeded DAG outputs are a prefix of them, and the rest are pass-through columns.
const size_t first_unneeded_passthrough
= std::ranges::lower_bound(pre_erase_positions, old_dag_outputs_size) - pre_erase_positions.begin();
std::vector<size_t> unneeded_dag_indices(pre_erase_positions.begin(), pre_erase_positions.begin() + first_unneeded_passthrough);
/// Build the list of pass-through input columns.
const auto passthrough_input_header_positions = dag.matchInputPositionsToHeader(input_header).passthrough;
/// Nobody reads the filter column any more, so it can be removed from the header.
if (!plan.remove_filter_column && std::ranges::binary_search(unneeded_dag_indices, filter_col_pre_erase_pos))
plan.remove_filter_column = true;
std::set<size_t> required_passthrough_input_header_positions;
for (size_t passthrough_index : required_passthrough_indices)
/// The filter column is dropped, so its `materialize` wrapper is no longer observable, and
/// `applyToOutputs` folds it away. That changes what survives the pruning, so the rest of
/// this looks at the DAG as it is going to be. This includes filters that were already marked for
/// removal by `ReadFromMergeTree`. The fold only looks through `materialize` and aliases, so without a
/// `materialize` in the predicate there is nothing to fold.
std::optional<ActionsDAG> folded_dag;
if (plan.remove_filter_column)
{
if (passthrough_index >= passthrough_input_header_positions.size())
throw Exception(
ErrorCodes::LOGICAL_ERROR, "Required output position {} is out of range for pass-through inputs", passthrough_index);
required_passthrough_input_header_positions.insert(passthrough_input_header_positions[passthrough_index]);
}
const auto has_to_remove_any_pass_through
= passthrough_input_header_positions.size() > required_passthrough_input_header_positions.size();
std::set<size_t> required_dag_index_set(required_dag_indices.begin(), required_dag_indices.end());
/// Check if the filter column is required by the caller. If not, we can remove it.
if (!remove_filter_column && !required_dag_index_set.contains(filter_col_pre_erase_pos))
{
remove_filter_column = true;
}
/// The filter column is dropped, so its `materialize` wrapper is no longer observable. This
/// includes filters that were already marked for removal by `ReadFromMergeTree`.
if (remove_filter_column)
dag.foldFilterPredicateThroughMaterialize(filter_column_name);
required_dag_index_set.insert(filter_col_pre_erase_pos);
/// Keep only the required DAG output nodes, plus always keep the filter column.
ActionsDAG::NodeRawConstPtrs new_dag_outputs;
new_dag_outputs.reserve(required_dag_index_set.size());
for (size_t i = 0; i < old_dag_outputs_size; ++i)
{
if (required_dag_index_set.contains(i))
new_dag_outputs.push_back(old_outputs[i]);
}
auto & dag_outputs = dag.getOutputs();
if (new_dag_outputs.size() != dag_outputs.size())
result.changed = true;
dag_outputs = std::move(new_dag_outputs);
if (was_remove_filter_column != remove_filter_column)
result.changed = true;
if (dag.removeUnusedActions(remove_inputs))
result.changed = true;
if (!remove_inputs && has_to_remove_any_pass_through)
{
for (size_t passthrough_input_header_position : passthrough_input_header_positions)
const auto predicate_nodes = findReachableNodes({old_outputs[filter_col_pre_erase_pos]});
const bool has_materialize = std::ranges::any_of(predicate_nodes, [](const auto * node)
{
if (!required_passthrough_input_header_positions.contains(passthrough_input_header_position))
{
const auto & column = input_header.getByPosition(passthrough_input_header_position);
dag.addInput(column);
}
return node->type == ActionsDAG::ActionType::FUNCTION && node->function_base->getName() == "materialize";
});
if (has_materialize)
{
folded_dag = dag.clone();
folded_dag->foldFilterPredicateThroughMaterialize(filter_col_pre_erase_pos);
/// The fold only ever replaces the filter output by a constant. The predicate has a `materialize`
/// below it, so it was not a constant before.
chassert(old_outputs[filter_col_pre_erase_pos]->type != ActionsDAG::ActionType::COLUMN);
plan.fold_filter_predicate = folded_dag->getOutputs()[filter_col_pre_erase_pos]->type == ActionsDAG::ActionType::COLUMN;
if (!plan.fold_filter_predicate)
folded_dag.reset();
}
result.changed = true;
}
const auto & analyzed_dag = folded_dag ? *folded_dag : dag;
const auto & analyzed_outputs = analyzed_dag.getOutputs();
if (remove_inputs)
/// The filter column is needed to filter, whether or not anyone reads it.
std::erase(unneeded_dag_indices, filter_col_pre_erase_pos);
plan.unneeded_dag_positions = std::move(unneeded_dag_indices);
plan.filter_output_position = filter_col_pre_erase_pos;
plan.changes_output_header = !plan.unneeded_dag_positions.empty() || remove_filter_column != plan.remove_filter_column;
/// What removeUnusedActions would keep once the outputs are pruned. It folds constants before it
/// collects the nodes to keep, and folding clears the children of a folded node, so stop at such a
/// node to see the same set. ARRAY_JOIN is always a root there.
auto roots = plan.neededDAGOutputs(analyzed_outputs);
for (const auto & node : analyzed_dag.getNodes())
if (node.type == ActionsDAG::ActionType::ARRAY_JOIN)
roots.push_back(&node);
const auto is_folded_constant = [](const ActionsDAG::Node * node) { return node->column && !node->children.empty(); };
const auto surviving_nodes = findReachableNodes(roots, is_folded_constant);
/// One entry per column of the input header: the input reading it, or nothing when it passes by.
/// A clone keeps the inputs in their order, so the folded DAG reads the same header positions.
const auto header_columns = mapHeaderColumnsToInputs(analyzed_dag.getInputs(), input_header);
plan.input_columns.resize(header_columns.size());
/// The pass-through columns follow the DAG outputs in the pre-erase header, in their order, so one walk over
/// the input header pairs them up with the unneeded positions.
size_t passthrough_position = old_dag_outputs_size;
size_t next_unneeded_passthrough = first_unneeded_passthrough;
/// A dropped column the child keeps is consumed by a new input, and inputs are paired with header columns by name,
/// in order. So a pass-through column after a needed one of the same name cannot be consumed, and is kept.
std::unordered_set<std::string_view> needed_passthrough_names;
std::vector<size_t> kept_unneeded_positions;
for (size_t position = 0; position < header_columns.size(); ++position)
{
auto required_input_positions = dag.matchInputPositionsToHeader(input_header).matched;
required_input_positions.insert(
required_input_positions.end(),
required_passthrough_input_header_positions.begin(),
required_passthrough_input_header_positions.end());
if (!header_columns.passesThrough(position))
{
/// Every input reads a header position of its own, so the column it reads is needed exactly when
/// the input survives.
const auto * input = analyzed_dag.getInputs()[header_columns.read_by[position]];
plan.input_columns[position] = surviving_nodes.contains(input) ? InputColumnUsage::ReadNeeded : InputColumnUsage::ReadDropped;
continue;
}
std::sort(required_input_positions.begin(), required_input_positions.end());
result.required_input_positions = std::move(required_input_positions);
result.input_positions_changed = dag.getInputs().size() != actions_dag_input_count_before || has_to_remove_any_pass_through;
bool is_unneeded = next_unneeded_passthrough < pre_erase_positions.size()
&& pre_erase_positions[next_unneeded_passthrough] == passthrough_position;
if (is_unneeded)
++next_unneeded_passthrough;
if (result.input_positions_changed)
result.changed = true;
/// The filter column comes before the pass-through columns, so the caller counts them one lower when it is erased.
const auto & name = input_header.getByPosition(position).name;
if (is_unneeded && needed_passthrough_names.contains(name))
{
is_unneeded = false;
kept_unneeded_positions.push_back(remove_filter_column ? passthrough_position - 1 : passthrough_position);
}
if (!is_unneeded)
needed_passthrough_names.insert(name);
plan.input_columns[position] = is_unneeded ? InputColumnUsage::PassesThroughDropped : InputColumnUsage::PassesThroughNeeded;
++passthrough_position;
}
if (next_unneeded_passthrough != pre_erase_positions.size())
throw Exception(
ErrorCodes::LOGICAL_ERROR, "Unneeded output position {} is out of range for pass-through inputs",
pre_erase_positions[next_unneeded_passthrough]);
plan.dropped_output_positions = unneeded_output_positions;
std::erase_if(plan.dropped_output_positions, [&](size_t position) { return std::ranges::binary_search(kept_unneeded_positions, position); });
return plan;
}
ActionsDAG::NodeRawConstPtrs FilterStep::UnneededColumnsPlan::neededDAGOutputs(const ActionsDAG::NodeRawConstPtrs & outputs) const
{
ActionsDAG::NodeRawConstPtrs needed;
needed.reserve(outputs.size() - unneeded_dag_positions.size());
size_t next_unneeded = 0;
for (size_t position = 0; position < outputs.size(); ++position)
{
if (next_unneeded < unneeded_dag_positions.size() && unneeded_dag_positions[next_unneeded] == position)
++next_unneeded;
else
needed.push_back(outputs[position]);
}
return needed;
}
std::vector<size_t> FilterStep::UnneededColumnsPlan::unneededInputPositions() const
{
std::vector<size_t> positions;
for (size_t position = 0; position < input_columns.size(); ++position)
if (input_columns[position] == InputColumnUsage::ReadDropped || input_columns[position] == InputColumnUsage::PassesThroughDropped)
positions.push_back(position);
return positions;
}
FilterDAGOutputPruningResult FilterStep::UnneededColumnsPlan::toResult(bool removed_any_action) const
{
FilterDAGOutputPruningResult result;
result.unneeded_input_positions = unneededInputPositions();
result.changed = changes_output_header || fold_filter_predicate || removed_any_action || !result.unneeded_input_positions.empty();
return result;
}
void FilterStep::UnneededColumnsPlan::applyToOutputs(ActionsDAG & dag, bool & remove_filter_column_) const
{
if (fold_filter_predicate)
dag.foldFilterPredicateThroughMaterialize(filter_output_position);
dag.getOutputs() = neededDAGOutputs(dag.getOutputs());
remove_filter_column_ = remove_filter_column;
}
FilterDAGOutputPruningResult FilterStep::pruneDAGOutputsByPosition(
ActionsDAG & dag,
const String & filter_column_name,
bool & remove_filter_column,
const Block & input_header,
const std::vector<size_t> & unneeded_output_positions)
{
const auto plan = analyzeUnneededColumns(dag, filter_column_name, remove_filter_column, input_header, unneeded_output_positions);
plan.applyToOutputs(dag, remove_filter_column);
const bool removed_any_action = dag.removeUnusedActions();
return plan.toResult(removed_any_action);
}
static bool isTrivialSubtree(const ActionsDAG::Node * node)
{
while (node->type == ActionsDAG::ActionType::ALIAS)
@@ -286,7 +362,7 @@ FilterStep::FilterStep(
/// The filter column is dropped from the output, so only the predicate's value is observable and it
/// is safe to fold a `materialize`-wrapped constant away (#78166). Doing it here covers every way a
/// dropped-filter step comes to be - in particular the fresh steps `tryPushDownFilter` creates, which
/// neither `tryMergeExpressions` nor `pruneFilterDAGOutputsByPosition` ever sees.
/// neither `tryMergeExpressions` nor `pruneDAGOutputsByPosition` ever sees.
if (remove_filter_column)
actions_dag.foldFilterPredicateThroughMaterialize(filter_column_name);
@@ -474,57 +550,46 @@ bool FilterStep::canRemoveUnusedColumns() const
return true;
}
FilterStep::RemoveUnusedColumnsResult FilterStep::removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs)
FilterStep::UnneededColumnsPlan FilterStep::analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
if (output_header == nullptr)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Output header is not set in FilterStep");
/// When extra columns were absorbed from a child step that cannot reduce its output,
/// prevent input removal to avoid re-creating the mismatch on subsequent optimization passes.
if (prevent_input_removal)
remove_inputs = false;
chassert(
actions_dag.getInputs().size() <= getInputHeaders().at(0)->columns()
&& "There cannot be more DAG inputs than columns in the input header");
const auto & input_header = input_headers.front();
auto pruning_result = pruneFilterDAGOutputsByPosition(
actions_dag, filter_column_name, remove_filter_column, *input_header, required_output_positions, remove_inputs);
if (!pruning_result.changed && output_header->columns() == required_output_positions.size())
return {};
if (actions_dag.getInputs().size() > getInputHeaders().at(0)->columns())
throw Exception(ErrorCodes::LOGICAL_ERROR, "There cannot be more inputs in the DAG than columns in the input header");
if (pruning_result.input_positions_changed)
{
Block new_input_header{};
for (size_t pos : pruning_result.required_input_positions)
new_input_header.insert(input_header->getByPosition(pos));
SharedHeader new_shared_input_header = std::make_shared<const Block>(std::move(new_input_header));
updateInputHeader(std::move(new_shared_input_header), 0);
return {true, {std::move(pruning_result.required_input_positions)}, required_output_positions};
}
updateOutputHeader();
return {true, {}, required_output_positions};
return analyzeUnneededColumns(actions_dag, filter_column_name, remove_filter_column, *input_headers.front(), unneeded_output_positions);
}
bool FilterStep::canRemoveColumnsFromOutput() const
FilterStep::UnneededInputPositions FilterStep::getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
if (output_header == nullptr)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Output header is not set in FilterStep");
if (!remove_filter_column && output_header->columns() == 1)
return false;
return canRemoveUnusedColumns();
return {analyzeUnneededColumns(unneeded_output_positions).unneededInputPositions()};
}
FilterStep::RemoveUnusedColumnsResult
FilterStep::removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs)
{
const auto plan = analyzeUnneededColumns(unneeded_output_positions);
const auto & pruned = inputs.at(0);
const auto input_header = input_headers.front();
plan.applyToOutputs(actions_dag, remove_filter_column);
RemoveUnusedColumnsResult result;
result.dropped_output_positions = plan.dropped_output_positions;
const bool dag_changed = alignInputsWithPrunedChild(actions_dag, plan.input_columns, *input_header, pruned);
result.step_changed = plan.changes_output_header || plan.fold_filter_predicate || dag_changed
|| !blocksHaveEqualStructure(*input_header, *pruned.header);
if (result.step_changed)
updateInputHeader(pruned.header, 0);
return result;
}
QueryPlanStepPtr FilterStep::clone() const
{
return std::make_unique<FilterStep>(*this);
src/Processors/QueryPlan/FilterStep.h
+66 -17
View File
@@ -6,22 +6,14 @@
namespace DB
{
/// What `FilterStep::pruneDAGOutputsByPosition` did.
struct FilterDAGOutputPruningResult
{
bool changed = false;
bool input_positions_changed = false;
std::vector<size_t> required_input_positions;
/// The positions of the input header that the remaining outputs and the filter do not need, sorted.
std::vector<size_t> unneeded_input_positions;
};
/// Prune filter DAG outputs by position and return the input positions needed to compute the remaining outputs and filter.
FilterDAGOutputPruningResult pruneFilterDAGOutputsByPosition(
ActionsDAG & dag,
const String & filter_column_name,
bool & remove_filter_column,
const Block & input_header,
const std::vector<size_t> & required_output_positions,
bool remove_inputs);
/// Implements WHERE, HAVING operations. See FilterTransform.
class FilterStep : public ITransformingStep
{
@@ -37,7 +29,6 @@ public:
, actions_dag(other.actions_dag.clone())
, filter_column_name(other.filter_column_name)
, remove_filter_column(other.remove_filter_column)
, prevent_input_removal(other.prevent_input_removal)
, condition(other.condition)
{}
@@ -70,21 +61,79 @@ public:
void decorrelateActions() { actions_dag.decorrelate(); }
bool canRemoveUnusedColumns() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs) override;
bool canRemoveColumnsFromOutput() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs) override;
void setPreventInputRemoval() { prevent_input_removal = true; }
bool isInputRemovalPrevented() const { return prevent_input_removal; }
UnneededInputPositions getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const override;
bool supportsDataflowStatisticsCollection() const override { return true; }
/// Removes the outputs of a filter DAG at `unneeded_output_positions` of its output header, and says which input
/// positions the remaining outputs and the filter do not need. For a filter that is not a step of its own, as
/// PREWHERE and the row policy filter of `ReadFromMergeTree` are.
static FilterDAGOutputPruningResult pruneDAGOutputsByPosition(
ActionsDAG & dag,
const String & filter_column_name,
bool & remove_filter_column,
const Block & input_header,
const std::vector<size_t> & unneeded_output_positions);
private:
void updateOutputHeader() override;
/// Everything removeUnusedColumns needs to know, computed without touching the DAG.
/// Shared by removeUnusedColumns, getUnneededColumns and pruneDAGOutputsByPosition so their answers cannot differ.
struct UnneededColumnsPlan
{
/// One entry per column of the input header, in header order.
std::vector<InputColumnUsage> input_columns;
/// The DAG outputs nobody needs, as positions in `getOutputs` before any is removed, sorted.
/// Never contains the filter column: it is needed to filter, whether or not anyone reads it.
/// Unlike for the other steps, these are not just the caller's positions.
/// The header the caller counts in may have the filter column erased from it,
/// so the caller's positions are shifted back over that column first.
std::vector<size_t> unneeded_dag_positions;
/// The caller's positions the step drops: a pass-through column after a needed one of the same name is kept,
/// even if nobody needs it.
std::vector<size_t> dropped_output_positions;
/// The position of the filter column in the DAG's outputs, before any is removed.
size_t filter_output_position = 0;
/// Whether the filter column is removed from the output header after the pruning: it already was, or nobody
/// reads it any more.
bool remove_filter_column = false;
/// Whether the output header changes.
bool changes_output_header = false;
/// Whether the filter predicate folds to a constant through `materialize` once the filter column is dropped;
/// the rest of the plan is worked out on the folded DAG.
bool fold_filter_predicate = false;
/// The DAG outputs that remain, in their order.
ActionsDAG::NodeRawConstPtrs neededDAGOutputs(const ActionsDAG::NodeRawConstPtrs & outputs) const;
/// The part of the pruning that concerns the outputs: the fold of the predicate, the outputs that remain, and
/// the filter column flag.
void applyToOutputs(ActionsDAG & dag, bool & remove_filter_column_) const;
/// The positions of the input header nothing needs any more: the columns neither read nor passed on.
std::vector<size_t> unneededInputPositions() const;
/// What the pruning did, once the outputs are pruned and removeUnusedActions has said whether it erased a node.
FilterDAGOutputPruningResult toResult(bool removed_any_action) const;
};
static UnneededColumnsPlan analyzeUnneededColumns(
const ActionsDAG & dag,
const String & filter_column_name,
bool remove_filter_column,
const Block & input_header,
const std::vector<size_t> & unneeded_output_positions);
UnneededColumnsPlan analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const;
ActionsDAG actions_dag;
String filter_column_name;
bool remove_filter_column;
bool prevent_input_removal = false;
std::optional<std::pair<UInt64, String>> condition; /// for query condition cache
};
src/Processors/QueryPlan/IQueryPlanStep.cpp
+65 -3
View File
@@ -7,10 +7,12 @@
#include <Processors/IProcessor.h>
#include <Processors/Port.h>
#include <Processors/QueryPlan/IQueryPlanStep.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Processors/QueryPlan/QueryPlanFormat.h>
#include <fmt/format.h>
#include <algorithm>
#include <unordered_map>
#include <unordered_set>
namespace DB
{
@@ -50,16 +52,76 @@ void IQueryPlanStep::setRuntimeDataflowStatisticsCacheUpdater(RuntimeDataflowSta
dataflow_cache_updater = std::move(updater);
}
IQueryPlanStep::RemoveUnusedColumnsResult IQueryPlanStep::removeUnusedColumns(const std::vector<size_t> & /*required_output_positions*/, bool /*remove_inputs*/)
IQueryPlanStep::PrunedInput IQueryPlanStep::PrunedInput::unchanged(const SharedHeader & header)
{
return {.dropped_positions = {}, .header = header};
}
bool IQueryPlanStep::alignInputsWithPrunedChild(
ActionsDAG & dag, const std::vector<InputColumnUsage> & usages, const Block & old_header, const PrunedInput & pruned)
{
std::vector<bool> kept(old_header.columns(), true);
for (size_t position : pruned.dropped_positions)
kept.at(position) = false;
const auto header_columns = mapHeaderColumnsToInputs(dag.getInputs(), old_header);
/// Inputs are paired with header columns by name, in order, so a new input consumes the first column of its name that
/// no input reads. A column that passes through before it would be consumed in its place.
std::unordered_set<std::string_view> passthrough_names;
const auto consume = [&](ColumnsWithTypeAndName & to_consume, const ColumnWithTypeAndName & column)
{
if (passthrough_names.contains(column.name))
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Cannot consume column {}: a column of the same name passes through before it", column.name);
to_consume.push_back(column);
};
std::unordered_set<const ActionsDAG::Node *> kept_inputs;
ColumnsWithTypeAndName to_consume;
for (size_t position = 0; position < old_header.columns(); ++position)
{
const auto usage = usages.at(position);
if (!kept[position])
{
if (usage == InputColumnUsage::ReadNeeded || usage == InputColumnUsage::PassesThroughNeeded)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"The child dropped column {}, which the step needs", old_header.getByPosition(position).name);
continue;
}
if (!header_columns.passesThrough(position))
kept_inputs.insert(dag.getInputs()[header_columns.read_by[position]]);
else if (usage == InputColumnUsage::PassesThroughDropped)
consume(to_consume, old_header.getByPosition(position));
else
passthrough_names.insert(old_header.getByPosition(position).name);
}
const size_t kept_count = old_header.columns() - pruned.dropped_positions.size();
for (size_t position = kept_count; position < pruned.header->columns(); ++position)
consume(to_consume, pruned.header->getByPosition(position));
bool changed = dag.removeUnusedActions(kept_inputs);
for (const auto & column : to_consume)
dag.addInput(column.name, column.type);
return changed || !to_consume.empty();
}
IQueryPlanStep::RemoveUnusedColumnsResult
IQueryPlanStep::removeUnusedColumns(const std::vector<size_t> & /*unneeded_output_positions*/, const std::vector<PrunedInput> & /*inputs*/)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeUnusedColumns is not implemented for step {}", getName());
}
bool IQueryPlanStep::canRemoveColumnsFromOutput() const
IQueryPlanStep::UnneededInputPositions
IQueryPlanStep::getUnneededColumns(const std::vector<size_t> & /*unneeded_output_positions*/) const
{
return false;
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getUnneededColumns is not implemented for step {}", getName());
}
bool IQueryPlanStep::hasCorrelatedExpressions() const
{
return false;
src/Processors/QueryPlan/IQueryPlanStep.h
+64 -29
View File
@@ -143,43 +143,69 @@ public:
void setRuntimeDataflowStatisticsCacheUpdater(RuntimeDataflowStatisticsCacheUpdaterPtr updater);
/// Returns true if the step has implemented removeUnusedColumns.
/// Removing unused columns. Every list of positions below is a sorted list of positions in a header.
/// Returns true if the step can take part in removing unused columns: it implements removeUnusedColumns,
/// and getUnneededColumns unless it has no children to ask about.
virtual bool canRemoveUnusedColumns() const { return false; }
/// What removeUnusedColumns did. The default is the answer when nothing changes.
struct RemoveUnusedColumnsResult
{
/// Sentinel for kept_output_positions entries that were added
/// (e.g., a dummy column in JoinStepLogical) and have no original output position.
static constexpr size_t NEWLY_ADDED_COLUMN_POSITION = std::numeric_limits<size_t>::max();
/// Whether the step itself changed: its expressions, its input headers, or the columns it outputs.
bool step_changed = false;
/// Whether the step was actually modified.
/// Needed to distinguish "removed all outputs" from "nothing changed",
/// since both can have empty required_input_positions and kept_output_positions.
bool changed = false;
/// Required input positions per child (outer index = child_id).
/// Empty outside vector means no inputs were changed.
/// Empty inside vector means the step doesn't require any inputs from the child.
std::vector<std::vector<size_t>> required_input_positions;
/// Which original output positions survived, in order.
/// Only meaningful if `changed` is true, otherwise it shouldn't be used.
/// Maps new output position to the original output position.
/// Entries with NEWLY_ADDED_COLUMN_POSITION indicate columns that weren't present in the original header.
std::vector<size_t> kept_output_positions;
/// The positions of the step's former output header that went away.
/// They are the subset of the given unneeded positions that the step can remove.
/// For example, a `FINAL` read keeps the columns of its sorting key.
/// The new output header has the remaining columns first, in their former order,
/// and may append columns of its own after them, such as the dummy column a join adds.
/// Appended columns are not in `dropped_output_positions`; only the header shows them.
std::vector<size_t> dropped_output_positions;
};
/// Removes the unnecessary inputs and outputs from the step based on required_output_positions.
/// required_output_positions must be a sorted vector of indices into the step's current output header.
/// Each position uniquely identifies a column even when names are duplicated.
/// It is guaranteed that the output header of the step will contain all columns at those positions
/// and might contain some other columns too.
/// Can be used only if canRemoveUnusedColumns returns true.
/// The order of the remaining outputs must be preserved.
virtual RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & /*required_output_positions*/, bool /*remove_inputs*/);
/// Per child: the positions of the child's current output header the step does not need.
/// Always one entry per child; an empty entry means the step needs every column of that child.
using UnneededInputPositions = std::vector<std::vector<size_t>>;
/// Returns true if the step can remove any columns from the output using removeUnusedColumns.
virtual bool canRemoveColumnsFromOutput() const;
/// What one column of a step's input header is to the step once the unused columns are gone.
enum class InputColumnUsage : uint8_t
{
ReadNeeded, /// an input reads it, and what that input feeds is still needed
ReadDropped, /// an input reads it, and nothing needs that input any more
PassesThroughNeeded, /// no input reads it, and the caller asked for the column itself
PassesThroughDropped, /// no input reads it, and nobody asked for it
};
/// What a child produces once its own unused columns are gone:
/// its `dropped_output_positions` and its new output header.
/// The header has the remaining columns first, in their order, and may append new ones after them,
/// such as the dummy column a join adds.
/// The positions say what the child no longer produces, not what it was asked for:
/// a child that cannot drop columns drops none, and the step consumes the ones it does not need.
struct PrunedInput
{
std::vector<size_t> dropped_positions;
SharedHeader header;
/// A child that did not change.
static PrunedInput unchanged(const SharedHeader & header);
};
/// Removes what the step no longer needs once nobody needs the columns at `unneeded_output_positions`
/// of its current output header.
/// `inputs` has one `PrunedInput` per child: the children are pruned first, since columns are dropped bottom-up.
/// A column a child keeps that the step does not need is consumed by the step.
/// A column the child dropped must be one the step does not need.
/// The order of the remaining outputs is preserved.
/// Can be used only if canRemoveUnusedColumns returns true.
virtual RemoveUnusedColumnsResult removeUnusedColumns(
const std::vector<size_t> & /*unneeded_output_positions*/, const std::vector<PrunedInput> & /*inputs*/);
/// What removeUnusedColumns would not need of each child once nobody needs these outputs.
/// The children are to be pruned by that before the step itself is.
/// Can be used only if canRemoveUnusedColumns returns true, for a step with children.
virtual UnneededInputPositions getUnneededColumns(const std::vector<size_t> & /*unneeded_output_positions*/) const;
/// Different Steps have different stages of execution.
/// For example JoinStep has build and probe stages.
@@ -196,6 +222,15 @@ public:
virtual StepAnalysisReport getAnalysisReport(StepProcessors /*step_processors*/) const { return {}; }
protected:
/// For a step with one child and one expression: brings the inputs of `dag`, whose outputs are pruned
/// already, in line with the child's new header. An input reading a column the child keeps stays, also
/// where nothing needs it any more, and one reading a column the child dropped goes. A column the child
/// keeps or appends that the step neither reads nor passes on is consumed by a new input, so that it
/// stops here. `usages` says what each column of `old_header` is to the step. Returns whether anything
/// changed.
static bool alignInputsWithPrunedChild(
ActionsDAG & dag, const std::vector<InputColumnUsage> & usages, const Block & old_header, const PrunedInput & pruned);
virtual void updateOutputHeader() = 0;
SharedHeaders input_headers;
src/Processors/QueryPlan/JoinStepLogical.cpp
+213 -117
View File
@@ -1,6 +1,7 @@
#include <Columns/ColumnConst.h>
#include <DataTypes/IDataType.h>
#include <Processors/QueryPlan/JoinStepLogical.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Processors/QueryPlan/QueryPlanFormat.h>
#include <base/scope_guard.h>
@@ -439,109 +440,240 @@ bool JoinStepLogical::canRemoveUnusedColumns() const
&& !has_duplicated_condition_input(*input_headers.at(1), right_condition_input_names);
}
JoinStepLogical::RemoveUnusedColumnsResult JoinStepLogical::removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs)
JoinStepLogical::UnneededColumnsPlan
JoinStepLogical::analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
auto & actions_dag = *expression_actions.getActionsDAG();
const size_t original_input_count = actions_dag.getInputs().size();
ActionsDAG::NodeRawConstPtrs required_nodes;
ActionsDAG::NodeRawConstPtrs new_actions_after_join = actions_after_join;
const auto & actions_dag = *expression_actions.getActionsDAG();
UnneededColumnsPlan plan;
/// For JoinStepLogical, the output header maps directly to DAG outputs (no pass-throughs).
/// Build a set of required DAG output positions.
std::set<size_t> required_positions_set(required_output_positions.begin(), required_output_positions.end());
const std::set<size_t> unneeded_positions_set(unneeded_output_positions.begin(), unneeded_output_positions.end());
/// Track which original output positions survive (required + non-removable like dummy).
std::vector<size_t> kept_output_positions;
kept_output_positions.reserve(required_output_positions.size());
bool removed_any_output = false;
ActionsDAG::NodeRawConstPtrs kept_output_nodes;
const auto & dag_outputs = actions_dag.getOutputs();
for (size_t i = 0; i < dag_outputs.size(); ++i)
if (!unneeded_positions_set.empty() && *unneeded_positions_set.rbegin() >= dag_outputs.size())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Unneeded output position {} is out of range for the {} outputs of the join",
*unneeded_positions_set.rbegin(), dag_outputs.size());
for (size_t position = 0; position < dag_outputs.size(); ++position)
{
const auto * output_node = dag_outputs[i];
if (required_positions_set.contains(i))
{
required_nodes.push_back(output_node);
kept_output_positions.push_back(i);
}
else if (!isDummyColumnOfThisStep(output_node))
{
/// Do not remove join_dummy_result from the outputs, because it was added to ensure at least one output column
removed_any_output = true;
new_actions_after_join.erase(
std::remove(new_actions_after_join.begin(), new_actions_after_join.end(), output_node), new_actions_after_join.end());
}
const auto * output_node = dag_outputs[position];
/// Do not remove join_dummy_result from the outputs, because it was added to ensure at least one
/// output column, so it is kept even when it is not required.
if (!unneeded_positions_set.contains(position) || isDummyColumnOfThisStep(output_node))
kept_output_nodes.push_back(output_node);
else
plan.dropped_output_positions.push_back(position);
}
/// Nothing is left to output, so a dummy column takes the place of the removed ones.
plan.adds_dummy_output = kept_output_nodes.empty();
/// The join conditions have to be computed whether or not anything reads them.
for (const auto & join_action : join_operator.expression)
plan.extra_pruning_roots.push_back(join_action.getNode());
for (const auto & join_action : join_operator.residual_filter)
plan.extra_pruning_roots.push_back(join_action.getNode());
bool has_required_input_from_left = false;
bool has_required_input_from_right = false;
for (const auto * required_node : std::array{&kept_output_nodes, &plan.extra_pruning_roots})
{
for (const auto * node : *required_node)
{
/// Dummy column kept even though not required.
required_nodes.push_back(output_node);
kept_output_positions.push_back(i);
const auto source_relations = JoinActionRef(node, expression_actions).getSourceRelations();
has_required_input_from_left |= source_relations.test(0);
has_required_input_from_right |= source_relations.test(1);
}
}
if (required_nodes.empty())
/// Keep one input of a side that nothing else needs, so that the side does not lose every column. The first input
/// of the side is taken: the inputs are not always all of the left side and then all of the right one, as an input
/// added later, such as one consuming a column a side appended, comes after all the others.
const auto & inputs = actions_dag.getInputs();
const auto keep_first_input_of = [&](size_t side)
{
if (getInputHeaders().at(side)->empty())
return;
const auto it = std::ranges::find_if(inputs, [&](const auto * input)
{
const JoinActionRef ref(input, expression_actions);
return side == 0 ? ref.fromLeft() : ref.fromRight();
});
if (it != inputs.end())
plan.extra_pruning_roots.push_back(*it);
};
if (!has_required_input_from_left)
keep_first_input_of(0);
if (!has_required_input_from_right)
keep_first_input_of(1);
if (kept_output_nodes.empty() && plan.extra_pruning_roots.empty() && !plan.adds_dummy_output)
throw Exception(ErrorCodes::LOGICAL_ERROR, "No required output nodes, actions_dag: {}", actions_dag.dumpDAG());
/// What removeUnusedActions would keep. It folds constants before it collects the nodes to keep, and
/// folding clears the children of a folded node, so stop at such a node to see the same set.
/// ARRAY_JOIN is always a root there, and so is every input while inputs may not be removed.
auto roots = kept_output_nodes;
roots.insert(roots.end(), plan.extra_pruning_roots.begin(), plan.extra_pruning_roots.end());
for (const auto & node : actions_dag.getNodes())
if (node.type == ActionsDAG::ActionType::ARRAY_JOIN)
roots.push_back(&node);
const auto is_folded_constant = [](const ActionsDAG::Node * node) { return node->column && !node->children.empty(); };
const auto surviving_nodes = findReachableNodes(roots, is_folded_constant);
plan.removes_any_action = surviving_nodes.size() < actions_dag.getNodes().size();
if (!plan.removes_any_action && plan.dropped_output_positions.empty())
{
plan.unneeded_input_positions.resize(input_headers.size());
return plan;
}
/// Rebuild input headers from the surviving inputs. Every input reads a header column of its own,
/// so record the position each one reads rather than re-deriving it from the name once the others
/// are gone: a side whose header repeats a name would otherwise be rebuilt around the wrong column.
/// The positions of each side are collected in header order, which is the order of the new header.
std::array<ActionsDAG::NodeRawConstPtrs, 2> side_inputs;
for (const auto * input : inputs)
{
const auto ref = JoinActionRef(input, expression_actions);
chassert(ref.fromLeft() || ref.fromRight());
side_inputs[ref.fromLeft() ? 0 : 1].push_back(input);
}
for (size_t side = 0; side < 2; ++side)
{
const auto & header = *input_headers[side];
const auto header_columns = mapHeaderColumnsToInputs(side_inputs[side], header);
std::vector<bool> is_required_input(header.columns(), false);
for (size_t position = 0; position < header_columns.size(); ++position)
{
if (header_columns.passesThrough(position))
continue;
if (!surviving_nodes.contains(side_inputs[side][header_columns.read_by[position]]))
continue;
is_required_input[position] = true;
}
std::vector<size_t> positions;
for (size_t position = 0; position < is_required_input.size(); ++position)
if (!is_required_input[position])
positions.push_back(position);
plan.unneeded_input_positions.push_back(std::move(positions));
}
return plan;
}
JoinStepLogical::UnneededInputPositions JoinStepLogical::getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
return analyzeUnneededColumns(unneeded_output_positions).unneeded_input_positions;
}
JoinStepLogical::RemoveUnusedColumnsResult
JoinStepLogical::removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs)
{
const auto plan = analyzeUnneededColumns(unneeded_output_positions);
auto & actions_dag = *expression_actions.getActionsDAG();
auto & dag_outputs = actions_dag.getOutputs();
/// A dropped output goes away, and is erased from `actions_after_join` as well. The dropped positions are sorted,
/// so one walk over the outputs finds them.
const auto & dropped_positions = plan.dropped_output_positions;
ActionsDAG::NodeRawConstPtrs new_actions_after_join = actions_after_join;
ActionsDAG::NodeRawConstPtrs new_outputs;
new_outputs.reserve(dag_outputs.size() - dropped_positions.size() + (plan.adds_dummy_output ? 1 : 0));
size_t next_dropped = 0;
for (size_t position = 0; position < dag_outputs.size(); ++position)
{
const auto * output_node = dag_outputs[position];
if (next_dropped < dropped_positions.size() && dropped_positions[next_dropped] == position)
{
++next_dropped;
new_actions_after_join.erase(
std::remove(new_actions_after_join.begin(), new_actions_after_join.end(), output_node), new_actions_after_join.end());
continue;
}
new_outputs.push_back(output_node);
}
if (plan.adds_dummy_output)
{
auto column_type = std::make_shared<DataTypeUInt8>();
auto column = column_type->createColumnConst(0, 0);
const auto * node = &actions_dag.addColumn(std::move(column), column_type, String(join_dummy_result_name));
new_actions_after_join.push_back(node);
required_nodes.push_back(node);
actions_dag.getOutputs().push_back(node);
kept_output_positions.push_back(RemoveUnusedColumnsResult::NEWLY_ADDED_COLUMN_POSITION);
new_outputs.push_back(node);
dag_outputs.push_back(node);
}
ActionsDAG::NodeRawConstPtrs new_outputs = required_nodes;
const bool changes_outputs = !dropped_positions.empty() || plan.adds_dummy_output;
for (const auto & join_action : join_operator.expression)
required_nodes.push_back(join_action.getNode());
/// An input reading a column its side keeps stays, also where nothing needs it any more; one reading a
/// column the side dropped goes. A column a side keeps beyond what the join reads, or appends, is
/// consumed by an input of its own, so that it stops here.
std::array<ActionsDAG::NodeRawConstPtrs, 2> side_inputs;
for (const auto * input : actions_dag.getInputs())
side_inputs[JoinActionRef(input, expression_actions).fromLeft() ? 0 : 1].push_back(input);
for (const auto & join_action : join_operator.residual_filter)
required_nodes.push_back(join_action.getNode());
if (required_nodes.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "No required output nodes, actions_dag: {}", actions_dag.dumpDAG());
bool has_required_input_from_left = false;
bool has_required_input_from_right = false;
for (const auto * required_node : required_nodes)
std::unordered_set<const ActionsDAG::Node *> kept_inputs;
std::array<ColumnsWithTypeAndName, 2> to_consume;
for (size_t side = 0; side < 2; ++side)
{
const auto source_relations = JoinActionRef(required_node, expression_actions).getSourceRelations();
has_required_input_from_left |= source_relations.test(0);
has_required_input_from_right |= source_relations.test(1);
if (has_required_input_from_left && has_required_input_from_right)
break;
const auto & old_header = *input_headers[side];
const auto & pruned = inputs.at(side);
std::vector<bool> kept(old_header.columns(), true);
for (size_t position : pruned.dropped_positions)
kept.at(position) = false;
for (size_t position : pruned.dropped_positions)
if (!std::ranges::binary_search(plan.unneeded_input_positions.at(side), position))
throw Exception(ErrorCodes::LOGICAL_ERROR,
"The {} side of the join dropped column {}, which the join reads",
side == 0 ? "left" : "right", old_header.getByPosition(position).name);
const auto header_columns = mapHeaderColumnsToInputs(side_inputs[side], old_header);
for (size_t position = 0; position < header_columns.size(); ++position)
if (kept[position] && !header_columns.passesThrough(position))
kept_inputs.insert(side_inputs[side][header_columns.read_by[position]]);
const size_t kept_count = old_header.columns() - pruned.dropped_positions.size();
for (size_t position = kept_count; position < pruned.header->columns(); ++position)
to_consume[side].push_back(pruned.header->getByPosition(position));
}
/// The inputs should be in the same order as input headers
if (!has_required_input_from_left && !actions_dag.getInputs().empty() && !getInputHeaders().at(0)->empty())
{
const auto * maybe_left_input = actions_dag.getInputs().front();
if (JoinActionRef(maybe_left_input, expression_actions).fromLeft())
required_nodes.push_back(maybe_left_input);
}
auto required_nodes = new_outputs;
required_nodes.insert(required_nodes.end(), plan.extra_pruning_roots.begin(), plan.extra_pruning_roots.end());
const auto number_of_left_inputs = input_headers.at(0)->columns();
if (!has_required_input_from_right && actions_dag.getInputs().size() > number_of_left_inputs && !getInputHeaders().at(1)->empty())
{
const auto * maybe_right_input = actions_dag.getInputs().at(number_of_left_inputs);
if (JoinActionRef(maybe_right_input, expression_actions).fromRight())
required_nodes.push_back(maybe_right_input);
}
const auto removed_any_actions = std::invoke(
const auto removed_any_action = std::invoke(
[&]()
{
actions_dag.getOutputs().swap(required_nodes);
SCOPE_EXIT({ actions_dag.getOutputs().swap(required_nodes); });
return actions_dag.removeUnusedActions(remove_inputs);
dag_outputs.swap(required_nodes);
SCOPE_EXIT({ dag_outputs.swap(required_nodes); });
return actions_dag.removeUnusedActions(kept_inputs);
});
if (!removed_any_actions && !removed_any_output)
return {};
actions_dag.getOutputs().swap(new_outputs);
dag_outputs.swap(new_outputs);
actions_after_join = std::move(new_actions_after_join);
for (size_t side = 0; side < 2; ++side)
for (const auto & column : to_consume[side])
expression_actions.addInput(column.name, column.type, side);
/// Update source mapping since nodes may have been removed.
{
JoinExpressionActions::NodeToSourceMapping node_sources;
@@ -553,52 +685,16 @@ JoinStepLogical::RemoveUnusedColumnsResult JoinStepLogical::removeUnusedColumns(
expression_actions.resetNodeSources(std::move(node_sources));
}
/// Check if any DAG inputs were actually removed.
const bool inputs_were_removed = remove_inputs && actions_dag.getInputs().size() < original_input_count;
RemoveUnusedColumnsResult result;
result.dropped_output_positions = dropped_positions;
result.step_changed = changes_outputs || removed_any_action || !to_consume[0].empty() || !to_consume[1].empty()
|| !blocksHaveEqualStructure(*input_headers[0], *inputs[0].header)
|| !blocksHaveEqualStructure(*input_headers[1], *inputs[1].header);
if (inputs_were_removed)
{
/// Rebuild input headers based on surviving DAG inputs.
/// Use source mapping to classify each surviving input as left (0) or right (1).
ActionsDAG::NodeRawConstPtrs surviving_left_inputs;
ActionsDAG::NodeRawConstPtrs surviving_right_inputs;
for (const auto * input_node : actions_dag.getInputs())
{
auto ref = JoinActionRef(input_node, expression_actions);
chassert(ref.fromLeft() || ref.fromRight());
if (ref.fromLeft())
surviving_left_inputs.push_back(input_node);
else if (ref.fromRight())
surviving_right_inputs.push_back(input_node);
}
if (result.step_changed)
updateInputHeaders({inputs[0].header, inputs[1].header});
auto build_new_header = [](const ActionsDAG::NodeRawConstPtrs & surviving_inputs, const Block & old_header)
{
auto positions = ActionsDAG::matchInputNodesToHeader(surviving_inputs, old_header).matched;
Block new_header;
for (size_t pos : positions)
new_header.insert(old_header.getByPosition(pos));
return std::pair{std::move(positions), std::make_shared<const Block>(std::move(new_header))};
};
auto [left_positions, new_left_header] = build_new_header(surviving_left_inputs, *input_headers[0]);
auto [right_positions, new_right_header] = build_new_header(surviving_right_inputs, *input_headers[1]);
updateInputHeaders({std::move(new_left_header), std::move(new_right_header)});
return {true, {std::move(left_positions), std::move(right_positions)}, std::move(kept_output_positions)};
}
updateOutputHeader();
return {true, {}, std::move(kept_output_positions)};
}
bool JoinStepLogical::canRemoveColumnsFromOutput() const
{
if (output_header == nullptr)
return false;
return canRemoveUnusedColumns() && output_header->columns() > 1;
return result;
}
void JoinStepLogical::updateOutputHeader()
src/Processors/QueryPlan/JoinStepLogical.h
+29 -2
View File
@@ -6,6 +6,8 @@
#include <utility>
#include <Interpreters/JoinOperator.h>
#include <Processors/QueryPlan/IQueryPlanStep.h>
#include <span>
#include <Processors/QueryPlan/ISourceStep.h>
#include <Processors/QueryPlan/ITransformingStep.h>
#include <Processors/QueryPlan/JoinStep.h>
@@ -199,8 +201,9 @@ public:
void setTableStatsHint(String table_stats_hint_) { table_stats_hint = std::move(table_stats_hint_); }
bool canRemoveUnusedColumns() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs) override;
bool canRemoveColumnsFromOutput() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs) override;
UnneededInputPositions getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const override;
bool isDisjunctionsOptimizationApplied() const { return disjunctions_optimization_applied; }
void setDisjunctionsOptimizationApplied(bool v) { disjunctions_optimization_applied = v; }
@@ -220,6 +223,30 @@ protected:
bool isDummyColumnOfThisStep(const ActionsDAG::Node * node) const;
/// Everything removeUnusedColumns needs to know, computed without touching the step.
/// Shared by removeUnusedColumns and getUnneededColumns so their answers cannot differ.
struct UnneededColumnsPlan
{
/// What the join does not need of each side.
UnneededInputPositions unneeded_input_positions;
/// The DAG outputs that go away, as positions in `getOutputs()`, sorted:
/// the unneeded ones, except an existing dummy column.
/// A dropped output is also erased from `actions_after_join`, the nodes computed on the joined block,
/// so that the list does not point to a node removed from the DAG.
/// The dummy column, when one is added, is appended after the outputs that remain.
std::vector<size_t> dropped_output_positions;
/// Nodes that have to survive pruning besides the kept outputs: the join conditions, and one
/// input per side that would otherwise lose every column.
ActionsDAG::NodeRawConstPtrs extra_pruning_roots;
/// Whether removeUnusedActions would erase any node.
bool removes_any_action = false;
/// Set when no output is left and the step has to put its dummy column back.
bool adds_dummy_output = false;
};
UnneededColumnsPlan analyzeUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const;
std::vector<std::pair<String, String>> describeJoinProperties() const;
JoinEstimation getEstimation() const;
src/Processors/QueryPlan/LimitStep.cpp
+33
View File
@@ -9,6 +9,7 @@
#include <Core/Defines.h>
#include <IO/Operators.h>
#include <Common/JSONBuilder.h>
#include <unordered_set>
namespace DB
{
@@ -41,6 +42,38 @@ LimitStep::LimitStep(
{
}
IQueryPlanStep::UnneededInputPositions LimitStep::getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const
{
auto unneeded = unneeded_output_positions;
if (with_ties)
{
/// `LimitTransform` compares the first column of each name in `description`, so only that copy stays.
const auto & header = *input_headers.front();
std::unordered_set<size_t> compared_positions;
for (const auto & column : description)
compared_positions.insert(header.getPositionByName(column.column_name));
std::erase_if(unneeded, [&](size_t position) { return compared_positions.contains(position); });
}
return {std::move(unneeded)};
}
IQueryPlanStep::RemoveUnusedColumnsResult
LimitStep::removeUnusedColumns(const std::vector<size_t> & /*unneeded_output_positions*/, const std::vector<PrunedInput> & inputs)
{
const auto & pruned = inputs.at(0);
RemoveUnusedColumnsResult result;
result.dropped_output_positions = pruned.dropped_positions;
result.step_changed = !blocksHaveEqualStructure(*input_headers.front(), *pruned.header);
if (result.step_changed)
updateInputHeader(pruned.header, 0);
return result;
}
void LimitStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
{
/// WITH TIES compares adjacent rows under `description`, so it needs a single ordered
src/Processors/QueryPlan/LimitStep.h
+6
View File
@@ -56,6 +56,12 @@ public:
bool hasCorrelatedExpressions() const override { return false; }
/// Passes every column through, so it drops what its child drops and needs nothing more, except the
/// columns `WITH TIES` compares rows by.
bool canRemoveUnusedColumns() const override { return true; }
UnneededInputPositions getUnneededColumns(const std::vector<size_t> & unneeded_output_positions) const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs) override;
/// A `Limit` at the replica-output boundary is a shard limit, so its output is replicated, not
/// partitioned: every replica emits up to `limit` rows and ships all of them.
bool supportsDataflowStatisticsCollection() const override { return true; }
src/Processors/QueryPlan/Optimizations/Optimizations.h
+19 -11
View File
@@ -209,16 +209,25 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, c
/// Removes unused columns from the query plan. Unused columns can appear after other optimizations, such as filter
/// push down over JOINs. If a column is only used for filtering after a JOIN, and the filter is pushed down into
/// the JOIN condition, then the column may become unused in the plan.
/// This optimization traverses the query plan and attempts to remove such unused columns from the steps if they
/// support the optimization (canRemoveUnusedColumns method).
/// It might happen that a child step supports removing unused columns, but it cannot remove any more columns
/// (canRemoveColumnsFromOutput method returns false, e.g. JoinStepLogical always needs to keep at least one column for
/// its output). In this case or when the children step doesn't support the optimization at all, then the inputs of the
/// optimized step doesn't change.
/// If the children support the optimization but cannot produce the expected output (e.g. JoinStepLogical can remove
/// arbitrary number of columns as long as at least one column remains in the output), then the optimization adds an
/// expression step to convert between the child's new output and the input of the parent node.
size_t tryRemoveUnusedColumns(QueryPlan::Node * node, QueryPlan::Nodes &, const Optimization::ExtraSettings &);
/// One walk over the plan: going down, each step that supports it (canRemoveUnusedColumns) is asked what it does
/// not need of its children once its parent does not need some of its columns; coming back up, it removes those,
/// its children already pruned. A column a child keeps beyond what it was asked for is consumed by the step above
/// it. A step that does not support it needs everything of its children. `root` keeps all its outputs.
enum class RemoveUnusedColumnsMode : uint8_t
{
/// Walks the whole subtree.
Global,
/// Walks down only while a step leaves something of a child unneeded, and not at all if `root` cannot prune.
/// For a local optimization: it finds the columns that a change of `root` left unread.
Local,
};
/// Returns the number of layers below `root` that changed, counting `root` as the first, or 0 when nothing did.
size_t removeUnusedColumns(QueryPlan::Node & root, RemoveUnusedColumnsMode mode);
/// The local mode as a local optimization. Not in `getOptimizations`: it runs only after the global mode, which
/// finds the unused columns of the whole plan at once.
size_t tryRemoveUnusedColumns(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Optimization::ExtraSettings &);
/// Build BloomFilter from right side of JOIN and add condition that looks up into this BloomFilter to the left side of the JOIN.
/// This condition can potentially be pushed down all the way to the storage and filter unmatched rows very early.
@@ -280,7 +289,6 @@ inline const auto & getOptimizations()
{tryConvertAnyJoinToSemiOrAntiJoin,
"convertAnyJoinToSemiOrAntiJoin",
&QueryPlanOptimizationSettings::convert_any_join_to_semi_or_anti_join},
{tryRemoveUnusedColumns, "removeUnusedColumns", &QueryPlanOptimizationSettings::remove_unused_columns},
{tryOptimizeTopK, "tryOptimizeTopK", &QueryPlanOptimizationSettings::try_use_top_k_optimization},
{tryTopKThroughJoin, "topKThroughJoin", &QueryPlanOptimizationSettings::top_k_through_join},
});
src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp
+58
View File
@@ -1,6 +1,7 @@
#include <Common/Exception.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Core/Block.h>
#include <Core/Field.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunction.h>
@@ -660,6 +661,63 @@ std::vector<ActionsDAGOutputLineage> traceActionsDAGLineage(const ActionsDAG & a
return result;
}
HeaderColumnsToInputs mapHeaderColumnsToInputs(const ActionsDAG::NodeRawConstPtrs & inputs, const Block & header)
{
/// Input positions are pushed in reverse so that the front-most one is taken first, which pairs the
/// n-th input of a name with the n-th header column of that name.
std::unordered_map<std::string_view, std::vector<size_t>> name_to_inputs;
for (size_t position = inputs.size(); position != 0; --position)
name_to_inputs[inputs[position - 1]->result_name].push_back(position - 1);
HeaderColumnsToInputs result;
result.read_by.resize(header.columns(), HeaderColumnsToInputs::passes_through);
size_t read_columns = 0;
for (size_t position = 0; position < header.columns(); ++position)
{
auto it = name_to_inputs.find(header.getByPosition(position).name);
if (it == name_to_inputs.end() || it->second.empty())
continue;
result.read_by[position] = it->second.back();
it->second.pop_back();
++read_columns;
}
if (read_columns != inputs.size())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"The header [{}] has a column for only {} of the DAG's {} inputs",
header.dumpNames(), read_columns, inputs.size());
return result;
}
NodeSet findReachableNodes(
const ActionsDAG::NodeRawConstPtrs & roots,
const std::function<bool(const ActionsDAG::Node *)> & is_barrier)
{
NodeSet visited;
std::stack<const ActionsDAG::Node *> stack;
for (const auto * root : roots)
if (visited.insert(root).second)
stack.push(root);
while (!stack.empty())
{
const auto * current = stack.top();
stack.pop();
if (is_barrier && is_barrier(current))
continue;
for (const auto * child : current->children)
if (visited.insert(child).second)
stack.push(child);
}
return visited;
}
bool isInjectiveFunction(const ActionsDAG::Node * node)
{
if (node->function_base->isInjective({}))
src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h
+36
View File
@@ -2,6 +2,8 @@
#include <Interpreters/ActionsDAG.h>
#include <limits>
namespace DB
{
@@ -137,6 +139,40 @@ std::optional<std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Nod
const std::unordered_set<const ActionsDAG::Node *> & allowed_inputs,
const ActionsDAG::NodeRawConstPtrs & nodes);
/// What each column of a header is to a DAG's inputs: one entry per header column, in header order,
/// holding the position in `inputs` of the input reading it, or `passes_through` when no input reads it
/// and the column goes past the DAG untouched.
///
/// Names can repeat in both, so the n-th input named `x` reads the n-th header column named `x`, which
/// is the rule `ActionsDAG::updateHeader` follows to build the block a step runs on. A caller holding
/// some of the inputs can therefore ask where exactly those read from, which
/// `ActionsDAG::matchInputPositionsToHeader` cannot answer: it returns a plain list of the positions
/// something reads, with no way back to which input reads which.
struct HeaderColumnsToInputs
{
static constexpr size_t passes_through = std::numeric_limits<size_t>::max();
std::vector<size_t> read_by;
size_t size() const { return read_by.size(); }
bool passesThrough(size_t header_position) const { return read_by[header_position] == passes_through; }
};
HeaderColumnsToInputs mapHeaderColumnsToInputs(const ActionsDAG::NodeRawConstPtrs & inputs, const Block & header);
/// All nodes reachable from `roots`, including the roots themselves. `roots` may hold duplicates.
/// A node for which `is_barrier` returns true is included, but the walk does not descend into its
/// children.
///
/// Used to answer what `removeUnusedActions` would keep without modifying the DAG: pass the nodes it
/// treats as roots - the kept outputs, every ARRAY_JOIN (it never drops them, they change the number
/// of rows), and every input when inputs may not be removed - and, when it may fold constants, a
/// barrier for the nodes it would fold, because folding clears their children before it collects the
/// nodes to keep.
NodeSet findReachableNodes(
const ActionsDAG::NodeRawConstPtrs & roots,
const std::function<bool(const ActionsDAG::Node *)> & is_barrier = {});
bool isInjectiveFunction(const ActionsDAG::Node * node);
/// Our objective is to replace injective function nodes in `actions` results with its children
src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
-12
View File
@@ -62,18 +62,10 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &, co
&& (parent_actions.hasStatefulFunctions() || dagContainsNonDeterministicFunction(parent_actions)))
return 0;
/// Propagate the flag from either side: if the child is a discarding step (or any
/// step whose inputs must not be stripped), the merged step must keep the same
/// invariant, otherwise `removeUnusedColumns` would prune the inherited inputs and
/// trigger an infinite loop with re-inserted discarding steps.
const bool prevent_input_removal = child_expr->isInputRemovalPrevented() || parent_expr->isInputRemovalPrevented();
auto merged = ActionsDAG::merge(std::move(child_actions), std::move(parent_actions));
auto expr = std::make_unique<ExpressionStep>(child_expr->getInputHeaders().front(), std::move(merged));
expr->setStepDescription(mergeStepDescriptions(parent_expr->getStepDescription(), child_expr->getStepDescription()), settings.max_step_description_length);
if (prevent_input_removal)
expr->setPreventInputRemoval();
parent_node->step = std::move(expr);
parent_node->children.swap(child_node->children);
@@ -90,8 +82,6 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &, co
&& (parent_actions.hasStatefulFunctions() || dagContainsNonDeterministicFunction(parent_actions)))
return 0;
const bool prevent_input_removal = child_expr->isInputRemovalPrevented() || parent_filter->isInputRemovalPrevented();
auto merged = ActionsDAG::merge(std::move(child_actions), std::move(parent_actions));
/// merge can drag materialize wrappers from a UNION child into the filter (#78166); folding through
/// them is left to the `FilterStep` constructor below, which does it after `deduplicateSubtrees` -
@@ -104,8 +94,6 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &, co
parent_filter->getFilterColumnName(),
parent_filter->removesFilterColumn());
filter->setStepDescription(mergeStepDescriptions(parent_filter->getStepDescription(), child_expr->getStepDescription()), settings.max_step_description_length);
if (prevent_input_removal)
filter->setPreventInputRemoval();
parent_node->step = std::move(filter);
parent_node->children.swap(child_node->children);
src/Processors/QueryPlan/Optimizations/mergeFilterIntoJoinCondition.cpp
+1 -1
View File
@@ -451,7 +451,7 @@ size_t tryMergeFilterIntoJoinCondition(QueryPlan::Node * parent_node, QueryPlan:
if (trivial_filter)
{
if (filter_step->removesFilterColumn())
filter_dag.removeUnusedResult(filter_step->getFilterColumnName());
filter_dag.removeFromOutputs(filter_step->getFilterColumnName());
parent_node->step = std::make_unique<ExpressionStep>(filter_step->getInputHeaders().front(), std::move(filter_dag));
}
src/Processors/QueryPlan/Optimizations/optimizeLazyFinal.cpp
+1 -1
View File
@@ -694,7 +694,7 @@ void optimizeLazyFinal(const Stack & stack, QueryPlan & query_plan, QueryPlan::N
/// Compute primary key expression and project to PK columns only.
/// Add all header columns as inputs so that unused ones are properly consumed
/// and can be dropped by tryRemoveUnusedColumns.
/// and can be dropped by `removeUnusedColumns`.
{
auto dag = primary_key_dag.clone();
NamesWithAliases projection;
src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp
+1 -1
View File
@@ -201,7 +201,7 @@ static void addRequiredInputDependenciesIntoNodesSet(const ActionsDAG & dag, std
std::unordered_set<const ActionsDAG::Node *> visited;
struct Frame
{
const ActionsDAG::Node * node;
const ActionsDAG::Node * node = nullptr;
size_t next_child = 0;
};
std::stack<Frame> stack;
src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+7 -54
View File
@@ -10,7 +10,6 @@
#include <Processors/QueryPlan/FilterStep.h>
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
#include <Processors/QueryPlan/Optimizations/optimizePrewhere.h>
#include <Processors/QueryPlan/Optimizations/removeUnusedColumns.h>
#include <Processors/QueryPlan/ReadFromMergeTree.h>
#include <Processors/QueryPlan/SourceStepWithFilter.h>
#include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
@@ -29,11 +28,6 @@ namespace Setting
extern const SettingsBool vector_search_with_rescoring;
}
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
namespace QueryPlanOptimizations
{
@@ -358,56 +352,15 @@ void optimizePrewhere(QueryPlan::Node & parent_node, const bool remove_unused_co
if (!remove_unused_columns)
return;
auto & parent_step = parent_node.step;
if (source_step_with_filter->canRemoveUnusedColumns() && source_step_with_filter->canRemoveColumnsFromOutput()
&& parent_step->canRemoveUnusedColumns())
{
/// Pass all output positions (we want to keep the same outputs, just prune inputs).
const auto & parent_output = parent_step->getOutputHeader();
std::vector<size_t> all_positions;
all_positions.reserve(parent_output->columns());
for (size_t i = 0; i < parent_output->columns(); ++i)
all_positions.push_back(i);
const auto unused_column_removal_result = parent_step->removeUnusedColumns(all_positions, true);
if (unused_column_removal_result.changed && !unused_column_removal_result.required_input_positions.empty())
{
/// The parent step returned the positions it needs from its child (child 0).
/// Pass them directly to the source step.
chassert(unused_column_removal_result.required_input_positions.size() == 1);
const auto & required_positions = unused_column_removal_result.required_input_positions[0];
auto source_removal_result = source_step_with_filter->removeUnusedColumns(required_positions, true);
const auto effective_kept_positions = effectiveKeptOutputPositions(
source_removal_result.changed,
std::move(source_removal_result.kept_output_positions),
source_step_with_filter->getOutputHeader()->columns());
/// The source step might keep extra columns it cannot remove (e.g., `ReadFromMergeTree` with
/// FINAL must keep sort key columns for merging). If so, absorb them into the parent's DAG.
/// The parent step is always an ExpressionStep or FilterStep (created above), so absorption
/// must succeed.
if (!blocksHaveEqualStructure(*parent_step->getInputHeaders().at(0), *source_step_with_filter->getOutputHeader()))
{
if (!absorbExtraChildColumns(parent_node, 0, required_positions, effective_kept_positions))
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Input-output header mismatch after removing unused columns after pushing down filters to prewhere "
"and failed to absorb extra columns. Input header: {}, output header: {}",
parent_step->getInputHeaders().at(0)->dumpStructure(),
source_step_with_filter->getOutputHeader()->dumpStructure());
}
}
/// Keep the outputs as they are, and prune what the new step and the read below it no longer need. A column the
/// read keeps anyway - `ReadFromMergeTree` with FINAL keeps the sorting key for the merge - is consumed by the step.
removeUnusedColumns(parent_node, RemoveUnusedColumnsMode::Local);
#if defined(DEBUG_OR_SANITIZER_BUILD)
{
assertBlocksHaveEqualStructure(
*source_step_with_filter->getOutputHeader(),
*parent_step->getInputHeaders()[0],
"after removing unused columns in optimizePrewhere");
}
assertBlocksHaveEqualStructure(
*source_step_with_filter->getOutputHeader(),
*parent_node.step->getInputHeaders()[0],
"after removing unused columns in optimizePrewhere");
#endif
}
}
}
src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
+109 -69
View File
@@ -131,95 +131,135 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & optimization_se
size_t next_child = 0;
};
std::stack<Frame> stack;
stack.push({.node = &root});
const size_t max_optimizations_to_apply = optimization_settings.max_optimizations_to_apply;
size_t total_applied_optimizations = 0;
const Optimization::ExtraSettings extra_settings = makeExtraSettings(optimization_settings);
while (!stack.empty())
/// Whether the limit of optimizations is reached, checked before each optimization. EXPLAIN stops there and shows
/// the plan as it is; a query throws.
const auto limit_reached = [&]() -> bool
{
auto & frame = stack.top();
if (!max_optimizations_to_apply || max_optimizations_to_apply >= total_applied_optimizations)
return false;
/// If traverse_depth_limit == 0, then traverse without limit (first entrance)
/// If traverse_depth_limit > 1, then traverse with (limit - 1)
if (frame.depth_limit != 1)
if (optimization_settings.is_explain)
return true;
throw Exception(
ErrorCodes::TOO_MANY_QUERY_PLAN_OPTIMIZATIONS,
"Too many optimizations applied to query plan. Current limit {}",
max_optimizations_to_apply);
};
/// Applies the local optimizations bottom-up until none applies any more, and with `remove_unused_columns_locally`
/// the local mode of removing unused columns as one of them. Returns false where EXPLAIN is to stop at the limit
/// of optimizations.
const auto apply_local_optimizations = [&](bool remove_unused_columns_locally) -> bool
{
std::stack<Frame> stack;
stack.push({.node = &root});
while (!stack.empty())
{
/// Traverse all children first.
if (frame.next_child < frame.node->children.size())
auto & frame = stack.top();
/// If traverse_depth_limit == 0, then traverse without limit (first entrance)
/// If traverse_depth_limit > 1, then traverse with (limit - 1)
if (frame.depth_limit != 1)
{
stack.push({
.node = frame.node->children[frame.next_child],
.depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0,
});
/// Traverse all children first.
if (frame.next_child < frame.node->children.size())
{
stack.push({
.node = frame.node->children[frame.next_child],
.depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0,
});
++frame.next_child;
continue;
}
}
/// An optimization applied to a child node may have changed a grandchild's
/// output header (e.g., filter push-down modifies a filter step's DAG, which
/// changes its output constness). The intermediate child step's cached input
/// header becomes stale. Refresh it before running optimizations on this node,
/// so that steps like mergeExpressions see consistent headers.
for (size_t i = 0; i < frame.node->children.size(); ++i)
{
auto child_output = frame.node->children[i]->step->getOutputHeader();
if (!blocksHaveEqualStructure(*frame.node->step->getInputHeaders()[i], *child_output))
frame.node->step->updateInputHeader(std::move(child_output), i);
}
size_t max_update_depth = 0;
/// Apply all optimizations.
for (const auto & optimization : getOptimizations())
{
if (!(optimization_settings.*(optimization.is_enabled)))
continue;
/// Just in case, skip optimization if it is not initialized.
if (!optimization.apply)
continue;
if (max_optimizations_to_apply && max_optimizations_to_apply < total_applied_optimizations)
{
if (optimization_settings.is_explain)
return;
throw Exception(
ErrorCodes::TOO_MANY_QUERY_PLAN_OPTIMIZATIONS,
"Too many optimizations applied to query plan. Current limit {}",
max_optimizations_to_apply);
++frame.next_child;
continue;
}
}
/// Try to apply optimization.
auto update_depth = optimization.apply(frame.node, nodes, extra_settings);
if (update_depth)
/// An optimization applied to a child node may have changed a grandchild's
/// output header (e.g., filter push-down modifies a filter step's DAG, which
/// changes its output constness). The intermediate child step's cached input
/// header becomes stale. Refresh it before running optimizations on this node,
/// so that steps like mergeExpressions see consistent headers.
for (size_t i = 0; i < frame.node->children.size(); ++i)
{
auto child_output = frame.node->children[i]->step->getOutputHeader();
if (!blocksHaveEqualStructure(*frame.node->step->getInputHeaders()[i], *child_output))
frame.node->step->updateInputHeader(std::move(child_output), i);
}
size_t max_update_depth = 0;
/// Tries to apply one optimization. Returns false where EXPLAIN is to stop at the limit of optimizations.
const auto apply = [&](const auto & apply_optimization, [[maybe_unused]] std::string_view name) -> bool
{
if (limit_reached())
return false;
auto update_depth = apply_optimization(frame.node, nodes, extra_settings);
if (update_depth)
{
#if defined(DEBUG_OR_SANITIZER_BUILD)
checkHeaders(*frame.node, String("after optimization ") + optimization.name, update_depth);
checkHeaders(*frame.node, fmt::format("after optimization {}", name), update_depth);
#endif
++total_applied_optimizations;
++total_applied_optimizations;
}
max_update_depth = std::max<size_t>(max_update_depth, update_depth);
return true;
};
/// Apply all optimizations.
for (const auto & optimization : getOptimizations())
{
if (!(optimization_settings.*(optimization.is_enabled)))
continue;
/// Just in case, skip optimization if it is not initialized.
if (!optimization.apply)
continue;
if (!apply(optimization.apply, optimization.name))
return false;
}
max_update_depth = std::max<size_t>(max_update_depth, update_depth);
if (remove_unused_columns_locally && !apply(tryRemoveUnusedColumns, "removeUnusedColumns"))
return false;
/// Traverse `max_update_depth` layers of tree again.
if (max_update_depth)
{
frame.depth_limit = max_update_depth;
frame.next_child = 0;
continue;
}
/// Nothing was applied.
stack.pop();
}
/// Traverse `max_update_depth` layers of tree again.
if (max_update_depth)
{
frame.depth_limit = max_update_depth;
frame.next_child = 0;
continue;
}
return true;
};
/// Nothing was applied.
stack.pop();
}
if (!apply_local_optimizations(/*remove_unused_columns_locally=*/false))
return;
if (!optimization_settings.remove_unused_columns)
return;
/// Removing unused columns looks at the whole plan at once, so it runs after the local optimizations. Fewer columns
/// can let more of them apply, so they run again, and the local mode of removing unused columns with them: a change
/// of a step can leave columns below it unread. It counts as one optimization, under the same limit.
if (limit_reached() || !removeUnusedColumns(root, RemoveUnusedColumnsMode::Global))
return;
++total_applied_optimizations;
apply_local_optimizations(/*remove_unused_columns_locally=*/true);
}
void tryMakeDistributedJoin(QueryPlan::Node & node, QueryPlan::Nodes & nodes, const QueryPlanOptimizationSettings & optimization_settings);
src/Processors/QueryPlan/Optimizations/removeUnusedColumns.cpp
+131 -294
View File
@@ -1,329 +1,166 @@
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
#include <Processors/QueryPlan/Optimizations/removeUnusedColumns.h>
#include <numeric>
#include <Core/Names.h>
#include <Interpreters/ActionsDAG.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/FilterStep.h>
#include <Core/Block.h>
#include <Processors/QueryPlan/IQueryPlanStep.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <stack>
namespace DB
{
namespace QueryPlanOptimizations
{
namespace
{
/// Position-based overload: compare kept_output_positions (what the child actually kept)
/// against required_positions (what the parent asked for) to determine extras to discard.
/// Builds a DAG with all child output columns as inputs and only the required columns as
/// outputs, so that the resulting ExpressionStep projects down to exactly the required set.
/// Correct even with duplicate column names because all columns are consumed as inputs
/// (no name-based pass-through matching).
bool addDiscardingExpressionStepIfNeeded(
QueryPlan::Nodes & nodes,
QueryPlan::Node & parent,
const size_t child_id,
const std::vector<size_t> & required_positions,
const std::vector<size_t> & kept_output_positions)
/// A step on the way down, with what the step above does not need of it and what its children produce once
/// they are pruned themselves.
struct PruningFrame
{
const auto output_header = parent.children[child_id]->step->getOutputHeader();
QueryPlan::Node * node = nullptr;
std::vector<size_t> unneeded_outputs;
size_t depth = 0;
std::set<size_t> required_set(required_positions.begin(), required_positions.end());
/// A step that cannot drop columns needs all its children produce, and outputs all it did. So does a
/// sink, which has no output at all, such as the root of a distributed plan fragment.
bool can_prune = false;
IQueryPlanStep::UnneededInputPositions unneeded_inputs;
/// Check whether there are any columns to discard.
bool has_columns_to_discard = false;
for (size_t kept_output_position : kept_output_positions)
{
if (!required_set.contains(kept_output_position))
{
has_columns_to_discard = true;
break;
}
}
if (!has_columns_to_discard)
{
/// Even when all column names match, the column representations might differ
/// (e.g., Const vs materialized after JoinStepLogical materializes its dummy column).
/// Sync the parent's input header with the child's output header.
if (!blocksHaveEqualStructure(*parent.step->getInputHeaders()[child_id], *output_header))
parent.step->updateInputHeader(output_header, child_id);
return false;
}
/// Add all child output columns as DAG inputs, in header order.
/// This ensures every header column is consumed by name matching in `updateHeader`,
/// avoiding ambiguity when there are duplicate column names.
ActionsDAG discarding_dag;
std::vector<const ActionsDAG::Node *> input_nodes;
input_nodes.reserve(output_header->columns());
for (size_t pos = 0; pos < output_header->columns(); ++pos)
{
const auto & col = output_header->getByPosition(pos);
input_nodes.push_back(&discarding_dag.addInput(col.name, col.type));
}
/// Only add the required columns as DAG outputs.
auto & dag_outputs = discarding_dag.getOutputs();
for (size_t new_pos = 0; new_pos < kept_output_positions.size(); ++new_pos)
{
if (required_set.contains(kept_output_positions[new_pos]))
dag_outputs.push_back(input_nodes[new_pos]);
}
auto discarding_step = std::make_unique<ExpressionStep>(output_header, std::move(discarding_dag));
discarding_step->setStepDescription("Discarding unused columns");
/// The discarding step intentionally consumes (and drops) columns that the child cannot
/// reduce away. Subsequent passes must not strip these inputs, otherwise the next
/// optimization pass would re-create the discarding step and loop forever (e.g. when
/// `mergeExpressions` collapses this step into a parent ExpressionStep).
discarding_step->setPreventInputRemoval();
auto & discarding_node = nodes.emplace_back();
discarding_node.step = std::move(discarding_step);
discarding_node.children.push_back(parent.children[child_id]);
parent.children[child_id] = &discarding_node;
return true;
}
enum class RemoveChildrenOutputResult : UInt8
{
NotUpdated = 0,
Updated = 1,
AddedDiscardingStep = 2,
UpdatedAndAddedDiscardingStep = 3,
};
}
static bool canAllChildrenCanRemoveOutputs(const QueryPlan::Node & node)
{
return std::all_of(
node.children.begin(),
node.children.end(),
[](const QueryPlan::Node * child) { return child->step->canRemoveUnusedColumns() && child->step->canRemoveColumnsFromOutput(); });
}
std::vector<size_t> effectiveKeptOutputPositions(bool changed, std::vector<size_t> && kept_output_positions, size_t num_output_columns)
{
if (changed)
return std::move(kept_output_positions);
std::vector<size_t> all_positions(num_output_columns);
std::iota(all_positions.begin(), all_positions.end(), 0);
return all_positions;
}
bool absorbExtraChildColumns(
QueryPlan::Node & node,
size_t child_id,
const std::vector<size_t> & required_positions,
const std::vector<size_t> & kept_output_positions)
{
ActionsDAG * dag = nullptr;
auto * expr_step = typeid_cast<ExpressionStep *>(node.step.get());
auto * filter_step = typeid_cast<FilterStep *>(node.step.get());
if (expr_step)
dag = &expr_step->getExpression();
else if (filter_step)
dag = &filter_step->getExpression();
else
return false;
const auto & child_output = node.children[child_id]->step->getOutputHeader();
/// Identify extra columns: positions in kept_output_positions that are not in required_positions.
std::set<size_t> required_set(required_positions.begin(), required_positions.end());
bool added_any = false;
for (size_t new_pos = 0; new_pos < kept_output_positions.size(); ++new_pos)
{
if (!required_set.contains(kept_output_positions[new_pos]))
{
const auto & col = child_output->getByPosition(new_pos);
dag->addInput(col.name, col.type);
added_any = true;
}
}
if (!added_any)
return false;
/// Use the child's output header directly as the new input header
/// to ensure column order matches exactly.
node.step->updateInputHeader(child_output, child_id);
/// Prevent future optimization passes from removing these absorbed inputs,
/// which would re-create the mismatch and cause infinite optimization loops.
if (expr_step)
expr_step->setPreventInputRemoval();
else
filter_step->setPreventInputRemoval();
return true;
}
struct ChildUpdateResult
{
bool updated = false;
bool added_discarding_step = false;
std::vector<IQueryPlanStep::PrunedInput> children;
};
static ChildUpdateResult removeSingleChildOutput(
QueryPlan::Nodes & nodes,
QueryPlan::Node & node,
const size_t child_id,
const std::vector<size_t> & required_positions
)
PruningFrame makeFrame(QueryPlan::Node & node, std::vector<size_t> unneeded_outputs, size_t depth)
{
auto & child_step = node.children[child_id]->step;
chassert(child_step->canRemoveUnusedColumns());
PruningFrame frame;
frame.node = &node;
frame.unneeded_outputs = std::move(unneeded_outputs);
frame.depth = depth;
const auto & step = *node.step;
frame.can_prune = step.hasOutputHeader() && step.canRemoveUnusedColumns();
ChildUpdateResult result{};
/// Going down, a step is only asked what it does not need of its children; it changes on the way back up,
/// once its children have changed and it knows what they really produce.
if (frame.can_prune && !node.children.empty())
frame.unneeded_inputs = step.getUnneededColumns(frame.unneeded_outputs);
// Here we never want to remove inputs because the grandchildren might not be able to remove outputs.
auto child_result = child_step->removeUnusedColumns(required_positions, false);
const bool child_updated = child_result.changed;
return frame;
}
if (child_updated)
result.updated = true;
const auto effective_kept_positions = effectiveKeptOutputPositions(
child_result.changed,
std::move(child_result.kept_output_positions),
child_step->getOutputHeader()->columns());
/// If the child's output doesn't match the parent's input (extra columns the child
/// couldn't remove, e.g. ReadFromMergeTree with FINAL keeping sort key columns,
/// or JoinStepLogical keeping a dummy column), reconcile the headers.
/// First try absorbing extras into the parent's DAG (no extra step needed).
/// Fall back to inserting a discarding ExpressionStep if absorption isn't possible.
/// Whether `new_header` has the columns of `old_header` that are not at `dropped_positions` first, in their order.
/// Columns appended after them are allowed, such as the dummy column a join adds.
bool keepsRemainingColumnsFirst(const Block & old_header, const std::vector<size_t> & dropped_positions, const Block & new_header)
{
size_t new_position = 0;
size_t next_dropped = 0;
for (size_t position = 0; position < old_header.columns(); ++position)
{
const auto & current_parent_input = node.step->getInputHeaders()[child_id];
const auto & child_output = child_step->getOutputHeader();
if (!blocksHaveEqualStructure(*current_parent_input, *child_output))
if (next_dropped < dropped_positions.size() && dropped_positions[next_dropped] == position)
{
if (!absorbExtraChildColumns(node, child_id, required_positions, effective_kept_positions))
++next_dropped;
continue;
}
if (new_position == new_header.columns() || new_header.getByPosition(new_position).name != old_header.getByPosition(position).name)
return false;
++new_position;
}
return next_dropped == dropped_positions.size();
}
/// Prunes the frame's step once its children are done, and says what it dropped.
IQueryPlanStep::PrunedInput pruneStep(PruningFrame & frame, bool & changed)
{
auto & step = *frame.node->step;
if (!frame.can_prune)
{
/// Asked for everything, a child keeps everything, but the representation of a column may change, such as a
/// constant that is materialized now.
for (size_t child = 0; child < frame.children.size(); ++child)
if (!blocksHaveEqualStructure(*step.getInputHeaders()[child], *frame.children[child].header))
step.updateInputHeader(frame.children[child].header, child);
if (!step.hasOutputHeader())
return {};
return IQueryPlanStep::PrunedInput::unchanged(step.getOutputHeader());
}
const auto old_output_header = step.getOutputHeader();
auto result = step.removeUnusedColumns(frame.unneeded_outputs, frame.children);
changed = result.step_changed;
/// The step above relies on this to match the columns of the new header with the old ones.
chassert(keepsRemainingColumnsFirst(*old_output_header, result.dropped_output_positions, *step.getOutputHeader()));
return {std::move(result.dropped_output_positions), step.getOutputHeader()};
}
}
size_t removeUnusedColumns(QueryPlan::Node & root, RemoveUnusedColumnsMode mode)
{
const bool is_local = mode == RemoveUnusedColumnsMode::Local;
/// The root keeps all its outputs: nothing above it is looked at.
std::stack<PruningFrame> stack;
stack.push(makeFrame(root, {}, 0));
if (is_local && !stack.top().can_prune)
return 0;
size_t changed_depth = 0;
while (!stack.empty())
{
/// A reference stays valid while frames are pushed above it: the stack is a `std::deque`.
auto & frame = stack.top();
const size_t child = frame.children.size();
if (child < frame.node->children.size())
{
auto & child_node = *frame.node->children[child];
/// A step that cannot drop columns keeps its children as they are. So does a local walk where nothing of
/// a child is unneeded any more: what is left there is the business of the child itself.
if (frame.can_prune)
{
if (addDiscardingExpressionStepIfNeeded(nodes, node, child_id, required_positions, effective_kept_positions))
auto & unneeded = frame.unneeded_inputs.at(child);
if (!is_local || !unneeded.empty())
{
#if defined(DEBUG_OR_SANITIZER_BUILD)
const auto & discarding_step = *node.children[child_id]->step;
assertBlocksHaveEqualStructure(
*discarding_step.getInputHeaders()[0], *child_step->getOutputHeader(), "after adding discarding step");
#endif
result.added_discarding_step = true;
stack.push(makeFrame(child_node, std::move(unneeded), frame.depth + 1));
if (!is_local || stack.top().can_prune)
continue;
/// A child that cannot drop columns keeps everything it was asked not to.
stack.pop();
}
}
else if (!is_local)
{
stack.push(makeFrame(child_node, {}, frame.depth + 1));
continue;
}
frame.children.push_back(IQueryPlanStep::PrunedInput::unchanged(child_node.step->getOutputHeader()));
continue;
}
}
#if defined(DEBUG_OR_SANITIZER_BUILD)
{
const auto & final_parent_inputs = node.step->getInputHeaders();
assertBlocksHaveEqualStructure(
*node.children[child_id]->step->getOutputHeader(), *final_parent_inputs[child_id], "after removing unused columns");
}
#endif
return result;
bool changed = false;
auto pruned = pruneStep(frame, changed);
if (changed)
changed_depth = std::max(changed_depth, frame.depth + 1);
stack.pop();
if (!stack.empty())
stack.top().children.push_back(std::move(pruned));
}
return changed_depth;
}
/// Remove unused columns from the children of `node`, using the required input positions
/// returned by the parent's `removeUnusedColumns` call. If `required_input_positions` is
/// non-empty, use those positions directly; otherwise fall back to computing positions
/// from headers (for the case where the caller doesn't have positions yet).
static RemoveChildrenOutputResult removeChildrenOutputs(
QueryPlan::Nodes & nodes,
QueryPlan::Node & node,
const std::vector<std::vector<size_t>> & required_input_positions)
size_t tryRemoveUnusedColumns(QueryPlan::Node * node, QueryPlan::Nodes &, const Optimization::ExtraSettings &)
{
bool updated_any_child = false;
bool added_any_discarding_step = false;
for (auto child_id = 0U; child_id < node.children.size(); ++child_id)
{
chassert(child_id < required_input_positions.size());
const auto & required_positions = required_input_positions[child_id];
const auto [child_updated, added_discarding_step] = removeSingleChildOutput(nodes, node, child_id, required_positions);
updated_any_child |= child_updated;
added_any_discarding_step |= added_discarding_step;
}
if (updated_any_child && added_any_discarding_step)
return RemoveChildrenOutputResult::UpdatedAndAddedDiscardingStep;
if (added_any_discarding_step)
return RemoveChildrenOutputResult::AddedDiscardingStep;
if (updated_any_child)
return RemoveChildrenOutputResult::Updated;
return RemoveChildrenOutputResult::NotUpdated;
}
size_t tryRemoveUnusedColumns(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Optimization::ExtraSettings &)
{
std::vector<std::vector<size_t>> required_input_positions;
size_t depth = 0;
const auto can_remove_inputs = canAllChildrenCanRemoveOutputs(*node);
/// If the node supports removeUnusedColumns, call it to prune its own DAG.
if (node->step->canRemoveUnusedColumns())
{
const auto num_outputs = node->step->getOutputHeader()->columns();
std::vector<size_t> all_outputs(num_outputs);
std::iota(all_outputs.begin(), all_outputs.end(), 0);
auto remove_result = node->step->removeUnusedColumns(all_outputs, can_remove_inputs);
if (remove_result.changed)
++depth;
required_input_positions = std::move(remove_result.required_input_positions);
}
/// If the children don't support unused column removal, let's return
if (!can_remove_inputs)
return depth;
/// If we don't have required input positions (either the node didn't prune inputs,
/// or it doesn't support removeUnusedColumns), compute them from the node's input
/// headers: each child must produce at least what the node's input header expects.
if (required_input_positions.empty())
{
const auto & input_headers = node->step->getInputHeaders();
required_input_positions.resize(input_headers.size());
for (size_t child_id = 0; child_id < input_headers.size(); ++child_id)
{
const auto num_input_cols = input_headers[child_id]->columns();
required_input_positions[child_id].resize(num_input_cols);
std::iota(required_input_positions[child_id].begin(), required_input_positions[child_id].end(), 0);
}
}
/// Propagate reduced requirements to children.
auto result = removeChildrenOutputs(nodes, *node, required_input_positions);
switch (result)
{
case RemoveChildrenOutputResult::NotUpdated:
return depth; // Only report if the node was changed
case RemoveChildrenOutputResult::Updated:
case RemoveChildrenOutputResult::AddedDiscardingStep:
return 2; // Node + its children were changed or node + discarding step was added
case RemoveChildrenOutputResult::UpdatedAndAddedDiscardingStep:
return 3; // Node + its children were changed and one or more discarding steps were added
}
UNREACHABLE();
return removeUnusedColumns(*node, RemoveUnusedColumnsMode::Local);
}
}
src/Processors/QueryPlan/Optimizations/removeUnusedColumns.h
-37
View File
@@ -1,37 +0,0 @@
#pragma once
#include <Processors/QueryPlan/QueryPlan.h>
#include <vector>
namespace DB::QueryPlanOptimizations
{
/// When the parent step removed some inputs but the child step couldn't fully reduce its output
/// to match (e.g., ReadFromMergeTree with FINAL must keep columns required for merging),
/// adjust the parent step to accept the extra columns from the child by adding them as
/// consumed DAG inputs and setting the input header to match the child's output exactly.
/// Also sets the `prevent_input_removal` flag to ensure these absorbed columns are not
/// removed on subsequent optimization passes.
/// Works with both ExpressionStep and FilterStep parents.
///
/// Uses required_positions (what the parent asked for) and kept_output_positions
/// (what the child actually kept) to identify extra columns by position,
/// which is correct even with duplicate column names.
bool absorbExtraChildColumns(
QueryPlan::Node & node,
size_t child_id,
const std::vector<size_t> & required_positions,
const std::vector<size_t> & kept_output_positions);
/// Resolves the `kept_output_positions` field of `RemoveUnusedColumnsResult` into an
/// explicit position list. When a step is unchanged (`changed` is false), the result's
/// `kept_output_positions` is empty by convention (meaning all original positions are
/// kept). This helper moves the vector through when it is already populated, or builds
/// the identity list [0, 1, ..., N-1] otherwise.
std::vector<size_t> effectiveKeptOutputPositions(
bool changed,
std::vector<size_t> && kept_output_positions,
size_t num_output_columns);
}
src/Processors/QueryPlan/Optimizations/splitFilter.cpp
+5 -3
View File
@@ -68,6 +68,11 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Op
if (split.second.trivial())
return 0;
/// The filter node may have been renamed by the split to avoid clashing with an input of the same name.
std::string split_filter_name = split.split_nodes_mapping.at(filter_dag_node)->result_name;
/// A renamed filter column reaches the second half as an input under the new name, with an alias back to the old
/// one, and removeUnusedResult removes the alias together with the input.
bool remove_filter = false;
if (filter_step->removesFilterColumn())
remove_filter = split.second.removeUnusedResult(filter_column_name);
@@ -78,9 +83,6 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Op
node->children.swap(filter_node.children);
node->children.push_back(&filter_node);
/// The filter node may have been renamed by the split to avoid clashing with an input of the same name.
std::string split_filter_name = split.split_nodes_mapping.at(filter_dag_node)->result_name;
filter_node.step = std::make_unique<FilterStep>(
filter_node.children.at(0)->step->getOutputHeader(),
std::move(split.first),
src/Processors/QueryPlan/Optimizations/useVectorSearch.cpp
+2 -2
View File
@@ -466,7 +466,7 @@ bool optimizeVectorSearchWithVectorIndexSecondPass(QueryPlan::Node & /*root*/, S
/// such as `length(vec)` needs the physical vector column even though `vec` is not an
/// output node of the DAG.
auto pruned_projection_expression = expression.clone();
pruned_projection_expression.removeUnusedResult(sort_column);
pruned_projection_expression.removeFromOutputs(sort_column);
pruned_projection_expression.removeUnusedActions();
for (const auto * input : pruned_projection_expression.getInputs())
@@ -595,7 +595,7 @@ bool optimizeVectorSearchWithVectorIndexSecondPass(QueryPlan::Node & /*root*/, S
const size_t sort_column_pos = std::find(outputs.begin(), outputs.end(), sort_column_node) - outputs.begin();
/// Now replace the "cosineDistance(vec, [1.0, 2.0...])" node in the DAG by the "_distance" node
expression.removeUnusedResult(sort_column); /// Removes the OUTPUT cosineDistance(...) FUNCTION Node
expression.removeFromOutputs(sort_column); /// Removes the OUTPUT cosineDistance(...) FUNCTION Node
expression.removeUnusedActions(); /// Removes the vector column INPUT node (it is no longer needed)
const auto * distance_node = &expression.addInput("_distance",std::make_shared<DataTypeFloat32>());
src/Processors/QueryPlan/Optimizations/useVectorSearchWithQuantizedCodes.cpp
-2
View File
@@ -454,8 +454,6 @@ bool optimizeVectorSearchWithQuantizedCodes(
auto discard_approx_step = std::make_unique<ExpressionStep>(shortlist_header, std::move(discard_approx_dag));
discard_approx_step->setStepDescription("quantized shortlist discard approximate distance");
/// Keep the discarded inputs, or a later pass strips them and re-exposes the column.
discard_approx_step->setPreventInputRemoval();
auto & discard_approx_node = nodes.emplace_back();
discard_approx_node.step = std::move(discard_approx_step);
src/Processors/QueryPlan/ReadFromMergeTree.cpp
+36 -41
View File
@@ -6480,15 +6480,16 @@ bool ReadFromMergeTree::canRemoveUnusedColumns() const
return true;
}
ReadFromMergeTree::RemoveUnusedColumnsResult ReadFromMergeTree::removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool /*remove_inputs*/)
ReadFromMergeTree::RemoveUnusedColumnsResult
ReadFromMergeTree::removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & /*inputs*/)
{
if (output_header == nullptr)
return {};
/// Positions in the final RFMT output that must be preserved for the parent step or FINAL.
std::set<size_t> required_final_output_positions(required_output_positions.begin(), required_output_positions.end());
/// Positions in all_column_names that must still be read from storage.
std::set<size_t> required_storage_column_positions;
/// Positions in the output header that go away: the unneeded ones, except the ones FINAL merges by.
std::set<size_t> dropped_output_set(unneeded_output_positions.begin(), unneeded_output_positions.end());
/// Positions in all_column_names that FINAL needs to read whatever else goes away.
std::set<size_t> final_storage_column_positions;
if (query_info.isFinal())
{
const auto required_for_final
@@ -6497,75 +6498,70 @@ ReadFromMergeTree::RemoveUnusedColumnsResult ReadFromMergeTree::removeUnusedColu
for (size_t pos = 0; pos < output_header->columns(); ++pos)
{
if (required_for_final.contains(output_header->getByPosition(pos).name))
required_final_output_positions.insert(pos);
dropped_output_set.erase(pos);
}
/// Merging columns absent from the output header are added by initializePipeline and projected back off there.
for (size_t pos = 0; pos < all_column_names.size(); ++pos)
{
if (required_for_final.contains(all_column_names[pos]) && output_header->has(all_column_names[pos]))
required_storage_column_positions.insert(pos);
final_storage_column_positions.insert(pos);
}
}
/// Sorted vector form of required_final_output_positions, used as the initial backward-pruning frontier.
std::vector<size_t> final_output_positions(
required_final_output_positions.begin(),
required_final_output_positions.end());
const std::vector<size_t> dropped_output_positions(dropped_output_set.begin(), dropped_output_set.end());
Block storage_header = storage_snapshot->getSampleBlockForColumns(all_column_names);
Block row_level_output_header = storage_header;
if (query_info.row_level_filter)
row_level_output_header = SourceStepWithFilter::applyPrewhereActions(std::move(row_level_output_header), query_info.row_level_filter, nullptr);
/// Positions in the row-policy output header, which is the input header for PREWHERE.
std::vector<size_t> required_row_level_output_positions;
/// Positions from the old final RFMT output that remain after pruning.
std::vector<size_t> kept_output_positions = final_output_positions;
/// Positions in the row-policy output header, which is the input header for PREWHERE, that nothing needs.
std::vector<size_t> unneeded_row_level_output_positions;
bool removed_output_from_prewhere = false;
if (query_info.prewhere_info)
{
auto prewhere_pruning = pruneFilterDAGOutputsByPosition(
auto prewhere_pruning = FilterStep::pruneDAGOutputsByPosition(
query_info.prewhere_info->prewhere_actions,
query_info.prewhere_info->prewhere_column_name,
query_info.prewhere_info->remove_prewhere_column,
row_level_output_header,
final_output_positions,
true);
dropped_output_positions);
removed_output_from_prewhere = prewhere_pruning.changed;
required_row_level_output_positions = std::move(prewhere_pruning.required_input_positions);
unneeded_row_level_output_positions = std::move(prewhere_pruning.unneeded_input_positions);
}
else
{
required_row_level_output_positions = final_output_positions;
unneeded_row_level_output_positions = dropped_output_positions;
}
bool removed_output_from_row_level_filter = false;
/// Positions in the storage header required by row policy and PREWHERE filters.
std::vector<size_t> required_storage_positions_from_filters;
/// Positions in the storage header that neither the row policy nor PREWHERE filters need.
std::vector<size_t> unneeded_storage_positions;
if (query_info.row_level_filter)
{
auto row_level_pruning = pruneFilterDAGOutputsByPosition(
auto row_level_pruning = FilterStep::pruneDAGOutputsByPosition(
query_info.row_level_filter->actions,
query_info.row_level_filter->column_name,
query_info.row_level_filter->do_remove_column,
storage_header,
required_row_level_output_positions,
true);
unneeded_row_level_output_positions);
removed_output_from_row_level_filter = row_level_pruning.changed;
required_storage_positions_from_filters = std::move(row_level_pruning.required_input_positions);
unneeded_storage_positions = std::move(row_level_pruning.unneeded_input_positions);
}
else
{
required_storage_positions_from_filters = required_row_level_output_positions;
unneeded_storage_positions = std::move(unneeded_row_level_output_positions);
}
required_storage_column_positions.insert(required_storage_positions_from_filters.begin(), required_storage_positions_from_filters.end());
std::set<size_t> dropped_storage_column_positions(unneeded_storage_positions.begin(), unneeded_storage_positions.end());
for (size_t pos : final_storage_column_positions)
dropped_storage_column_positions.erase(pos);
Names new_column_names;
for (size_t pos = 0; pos < all_column_names.size(); ++pos)
{
if (required_storage_column_positions.contains(pos))
if (!dropped_storage_column_positions.contains(pos))
new_column_names.push_back(all_column_names[pos]);
}
@@ -6574,17 +6570,20 @@ ReadFromMergeTree::RemoveUnusedColumnsResult ReadFromMergeTree::removeUnusedColu
all_column_names = std::move(new_column_names);
const size_t former_output_column_count = output_header->columns();
output_header = std::make_shared<const Block>(MergeTreeSelectProcessor::transformHeader(
storage_snapshot->getSampleBlockForColumns(all_column_names),
query_info.row_level_filter,
query_info.prewhere_info));
if (kept_output_positions.size() != output_header->columns())
if (former_output_column_count - dropped_output_positions.size() != output_header->columns())
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Unexpected number of kept output positions after removing unused columns from ReadFromMergeTree: expected {}, got {}",
output_header->columns(),
kept_output_positions.size());
"Unexpected number of output columns after removing unused columns from ReadFromMergeTree: "
"{} columns, {} of them dropped, but {} left",
former_output_column_count,
dropped_output_positions.size(),
output_header->columns());
/// Update analysis result if it exists
if (analyzed_result_ptr)
@@ -6592,16 +6591,12 @@ ReadFromMergeTree::RemoveUnusedColumnsResult ReadFromMergeTree::removeUnusedColu
required_source_columns = all_column_names;
return {true, {}, std::move(kept_output_positions)};
RemoveUnusedColumnsResult result;
result.step_changed = true;
result.dropped_output_positions = dropped_output_positions;
return result;
}
bool ReadFromMergeTree::canRemoveColumnsFromOutput() const
{
if (output_header == nullptr)
return false;
return canRemoveUnusedColumns() && output_header->columns() > 0;
}
void ReadFromMergeTree::setDistributedRead(size_t bucket_count)
{
src/Processors/QueryPlan/ReadFromMergeTree.h
+1 -2
View File
@@ -579,8 +579,7 @@ public:
Strings getShardsForDistributedRead() const;
bool canRemoveUnusedColumns() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & required_output_positions, bool remove_inputs) override;
bool canRemoveColumnsFromOutput() const override;
RemoveUnusedColumnsResult removeUnusedColumns(const std::vector<size_t> & unneeded_output_positions, const std::vector<PrunedInput> & inputs) override;
bool isSelectedForTopKFilterOptimization() const { return top_k_filter_info.has_value(); }
const std::optional<TopKFilterInfo> & getTopKFilterInfo() const { return top_k_filter_info; }
src/Processors/QueryPlan/tests/gtest_join_step_logical_unused_columns.cpp
+90
View File
@@ -0,0 +1,90 @@
#include <gtest/gtest.h>
#include <Core/Block.h>
#include <Core/ProtocolDefines.h>
#include <DataTypes/DataTypesNumber.h>
#include <Interpreters/JoinExpressionActions.h>
#include <Interpreters/JoinOperator.h>
#include <Processors/QueryPlan/JoinStepLogical.h>
#include <Processors/QueryPlan/QueryPlanSerializationSettings.h>
#include <Processors/QueryPlan/SortingStep.h>
using namespace DB;
namespace
{
SharedHeader makeHeader(const Names & names)
{
Block header;
for (const auto & name : names)
{
auto type = std::make_shared<DataTypeUInt64>();
header.insert({type->createColumn(), type, name});
}
return std::make_shared<const Block>(std::move(header));
}
/// A CROSS JOIN whose DAG reads `right_inputs` of the right side first and then `appended_left_inputs` of the left
/// side, as when a left input was added after the others, such as one consuming a column the left side appended. Every
/// input is an output, so the join reads nothing for itself.
std::unique_ptr<JoinStepLogical> makeJoin(const Names & left_inputs, const Names & right_inputs, const Names & appended_left_inputs)
{
Names left_names = left_inputs;
left_names.insert(left_names.end(), appended_left_inputs.begin(), appended_left_inputs.end());
auto left_header = makeHeader(left_names);
auto right_header = makeHeader(right_inputs);
JoinExpressionActions expression_actions(*makeHeader(left_inputs), *right_header);
for (const auto & name : appended_left_inputs)
expression_actions.addInput(name, std::make_shared<DataTypeUInt64>(), 0);
auto actions_dag = expression_actions.getActionsDAG();
for (const auto * input : actions_dag->getInputs())
actions_dag->getOutputs().push_back(input);
QueryPlanSerializationSettings settings;
return std::make_unique<JoinStepLogical>(
left_header,
right_header,
JoinOperator{},
std::move(expression_actions),
ActionsDAG::NodeRawConstPtrs{},
JoinSettings(settings, DBMS_QUERY_PLAN_SERIALIZATION_VERSION),
SortingStep::Settings(settings));
}
std::vector<size_t> allPositions(const IQueryPlanStep & step)
{
std::vector<size_t> positions(step.getOutputHeader()->columns());
for (size_t position = 0; position < positions.size(); ++position)
positions[position] = position;
return positions;
}
}
/// The right side's only input comes right after the first left one, where the left header width does not point.
/// Nothing reads any output, so the join keeps the first column of each side, and only the appended left one goes.
TEST(JoinStepLogicalUnusedColumns, KeepsAColumnOfEachSideWhenInputsAreNotLeftThenRight)
{
const auto join = makeJoin({"a"}, {"c"}, {"d"});
ASSERT_TRUE(join->canRemoveUnusedColumns());
const auto unneeded = join->getUnneededColumns(allPositions(*join));
ASSERT_EQ(unneeded.size(), 2u);
EXPECT_EQ(unneeded[0], std::vector<size_t>({1}));
EXPECT_TRUE(unneeded[1].empty());
}
/// The only left input comes after the right one, so the first input of the join is a right one.
TEST(JoinStepLogicalUnusedColumns, KeepsTheLeftColumnWhenTheFirstInputIsARightOne)
{
const auto join = makeJoin({}, {"c"}, {"d"});
ASSERT_TRUE(join->canRemoveUnusedColumns());
const auto unneeded = join->getUnneededColumns(allPositions(*join));
ASSERT_EQ(unneeded.size(), 2u);
EXPECT_TRUE(unneeded[0].empty());
EXPECT_TRUE(unneeded[1].empty());
}
src/Processors/tests/gtest_expression_step_required_columns.cpp
+300
View File
@@ -0,0 +1,300 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <Columns/ColumnsNumber.h>
#include <Core/Block.h>
#include <DataTypes/DataTypesNumber.h>
#include <Interpreters/ActionsDAG.h>
#include <Interpreters/ExpressionActions.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/FilterStep.h>
#include <fmt/ranges.h>
using namespace DB;
namespace
{
/// The sorted positions in `[0, count)` that are not in the sorted `positions`.
std::vector<size_t> complementPositions(size_t count, const std::vector<size_t> & positions)
{
std::vector<size_t> complement;
for (size_t position = 0; position < count; ++position)
if (!std::ranges::binary_search(positions, position))
complement.push_back(position);
return complement;
}
ColumnWithTypeAndName column(const String & name)
{
auto type = std::make_shared<DataTypeUInt64>();
return {type->createColumn(), type, name};
}
/// Input header (a, b, c) and a DAG reading a and b, so c is a pass-through.
/// Output header is (x, y, c).
std::unique_ptr<ExpressionStep> makeStep()
{
auto input_header = std::make_shared<const Block>(Block{column("a"), column("b"), column("c")});
ActionsDAG dag;
const auto & a = dag.addInput(column("a"));
const auto & b = dag.addInput(column("b"));
dag.getOutputs() = {&dag.addAlias(a, "x"), &dag.addAlias(b, "y")};
return std::make_unique<ExpressionStep>(input_header, std::move(dag));
}
/// Two header columns share a name and the DAG reads both, so a position cannot be recovered from the
/// name alone. The second input is the one that survives below.
std::unique_ptr<ExpressionStep> makeStepWithDuplicateNames()
{
auto input_header = std::make_shared<const Block>(Block{column("d"), column("d")});
ActionsDAG dag;
const auto & first = dag.addInput(column("d"));
const auto & second = dag.addInput(column("d"));
dag.getOutputs() = {&dag.addAlias(first, "first"), &dag.addAlias(second, "second")};
return std::make_unique<ExpressionStep>(input_header, std::move(dag));
}
/// Input header (a, b, c) filtered by `f`, computed from a. The DAG outputs are (f, x) where x aliases
/// b, so c is a pass-through. `remove_filter_column` decides whether f shows up in the output header.
std::unique_ptr<FilterStep> makeFilterStep(bool remove_filter_column)
{
auto input_header = std::make_shared<const Block>(Block{column("a"), column("b"), column("c")});
ActionsDAG dag;
const auto & a = dag.addInput(column("a"));
const auto & b = dag.addInput(column("b"));
dag.getOutputs() = {&dag.addAlias(a, "f"), &dag.addAlias(b, "x")};
return std::make_unique<FilterStep>(input_header, std::move(dag), "f", remove_filter_column);
}
/// A child that kept `positions` of `header`, and appended `appended` after them.
IQueryPlanStep::PrunedInput prunedChild(const Block & header, const std::vector<size_t> & positions, const ColumnsWithTypeAndName & appended = {})
{
Block pruned_header;
for (size_t position : positions)
pruned_header.insert(header.getByPosition(position));
for (const auto & column : appended)
pruned_header.insert(column);
return {complementPositions(header.columns(), positions), std::make_shared<const Block>(std::move(pruned_header))};
}
/// The columns at `positions` of `header`, by name.
String namesAt(const Block & header, const std::vector<size_t> & positions)
{
Names names;
for (size_t position : positions)
names.push_back(header.getByPosition(position).name);
return fmt::format("{}", fmt::join(names, ", "));
}
/// Runs `dag` on one row the way `ExpressionTransform` does, with `values` in the columns of `header`, in order.
/// Returns the values of the result columns, in order.
std::vector<UInt64> runOnOneRow(const ActionsDAG & dag, const Block & header, const std::vector<UInt64> & values)
{
ExpressionActions actions(dag.clone());
Columns columns;
for (UInt64 value : values)
columns.push_back(ColumnUInt64::create(1, value));
size_t num_rows = 1;
std::vector<UInt64> result;
for (const auto & column : actions.executeOnColumns(std::move(columns), header, actions.getInputPositions(header), num_rows))
result.push_back(column->getUInt(0));
return result;
}
/// Tells the step that only the columns at `required_output_positions` are needed, asks it what it does not
/// need, prunes it with a child that drops exactly that, and checks that the step outputs exactly the
/// required columns and reads exactly what it did not give up.
template <typename Step>
void checkPruning(const std::function<std::unique_ptr<Step>()> & make_step, const std::vector<size_t> & required_output_positions)
{
const auto step = make_step();
const auto before = step->getOutputHeader()->dumpStructure();
const auto dag_before = step->getExpression().dumpDAG();
const auto output_before = *step->getOutputHeader();
const auto input_header = *step->getInputHeaders().front();
const auto unneeded_output_positions = complementPositions(output_before.columns(), required_output_positions);
const auto unneeded = step->getUnneededColumns(unneeded_output_positions);
/// The question must not change the step.
EXPECT_EQ(step->getOutputHeader()->dumpStructure(), before);
EXPECT_EQ(step->getExpression().dumpDAG(), dag_before);
ASSERT_EQ(unneeded.size(), 1u);
const auto required_inputs = complementPositions(input_header.columns(), unneeded.front());
const auto applied = step->removeUnusedColumns(unneeded_output_positions, {prunedChild(input_header, required_inputs)});
EXPECT_EQ(applied.dropped_output_positions, unneeded_output_positions);
EXPECT_EQ(applied.step_changed, !unneeded_output_positions.empty());
EXPECT_EQ(step->getInputHeaders().front()->dumpNames(), namesAt(input_header, required_inputs));
EXPECT_EQ(step->getOutputHeader()->dumpNames(), namesAt(output_before, required_output_positions));
}
}
TEST(ExpressionStepRequiredColumns, PrunesToTheRequiredColumns)
{
/// Keep one DAG output, the other output and the pass-through go away.
checkPruning<ExpressionStep>(makeStep, {0});
/// Keep a DAG output and the pass-through.
checkPruning<ExpressionStep>(makeStep, {0, 2});
/// Keep the pass-through only.
checkPruning<ExpressionStep>(makeStep, {2});
/// Keep everything: nothing to do.
checkPruning<ExpressionStep>(makeStep, {0, 1, 2});
checkPruning<ExpressionStep>(makeStepWithDuplicateNames, {0});
checkPruning<ExpressionStep>(makeStepWithDuplicateNames, {1});
}
TEST(FilterStepRequiredColumns, PrunesToTheRequiredColumns)
{
/// Output header is (x, c): the filter column is erased, so the caller's positions have to be mapped
/// back over it.
const auto make_without_filter_column = [] { return makeFilterStep(/*remove_filter_column=*/true); };
checkPruning<FilterStep>(make_without_filter_column, {0});
checkPruning<FilterStep>(make_without_filter_column, {1});
checkPruning<FilterStep>(make_without_filter_column, {0, 1});
/// Output header is (f, x, c). Not asking for f makes the step drop it from the header, while the DAG
/// still has to compute it to filter.
const auto make_with_filter_column = [] { return makeFilterStep(/*remove_filter_column=*/false); };
checkPruning<FilterStep>(make_with_filter_column, {0});
checkPruning<FilterStep>(make_with_filter_column, {1});
checkPruning<FilterStep>(make_with_filter_column, {2});
checkPruning<FilterStep>(make_with_filter_column, {0, 1, 2});
}
TEST(FilterStepRequiredColumns, KeepsTheFilterInputAndDropsTheRest)
{
const auto step = makeFilterStep(/*remove_filter_column=*/true);
const auto input_header = *step->getInputHeaders().front();
/// Output header is (x, c). Without x, b is not needed, but a is, because the filter reads it.
const auto unneeded = step->getUnneededColumns({0});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_EQ(unneeded.front(), std::vector<size_t>({1}));
step->removeUnusedColumns({0}, {prunedChild(input_header, {0, 2})});
EXPECT_EQ(step->getOutputHeader()->dumpNames(), "c");
EXPECT_EQ(step->getInputHeaders().front()->dumpNames(), "a, c");
}
TEST(ExpressionStepRequiredColumns, ReportsThePositionEachInputReads)
{
const auto step = makeStep();
/// Output header is (x, y, c). Output 1 is an alias of b, which reads header position 1, so without the
/// other outputs a and c are not needed.
const auto unneeded = step->getUnneededColumns({0, 2});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_EQ(unneeded.front(), std::vector<size_t>({0, 2}));
}
TEST(ExpressionStepRequiredColumns, DuplicateNamesKeepTheirOwnPosition)
{
const auto step = makeStepWithDuplicateNames();
/// Output 1 is an alias of the second input, which reads header position 1, so without output 0 the
/// first input is not needed. Resolving the surviving input by name would answer 1 and feed the
/// expression the wrong column.
const auto unneeded = step->getUnneededColumns({0});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_EQ(unneeded.front(), std::vector<size_t>({0}));
}
/// Asked for every output, the step needs every input, and pruning it with a child that did not change
/// changes nothing.
TEST(ExpressionStepRequiredColumns, ChangesNothingWhenEverythingIsRequired)
{
const auto step = makeStep();
const auto input_header = step->getInputHeaders().front();
const auto unneeded = step->getUnneededColumns({});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_TRUE(unneeded.front().empty());
const auto applied = step->removeUnusedColumns({}, {IQueryPlanStep::PrunedInput::unchanged(input_header)});
EXPECT_FALSE(applied.step_changed);
EXPECT_TRUE(applied.dropped_output_positions.empty());
}
/// A child can keep more than it was asked for - a FINAL read keeps its sorting key - and append columns
/// of its own - a join its dummy column. The step consumes them, and outputs the required columns only.
TEST(ExpressionStepRequiredColumns, ConsumesWhatTheChildKeepsBeyondTheAsk)
{
const auto step = makeStep();
const auto input_header = *step->getInputHeaders().front();
/// Without y and c, the step needs only a; the child keeps a, b and c, and appends z.
const auto unneeded = step->getUnneededColumns({1, 2});
EXPECT_EQ(unneeded.front(), std::vector<size_t>({1, 2}));
const auto applied = step->removeUnusedColumns({1, 2}, {prunedChild(input_header, {0, 1, 2}, {column("z")})});
EXPECT_TRUE(applied.step_changed);
EXPECT_EQ(step->getInputHeaders().front()->dumpNames(), "a, b, c, z");
EXPECT_EQ(step->getOutputHeader()->dumpNames(), "x");
}
TEST(FilterStepRequiredColumns, ConsumesWhatTheChildKeepsBeyondTheAsk)
{
const auto step = makeFilterStep(/*remove_filter_column=*/true);
const auto input_header = *step->getInputHeaders().front();
/// Output header is (x, c). Without c, the step needs a and b; the child keeps c as well.
const auto unneeded = step->getUnneededColumns({1});
EXPECT_EQ(unneeded.front(), std::vector<size_t>({2}));
step->removeUnusedColumns({1}, {prunedChild(input_header, {0, 1, 2})});
EXPECT_EQ(step->getInputHeaders().front()->dumpNames(), "a, b, c");
EXPECT_EQ(step->getOutputHeader()->dumpNames(), "x");
}
/// Input header (a, d, d) where only a is read, so both `d` pass through. Inputs are paired with header columns by
/// name, in order, so a column consumed by a new input is the first one of its name that no input reads. Here that is
/// the needed one: the step must not drop the second `d`, or the first one would be consumed in its place.
TEST(ExpressionStepRequiredColumns, KeepsAPassThroughColumnAfterANeededOneOfTheSameName)
{
auto input_header = std::make_shared<const Block>(Block{column("a"), column("d"), column("d")});
ActionsDAG dag;
dag.getOutputs() = {&dag.addAlias(dag.addInput(column("a")), "x")};
ExpressionStep step(input_header, std::move(dag));
/// Output header is (x, d, d). Nobody needs the second `d`, but the step keeps it.
const auto unneeded = step.getUnneededColumns({2});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_TRUE(unneeded.front().empty());
const auto applied = step.removeUnusedColumns({2}, {prunedChild(*input_header, {0, 1, 2})});
EXPECT_TRUE(applied.dropped_output_positions.empty());
EXPECT_EQ(runOnOneRow(step.getExpression(), *step.getInputHeaders().front(), {7, 1, 2}), std::vector<UInt64>({7, 1, 2}));
}
/// The same for a filter, whose output header has the filter column erased from it.
TEST(FilterStepRequiredColumns, KeepsAPassThroughColumnAfterANeededOneOfTheSameName)
{
auto input_header = std::make_shared<const Block>(Block{column("a"), column("d"), column("d")});
ActionsDAG dag;
dag.getOutputs() = {&dag.addAlias(dag.addInput(column("a")), "f")};
FilterStep step(input_header, std::move(dag), "f", /*remove_filter_column=*/true);
/// Output header is (d, d). Nobody needs the second `d`, but the step keeps it.
const auto unneeded = step.getUnneededColumns({1});
ASSERT_EQ(unneeded.size(), 1u);
EXPECT_TRUE(unneeded.front().empty());
const auto applied = step.removeUnusedColumns({1}, {prunedChild(*input_header, {0, 1, 2})});
EXPECT_TRUE(applied.dropped_output_positions.empty());
/// The DAG still outputs the filter column `f` first.
EXPECT_EQ(runOnOneRow(step.getExpression(), *step.getInputHeaders().front(), {7, 1, 2}), std::vector<UInt64>({7, 1, 2}));
}
src/Processors/tests/gtest_remove_unused_columns_modes.cpp
+163
View File
@@ -0,0 +1,163 @@
#include <gtest/gtest.h>
#include <Core/Block.h>
#include <DataTypes/DataTypesNumber.h>
#include <Interpreters/ActionsDAG.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/OffsetStep.h>
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ReadNothingStep.h>
using namespace DB;
using namespace DB::QueryPlanOptimizations;
namespace
{
SharedHeader makeHeader(const Names & names)
{
Block header;
for (const auto & name : names)
{
auto type = std::make_shared<DataTypeUInt64>();
header.insert({type->createColumn(), type, name});
}
return std::make_shared<const Block>(std::move(header));
}
/// A plan built bottom-up, each step over the one added before it.
struct Chain
{
QueryPlan::Nodes nodes;
QueryPlan::Node & top() { return nodes.back(); }
void addSource(const Names & names)
{
nodes.emplace_back().step = std::make_unique<ReadNothingStep>(makeHeader(names));
}
/// An expression that outputs the columns `outputs` of the step below. Every column below is an input of its DAG,
/// so nothing passes through, and a column not in `outputs` is read and not needed.
void addProjection(const Names & outputs)
{
const auto input_header = top().step->getOutputHeader();
ActionsDAG dag;
std::unordered_map<String, const ActionsDAG::Node *> inputs;
for (const auto & column : *input_header)
inputs[column.name] = &dag.addInput(column.name, column.type);
for (const auto & name : outputs)
dag.getOutputs().push_back(inputs.at(name));
add(std::make_unique<ExpressionStep>(input_header, std::move(dag)));
}
/// A step that cannot drop columns.
void addOffset() { add(std::make_unique<OffsetStep>(top().step->getOutputHeader(), 1)); }
/// The names of the output header of the step `depth` steps below the top.
String outputAt(size_t depth)
{
auto it = nodes.rbegin();
std::advance(it, depth);
return it->step->getOutputHeader()->dumpNames();
}
private:
void add(QueryPlanStepPtr step)
{
auto * child = &top();
auto & node = nodes.emplace_back();
node.step = std::move(step);
node.children = {child};
}
};
}
/// Each step leaves the columns b and c of the step below unneeded, so the walk goes down to the read. The read cannot
/// drop columns, and the step above it consumes the ones it keeps.
TEST(RemoveUnusedColumnsModes, LocalWalksDownWhileSomethingIsUnneeded)
{
Chain chain;
chain.addSource({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addProjection({"a"});
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Local), 3u);
EXPECT_EQ(chain.outputAt(0), "a");
EXPECT_EQ(chain.outputAt(1), "a");
EXPECT_EQ(chain.outputAt(2), "a");
EXPECT_EQ(chain.outputAt(3), "a, b, c");
/// Nothing is left to remove.
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Local), 0u);
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Global), 0u);
}
/// The top step needs all of the step below it, so the local walk stops there, while the step below leaves columns of
/// its own child unread. That is the business of the local walk that starts at that step, or of the global one.
TEST(RemoveUnusedColumnsModes, LocalStopsWhereNothingIsUnneeded)
{
{
Chain chain;
chain.addSource({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addProjection({"a"});
chain.addProjection({"a"});
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Local), 0u);
EXPECT_EQ(chain.outputAt(2), "a, b, c");
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Global), 3u);
EXPECT_EQ(chain.outputAt(2), "a");
}
{
Chain chain;
chain.addSource({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addProjection({"a"});
auto & middle = chain.top();
chain.addProjection({"a"});
/// Started at the step that reads only a, the local walk removes b and c below it.
EXPECT_EQ(removeUnusedColumns(middle, RemoveUnusedColumnsMode::Local), 2u);
EXPECT_EQ(chain.outputAt(2), "a");
}
}
/// A step that cannot drop columns needs everything of its children, so a local walk from it has nothing to do. The
/// global walk goes on below it.
TEST(RemoveUnusedColumnsModes, LocalDoesNothingIfTheRootCannotPrune)
{
Chain chain;
chain.addSource({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addProjection({"a"});
chain.addOffset();
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Local), 0u);
EXPECT_EQ(chain.outputAt(2), "a, b, c");
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Global), 3u);
EXPECT_EQ(chain.outputAt(2), "a");
}
/// The step below cannot drop the columns the top step does not need, so the top step keeps consuming them and does
/// not change.
TEST(RemoveUnusedColumnsModes, LocalStopsAtAChildThatCannotPrune)
{
Chain chain;
chain.addSource({"a", "b", "c"});
chain.addProjection({"a", "b", "c"});
chain.addOffset();
chain.addProjection({"a"});
EXPECT_EQ(removeUnusedColumns(chain.top(), RemoveUnusedColumnsMode::Local), 0u);
EXPECT_EQ(chain.outputAt(0), "a");
EXPECT_EQ(chain.outputAt(1), "a, b, c");
EXPECT_EQ(chain.outputAt(2), "a, b, c");
}
tests/queries/0_stateless/03545_analyzer_correlated_subquery_exists_asterisk.reference
+10 -17
View File
@@ -1,7 +1,7 @@
Expression ((Project names + (Projection + WHERE)))
Actions: INPUT :: 0 -> __join_result_dummy UInt8 : 0
COLUMN Const(UInt8) -> 1 UInt8 : 1
Positions: 1
Actions: COLUMN Const(UInt8) -> 1 UInt8 : 0
INPUT :: 0 -> __join_result_dummy UInt8 : 1
Positions: 0
Expression (Post Join Actions)
Actions: INPUT :: 0 -> __join_result_dummy UInt8 : 0
Positions: 0
@@ -24,15 +24,11 @@ Positions: 1
Granules: 1
Expression (Right Pre Join Actions)
Actions: INPUT :: 0 -> exists(__table2).__table1.i1 Int64 : 0
INPUT :: 1 -> exists(__table2) UInt8 : 1
Positions: 0 1
Positions: 0
Expression ((Create result for always true EXISTS expression + ))
Actions: INPUT : 0 -> __table1.i1 Int64 : 0
COLUMN Const(UInt8) -> exists(__table2) UInt8 : 1
ALIAS __table1.i1 :: 0 -> exists(__table2).__table1.i1 Int64 : 2
FUNCTION materialize(exists(__table2) :: 1) -> materialize(exists(__table2)) UInt8 : 0
ALIAS materialize(exists(__table2)) :: 0 -> exists(__table2) UInt8 : 1
Positions: 2 1
ALIAS __table1.i1 :: 0 -> exists(__table2).__table1.i1 Int64 : 1
Positions: 1
Expression (Post Join Actions)
Actions: INPUT :: 0 -> __table1.i1 Int64 : 0
Positions: 0
@@ -59,14 +55,11 @@ Positions: 1
Granules: 1
Expression (Right Pre Join Actions)
Actions: INPUT :: 0 -> __table4.i2 Int64 : 0
INPUT :: 1 -> __table4.i1 Int64 : 1
Positions: 0 1
Positions: 0
Expression (Change column names to column identifiers)
Actions: INPUT : 0 -> i1 Int64 : 0
INPUT : 1 -> i2 Int64 : 1
ALIAS i1 :: 0 -> __table4.i1 Int64 : 2
ALIAS i2 :: 1 -> __table4.i2 Int64 : 0
Positions: 2 0
Actions: INPUT : 0 -> i2 Int64 : 0
ALIAS i2 :: 0 -> __table4.i2 Int64 : 1
Positions: 1
ReadFromMergeTree (default.test)
ReadType: Default
Parts: 1
tests/queries/0_stateless/03567_max_joined_block_size_bytes.reference
+16 -16
View File
@@ -25,7 +25,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -45,7 +45,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -140,7 +140,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -160,7 +160,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -255,7 +255,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -275,7 +275,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -370,7 +370,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -390,7 +390,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -485,7 +485,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -505,7 +505,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -600,7 +600,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -620,7 +620,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -715,7 +715,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -735,7 +735,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -830,7 +830,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = 'parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -850,7 +850,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='parallel_hash';
ok
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
tests/queries/0_stateless/03567_max_joined_block_size_bytes.sql.j2
+4 -2
View File
@@ -1,4 +1,6 @@
-- Tags: long, no-tsan, no-asan, no-msan, no-ubsan
-- The small blocks below come from the heavy column `v` passing through the join. Nothing else reads it above the
-- join, so `NOT ignore(v)` does, or removing unused columns would take it out of the join.
SET max_block_size = 65536;
SET min_joined_block_size_rows = 65536;
SET min_joined_block_size_bytes = 512000;
@@ -38,7 +40,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '10Mi', join_algorithm = '{{ join_algorithm }}';
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
@@ -58,7 +60,7 @@ FROM
)
SETTINGS max_joined_block_size_bytes = '1Mi', join_algorithm='{{ join_algorithm }}';
SELECT DISTINCT if (blockSize() as bs < 1000, 'ok', toString(bs))
SELECT DISTINCT if (blockSize() as bs < 1000 AND NOT ignore(v), 'ok', toString(bs))
FROM
(
SELECT v
tests/queries/0_stateless/03595_convert_any_join_to_semi_or_anti.reference
+6 -14
View File
@@ -30,15 +30,11 @@ Positions: 0
ReadFromMemoryStorage
Expression (Right Pre Join Actions)
Actions: INPUT :: 0 -> exists(__table2).__table1.uid Int16 : 0
INPUT :: 1 -> exists(__table2) UInt8 : 1
Positions: 0 1
Positions: 0
Expression (Create result for always true EXISTS expression)
Actions: INPUT : 0 -> __table1.uid Int16 : 0
COLUMN Const(UInt8) -> exists(__table2) UInt8 : 1
ALIAS __table1.uid :: 0 -> exists(__table2).__table1.uid Int16 : 2
FUNCTION materialize(exists(__table2) :: 1) -> materialize(exists(__table2)) UInt8 : 0
ALIAS materialize(exists(__table2)) :: 0 -> exists(__table2) UInt8 : 1
Positions: 2 1
ALIAS __table1.uid :: 0 -> exists(__table2).__table1.uid Int16 : 1
Positions: 1
Filter (WHERE)
Filter column: notEquals(__table1.uid, __table4.uid) (removed)
Actions: INPUT : 0 -> __table4.uid Int16 : 0
@@ -106,15 +102,11 @@ Positions: 0
ReadFromMemoryStorage
Expression (Right Pre Join Actions)
Actions: INPUT :: 0 -> exists(__table2).__table1.uid Int16 : 0
INPUT :: 1 -> exists(__table2) UInt8 : 1
Positions: 0 1
Positions: 0
Expression (Create result for always true EXISTS expression)
Actions: INPUT : 0 -> __table1.uid Int16 : 0
COLUMN Const(UInt8) -> exists(__table2) UInt8 : 1
ALIAS __table1.uid :: 0 -> exists(__table2).__table1.uid Int16 : 2
FUNCTION materialize(exists(__table2) :: 1) -> materialize(exists(__table2)) UInt8 : 0
ALIAS materialize(exists(__table2)) :: 0 -> exists(__table2) UInt8 : 1
Positions: 2 1
ALIAS __table1.uid :: 0 -> exists(__table2).__table1.uid Int16 : 1
Positions: 1
Filter (WHERE)
Filter column: notEquals(__table1.uid, __table4.uid) (removed)
Actions: INPUT : 0 -> __table4.uid Int16 : 0
tests/queries/0_stateless/03836_tpch_join_order_plans.reference
+7 -8
View File
@@ -474,14 +474,13 @@ Expression (Project names)
Expression ((((((WHERE + (Renaming correlated columns to equivalent expressions in subquery + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))))) + WHERE) + (Create renaming actions for scalar subquery + (Create correlated subquery result alias + (Project names + Projection)))) + Before GROUP BY) + (WHERE + (Renaming correlated columns to equivalent expressions in subquery + Change column names to column )
ShuffleExchange
ReadFromMergeTree (ParallelRead default.lineitem)
Expression
ShuffleExchange
JoinLogical (Broadcast HashJoin swapped)
Expression (((((WHERE + (Renaming correlated columns to equivalent expressions in subquery + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))))) + WHERE) + WHERE) + Change column names to column identifiers))
ReadFromMergeTree (ParallelRead default.partsupp)
Expression (Create result for always true EXISTS expression)
Expression (((WHERE + (Renaming correlated columns to equivalent expressions in subquery + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))))) + (WHERE + Change column names to column identifiers)))
ReadFromMergeTree (ReplicatedRead default.part)
ShuffleExchange
JoinLogical (Broadcast HashJoin swapped)
Expression (((((WHERE + (Renaming correlated columns to equivalent expressions in subquery + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))))) + WHERE) + WHERE) + Change column names to column identifiers))
ReadFromMergeTree (ParallelRead default.partsupp)
Expression (Create result for always true EXISTS expression)
Expression (((WHERE + (Renaming correlated columns to equivalent expressions in subquery + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))))) + (WHERE + Change column names to column identifiers)))
ReadFromMergeTree (ReplicatedRead default.part)
-- Q21
Expression (Project names)
Limit (preliminary LIMIT)
tests/queries/0_stateless/04825_block_nested_loop_join_block_size.sql
+9 -6
View File
@@ -23,6 +23,9 @@ CREATE TABLE bnl_bs_build (y UInt64, t String) ENGINE = MergeTree ORDER BY y;
INSERT INTO bnl_bs_probe SELECT number, repeat('a', 1000) FROM numbers(500);
INSERT INTO bnl_bs_build SELECT number, repeat('b', 1000) FROM numbers(500);
-- The checks of the byte limit also read the wide column above the join, so that it is not removed as unused
-- and the rows the join emits stay a kilobyte wide.
--
-- Neither limit set: the whole result of a probe chunk goes out in as few blocks as `max_block_size`
-- allows, so the blocks are far larger than either limit below would let them be.
SELECT 'matched, unlimited', max(bs) > 10000
@@ -36,8 +39,8 @@ FROM (SELECT blockSize() AS bs FROM (
SETTINGS max_joined_block_size_rows = 111, max_joined_block_size_bytes = 0;
-- A row is about a kilobyte wide, so a budget of 10 KB is reached long before 100000 rows are.
SELECT 'matched, bytes', max(bs) < 1000
FROM (SELECT blockSize() AS bs FROM (
SELECT 'matched, bytes', max(bs) < 1000 AND min(length(s)) = 1000
FROM (SELECT blockSize() AS bs, s FROM (
SELECT l.s, r.y FROM bnl_bs_probe l LEFT JOIN bnl_bs_build r ON l.x < r.y))
SETTINGS max_joined_block_size_rows = 100000, max_joined_block_size_bytes = 10000;
@@ -47,8 +50,8 @@ FROM (SELECT blockSize() AS bs FROM (
SELECT l.s FROM bnl_bs_probe l LEFT JOIN bnl_bs_build r ON l.x + 1000 < r.y))
SETTINGS max_joined_block_size_rows = 0, max_joined_block_size_bytes = 0;
SELECT 'unmatched probe rows, bytes', max(bs) < 100
FROM (SELECT blockSize() AS bs FROM (
SELECT 'unmatched probe rows, bytes', max(bs) < 100 AND min(length(s)) = 1000
FROM (SELECT blockSize() AS bs, s FROM (
SELECT l.s FROM bnl_bs_probe l LEFT JOIN bnl_bs_build r ON l.x + 1000 < r.y))
SETTINGS max_joined_block_size_rows = 0, max_joined_block_size_bytes = 10000;
@@ -57,8 +60,8 @@ FROM (SELECT blockSize() AS bs FROM (
SELECT r.t FROM bnl_bs_probe l RIGHT JOIN bnl_bs_build r ON l.x > r.y + 1000))
SETTINGS max_joined_block_size_rows = 0, max_joined_block_size_bytes = 0;
SELECT 'unmatched build rows, bytes', max(bs) < 100
FROM (SELECT blockSize() AS bs FROM (
SELECT 'unmatched build rows, bytes', max(bs) < 100 AND min(length(t)) = 1000
FROM (SELECT blockSize() AS bs, t FROM (
SELECT r.t FROM bnl_bs_probe l RIGHT JOIN bnl_bs_build r ON l.x > r.y + 1000))
SETTINGS max_joined_block_size_rows = 0, max_joined_block_size_bytes = 10000;
tests/queries/0_stateless/04864_json_prefix_deserialization_marks_load_not_serialized.sh
+6 -5
View File
@@ -77,12 +77,13 @@ SYSTEM ENABLE FAILPOINT merge_tree_marks_load_sync_sleep;
-- Three arms. 'async' additionally reaches the carriers that exist only while a marks load can
-- still be in flight; 'prefetch' additionally executes the prefix prefetch callback. Only 'async'
-- carries the overlap verdict at the end of this query; the other two run for the code paths they
-- reach and for the deterministic checks below.
SELECT count() FROM (SELECT j FROM t LIMIT 1)
-- reach and for the deterministic checks below. NOT ignore(j) reads the column above the LIMIT:
-- without it nothing does, and removing unused columns would not read the column at all.
SELECT count() FROM (SELECT j FROM t LIMIT 1) WHERE NOT ignore(j)
SETTINGS merge_tree_use_prefixes_deserialization_thread_pool = 1, log_comment = 'sync'
FORMAT Null;
SELECT count() FROM (SELECT j FROM t LIMIT 1)
SELECT count() FROM (SELECT j FROM t LIMIT 1) WHERE NOT ignore(j)
SETTINGS merge_tree_use_prefixes_deserialization_thread_pool = 1, load_marks_asynchronously = 1,
log_comment = 'async'
FORMAT Null;
@@ -90,7 +91,7 @@ FORMAT Null;
-- The read method is pinned because a synchronous one turns ReadBuffer::prefetch into a no-op, and
-- the effective default is only conditionally 'pread_threadpool': applySettingsQuirks downgrades it
-- to 'pread' on a host where preadv2(RWF_NOWAIT) is unavailable, without marking it changed.
SELECT count() FROM (SELECT j FROM t LIMIT 1)
SELECT count() FROM (SELECT j FROM t LIMIT 1) WHERE NOT ignore(j)
SETTINGS merge_tree_use_prefixes_deserialization_thread_pool = 1, load_marks_asynchronously = 1,
local_filesystem_read_prefetch = 1, local_filesystem_read_method = 'pread_threadpool',
enable_filesystem_read_prefetches_log = 1, log_comment = 'prefetch'
@@ -168,7 +169,7 @@ retry_async_arm() {
measurement=$(${CLICKHOUSE_LOCAL} --config-file "$TD/config.xml" -q "
SYSTEM ENABLE FAILPOINT merge_tree_marks_load_sync_sleep;
SELECT count() FROM (SELECT j FROM t LIMIT 1)
SELECT count() FROM (SELECT j FROM t LIMIT 1) WHERE NOT ignore(j)
SETTINGS merge_tree_use_prefixes_deserialization_thread_pool = 1, load_marks_asynchronously = 1
FORMAT Null;
tests/queries/0_stateless/05291_remove_unused_columns_through_limit.reference
+14
View File
@@ -0,0 +1,14 @@
Output: count()
Header: count() UInt64
Header:
Header:
Output: Empty
500
200
Header: count() UInt64
Limit (LIMIT WITH TIES)
Header: __table2.b UInt64
Header: __table2.b UInt64
Header: a UInt64
b UInt64
100
tests/queries/0_stateless/05291_remove_unused_columns_through_limit.sql
+24
View File
@@ -0,0 +1,24 @@
-- `LIMIT` passes every column through, so the columns nobody reads above it are removed below it
-- too, down to the read. `WITH TIES` still needs the columns it compares rows by.
SET enable_analyzer = 1;
SET query_plan_remove_unused_columns = 1;
-- The plan is checked on one node.
SET enable_parallel_replicas = 0;
DROP TABLE IF EXISTS t_limit_unused;
CREATE TABLE t_limit_unused (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a;
INSERT INTO t_limit_unused SELECT number, number % 10 FROM numbers(1000);
-- Nothing above the `LIMIT` reads a column, so the read reads none.
SELECT replaceRegexpOne(explain, '^[│└─ ]+', '') FROM (EXPLAIN header = 1 SELECT count() FROM (SELECT 1 FROM t_limit_unused LIMIT 500))
WHERE explain LIKE '%Header:%' OR explain LIKE '%Output:%';
SELECT count() FROM (SELECT 1 FROM t_limit_unused LIMIT 500);
SELECT count() FROM (SELECT a FROM t_limit_unused LIMIT 200 OFFSET 100);
-- `WITH TIES` keeps the sorting column, and the ties are all counted.
SELECT replaceRegexpOne(explain, '^[│└─ ]+', '') FROM (EXPLAIN header = 1 SELECT count() FROM (SELECT a FROM t_limit_unused ORDER BY b LIMIT 3 WITH TIES))
WHERE explain LIKE '%Limit (%' OR explain LIKE '%Header:%' OR explain LIKE '%b UInt64%';
SELECT count() FROM (SELECT a FROM t_limit_unused ORDER BY b LIMIT 3 WITH TIES);
DROP TABLE t_limit_unused;
tests/queries/0_stateless/05292_split_filter_renamed_filter_column_dangling_input.reference
+4
View File
@@ -0,0 +1,4 @@
1
1
tests/queries/0_stateless/05292_split_filter_renamed_filter_column_dangling_input.sql
+13
View File
@@ -0,0 +1,13 @@
-- The constant filter column `NULL` has the same name as the aggregation key it is computed over, so splitting
-- the filter renames it, and the expression above read it through an input with no column in the header. Removing
-- unused columns rejected that input with a logical error. Found by the AST fuzzer.
SET enable_analyzer = 1;
SET query_plan_merge_expressions = 1;
SET query_plan_split_filter = 1;
SET query_plan_remove_unused_columns = 1;
SELECT x FROM (SELECT arrayJoin([1]) AS x GROUP BY NULL WITH TOTALS) WHERE NULL;
SELECT x FROM (SELECT arrayJoin([1]) AS x GROUP BY NULL WITH TOTALS) WHERE NULL
INTERSECT DISTINCT SELECT DISTINCT 1;