[clang-repl] Split incremental executor builders by backend (#230047)

Address
https://github.com/llvm/llvm-project/pull/175910#discussion_r4208494365

Refactoring pr. The tests pass on both backends.
This commit is contained in:
Anutosh Bhat authored and GitHub committed 2026-10-22 19:43:33 +00:00
1 parent bb31003cf3
commit 254a47698e
12 files changed
+257 -100

No files matched your search

@@ -13,67 +13,66 @@
#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H
#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include <memory>
#include <string>
#include <vector>
namespace llvm {
class Triple;
namespace orc {
class ExecutorAddr;
class LLJITBuilder;
class ThreadSafeContext;
} // namespace orc
} // namespace llvm
namespace clang {
class IncrementalExecutor;
class IncrementalCompilerBuilder;
class TargetInfo;
namespace driver {
class Compilation;
} // namespace driver
/// Common configuration and interface for incremental executor builders.
class IncrementalExecutorBuilder {
public:
/// Indicates whether out-of-process JIT execution is enabled.
bool IsOutOfProcess = false;
/// Path to the out-of-process JIT executor.
std::string OOPExecutor = "";
std::string OOPExecutorConnect = "";
/// Indicates whether to use shared memory for communication.
bool UseSharedMemory = false;
/// Representing the slab allocation size for memory management in kb.
unsigned SlabAllocateSize = 0;
/// Path to the ORC runtime library.
std::string OrcRuntimePath = "";
/// PID of the out-of-process JIT executor.
uint32_t ExecutorPID = 0;
/// Custom lambda to be executed inside child process/executor
std::function<void()> CustomizeFork = nullptr;
/// An optional code model to provide to the JITTargetMachineBuilder
std::optional<llvm::CodeModel::Model> CM = std::nullopt;
/// An optional external IncrementalExecutor
/// Options passed to the backend when configuring execution.
struct Options {
bool IsOutOfProcess = false;
std::string ExecutorPath;
std::string RuntimePath;
unsigned SlabAllocateSize = 0;
bool UseSharedMemory = false;
};
/// An optional external IncrementalExecutor.
std::unique_ptr<IncrementalExecutor> IE;
/// mllvm args from the frontend; on wasm these are re-applied after each
/// lldMain call because lld resets all cl options for test-isolation
/// purposes.
/// Frontend -mllvm arguments for backends that need to restore LLVM options.
std::vector<std::string> LLVMArgs;
#ifndef __EMSCRIPTEN__
/// An optional external orc jit builder
std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder;
#endif
/// A default callback that can be used in the IncrementalCompilerBuilder to
/// retrieve the path to the orc runtime.
std::function<llvm::Error(const driver::Compilation &)>
UpdateOrcRuntimePathCB = [this](const driver::Compilation &C) {
return UpdateOrcRuntimePath(C);
};
~IncrementalExecutorBuilder();
virtual ~IncrementalExecutorBuilder();
/// Create the default builder for the platform clangInterpreter is built for.
/// The selected backend provides this definition.
static std::unique_ptr<IncrementalExecutorBuilder> createDefault();
/// Apply execution options and configure the compiler for this backend.
virtual llvm::Error configure(IncrementalCompilerBuilder &CB,
const Options &Opts) = 0;
/// Probe whether this backend can JIT code for the host.
virtual bool supportsJIT() const = 0;
/// Return the host triple used by this backend.
virtual llvm::Expected<llvm::Triple> getHostJITTriple() const = 0;
/// Return the supplied executor, or create one using the selected backend.
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
create(llvm::orc::ThreadSafeContext &TSC, const clang::TargetInfo &TI);
private:
llvm::Error UpdateOrcRuntimePath(const driver::Compilation &C);
virtual llvm::Expected<std::unique_ptr<IncrementalExecutor>>
createExecutor(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) = 0;
};
struct PartialTranslationUnit;
@@ -0,0 +1,78 @@
//===-- OrcIncrementalExecutorBuilder.h - ORC Builder -----------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file declares the builder for ORC incremental execution.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_INTERPRETER_ORCINCREMENTALEXECUTORBUILDER_H
#define LLVM_CLANG_INTERPRETER_ORCINCREMENTALEXECUTORBUILDER_H
#include "clang/Interpreter/IncrementalExecutor.h"
#include "llvm/Support/CodeGen.h"
#include <cstdint>
#include <functional>
#include <optional>
namespace llvm::orc {
class LLJITBuilder;
}
namespace clang {
namespace driver {
class Compilation;
}
/// Configuration for in-process and out-of-process ORC execution.
class OrcIncrementalExecutorBuilder : public IncrementalExecutorBuilder {
public:
/// Indicates whether out-of-process JIT execution is enabled.
bool IsOutOfProcess = false;
/// Path to the out-of-process JIT executor.
std::string OOPExecutor = "";
std::string OOPExecutorConnect = "";
/// Indicates whether to use shared memory for communication.
bool UseSharedMemory = false;
/// Representing the slab allocation size for memory management in kb.
unsigned SlabAllocateSize = 0;
/// Path to the ORC runtime library.
std::string OrcRuntimePath = "";
/// PID of the out-of-process JIT executor.
uint32_t ExecutorPID = 0;
/// Custom lambda to be executed inside child process/executor
std::function<void()> CustomizeFork = nullptr;
/// An optional code model to provide to the JITTargetMachineBuilder
std::optional<llvm::CodeModel::Model> CM = std::nullopt;
/// An optional external ORC JIT builder.
std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder;
/// A default callback that can be used in the IncrementalCompilerBuilder to
/// retrieve the path to the orc runtime.
std::function<llvm::Error(const driver::Compilation &)>
UpdateOrcRuntimePathCB = [this](const driver::Compilation &C) {
return UpdateOrcRuntimePath(C);
};
OrcIncrementalExecutorBuilder();
~OrcIncrementalExecutorBuilder() override;
llvm::Error configure(IncrementalCompilerBuilder &CB,
const Options &Opts) override;
bool supportsJIT() const override;
llvm::Expected<llvm::Triple> getHostJITTriple() const override;
private:
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
createExecutor(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) override;
llvm::Error UpdateOrcRuntimePath(const driver::Compilation &C);
};
} // namespace clang
#endif // LLVM_CLANG_INTERPRETER_ORCINCREMENTALEXECUTORBUILDER_H
+2 -1
View File
@@ -23,7 +23,7 @@ else()
OrcTargetProcess
)
set(EXECUTOR_SOURCES
IncrementalExecutor.cpp
OrcIncrementalExecutorBuilder.cpp
OrcIncrementalExecutor.cpp
)
endif()
@@ -32,6 +32,7 @@ add_clang_library(clangInterpreter
DeviceOffload.cpp
CodeCompletion.cpp
IncrementalAction.cpp
IncrementalExecutorBuilder.cpp
IncrementalParser.cpp
Interpreter.cpp
InterpreterValuePrinter.cpp
@@ -0,0 +1,27 @@
//===-- IncrementalExecutorBuilder.cpp - Executor Builder ------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This implements the common incremental executor builder.
//
//===----------------------------------------------------------------------===//
#include "clang/Interpreter/IncrementalExecutor.h"
namespace clang {
IncrementalExecutorBuilder::~IncrementalExecutorBuilder() = default;
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) {
if (IE)
return std::move(IE);
return createExecutor(TSC, TI);
}
} // namespace clang
+1 -1
View File
@@ -640,7 +640,7 @@ llvm::Error Interpreter::CreateExecutor() {
std::error_code());
if (!IncrExecutorBuilder)
IncrExecutorBuilder = std::make_unique<IncrementalExecutorBuilder>();
IncrExecutorBuilder = IncrementalExecutorBuilder::createDefault();
// Propagate mllvm args so the wasm executor can restore them after each
// lldMain invocation (which resets all cl options for test isolation).
@@ -1,4 +1,4 @@
//===--- IncrementalExecutor.cpp - Incremental Execution --------*- C++ -*-===//
//===-- OrcIncrementalExecutorBuilder.cpp - ORC Builder --------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
//
//===----------------------------------------------------------------------===//
//
// This has the implementation of the base facilities for incremental execution.
// This implements the builder for ORC incremental execution.
//
//===----------------------------------------------------------------------===//
#include "clang/Interpreter/IncrementalExecutor.h"
#include "clang/Interpreter/OrcIncrementalExecutorBuilder.h"
#include "OrcIncrementalExecutor.h"
#include "clang/Basic/TargetInfo.h"
@@ -18,6 +18,7 @@
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Interpreter/Interpreter.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
@@ -81,7 +82,43 @@ static void *getEmuTLSGetAddressPtr() { return nullptr; }
#endif
namespace clang {
IncrementalExecutorBuilder::~IncrementalExecutorBuilder() = default;
OrcIncrementalExecutorBuilder::OrcIncrementalExecutorBuilder() = default;
OrcIncrementalExecutorBuilder::~OrcIncrementalExecutorBuilder() = default;
std::unique_ptr<IncrementalExecutorBuilder>
IncrementalExecutorBuilder::createDefault() {
return std::make_unique<OrcIncrementalExecutorBuilder>();
}
llvm::Error
OrcIncrementalExecutorBuilder::configure(IncrementalCompilerBuilder &CB,
const Options &Opts) {
IsOutOfProcess = Opts.IsOutOfProcess;
OOPExecutor = Opts.ExecutorPath;
OrcRuntimePath = Opts.RuntimePath;
if (OrcRuntimePath.empty())
CB.SetDriverCompilationCallback(UpdateOrcRuntimePathCB);
SlabAllocateSize = Opts.SlabAllocateSize;
UseSharedMemory = Opts.UseSharedMemory;
return llvm::Error::success();
}
bool OrcIncrementalExecutorBuilder::supportsJIT() const {
auto Triple = getHostJITTriple();
if (!Triple) {
llvm::consumeError(Triple.takeError());
return false;
}
return true;
}
llvm::Expected<llvm::Triple>
OrcIncrementalExecutorBuilder::getHostJITTriple() const {
auto J = llvm::orc::LLJITBuilder().create();
if (!J)
return J.takeError();
return (*J)->getTargetTriple();
}
static llvm::Expected<llvm::orc::JITTargetMachineBuilder>
createJITTargetMachineBuilder(const llvm::Triple &TT) {
@@ -321,7 +358,8 @@ createLLJITBuilder(std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC,
static llvm::Expected<
std::pair<std::unique_ptr<llvm::orc::LLJITBuilder>, uint32_t>>
outOfProcessJITBuilder(const IncrementalExecutorBuilder &IncrExecutorBuilder) {
outOfProcessJITBuilder(
const OrcIncrementalExecutorBuilder &IncrExecutorBuilder) {
std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC;
uint32_t childPid = -1;
if (!IncrExecutorBuilder.OOPExecutor.empty()) {
@@ -367,10 +405,8 @@ outOfProcessJITBuilder(const IncrementalExecutorBuilder &IncrExecutorBuilder) {
}
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) {
if (IE)
return std::move(IE);
OrcIncrementalExecutorBuilder::createExecutor(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) {
llvm::Triple TT = TI.getTriple();
if (!TT.isOSWindows() && IsOutOfProcess) {
if (!JITBuilder) {
@@ -433,7 +469,7 @@ IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC,
return std::move(Executor);
}
llvm::Error IncrementalExecutorBuilder::UpdateOrcRuntimePath(
llvm::Error OrcIncrementalExecutorBuilder::UpdateOrcRuntimePath(
const clang::driver::Compilation &C) {
if (!IsOutOfProcess)
return llvm::Error::success();
+21 -20
View File
@@ -15,6 +15,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/TargetParser/Host.h"
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/Module.h>
#include <llvm/MC/TargetRegistry.h>
@@ -62,19 +63,28 @@ bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
namespace clang {
IncrementalExecutorBuilder::~IncrementalExecutorBuilder() = default;
std::unique_ptr<IncrementalExecutorBuilder>
IncrementalExecutorBuilder::createDefault() {
return std::make_unique<WasmIncrementalExecutorBuilder>();
}
llvm::Error
WasmIncrementalExecutorBuilder::configure(IncrementalCompilerBuilder &CB,
const Options &Opts) {
if (Opts.IsOutOfProcess)
return llvm::createStringError("Out-of-process execution is not supported "
"by the WebAssembly executor");
return llvm::Error::success();
}
llvm::Expected<llvm::Triple>
WasmIncrementalExecutorBuilder::getHostJITTriple() const {
return llvm::Triple(llvm::sys::getProcessTriple());
}
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) {
if (IE)
return std::move(IE);
if (IsOutOfProcess)
return llvm::createStringError(llvm::inconvertibleErrorCode(),
"Out-of-process execution is not supported "
"by the WebAssembly executor");
WasmIncrementalExecutorBuilder::createExecutor(
llvm::orc::ThreadSafeContext &TSC, const clang::TargetInfo &TI) {
llvm::Error Err = llvm::Error::success();
std::unique_ptr<IncrementalExecutor> Executor =
std::make_unique<WasmIncrementalExecutor>(Err, LLVMArgs);
@@ -83,15 +93,6 @@ IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC,
return std::move(Executor);
}
llvm::Error IncrementalExecutorBuilder::UpdateOrcRuntimePath(
const clang::driver::Compilation &C) {
if (IsOutOfProcess)
return llvm::createStringError(llvm::inconvertibleErrorCode(),
"Out-of-process execution is not supported "
"by the WebAssembly executor");
return llvm::Error::success();
}
WasmIncrementalExecutor::WasmIncrementalExecutor(
llvm::Error &Err, std::vector<std::string> LLVMArgs)
: StoredLLVMArgs(std::move(LLVMArgs)) {
+13
View File
@@ -24,6 +24,19 @@
namespace clang {
class WasmIncrementalExecutorBuilder : public IncrementalExecutorBuilder {
public:
llvm::Error configure(IncrementalCompilerBuilder &CB,
const Options &Opts) override;
bool supportsJIT() const override { return false; }
llvm::Expected<llvm::Triple> getHostJITTriple() const override;
private:
llvm::Expected<std::unique_ptr<IncrementalExecutor>>
createExecutor(llvm::orc::ThreadSafeContext &TSC,
const clang::TargetInfo &TI) override;
};
class WasmIncrementalExecutor : public IncrementalExecutor {
public:
WasmIncrementalExecutor(llvm::Error &Err, std::vector<std::string> LLVMArgs);
+11 -2
View File
@@ -3,8 +3,6 @@ set( LLVM_LINK_COMPONENTS
Core
LineEditor
Option
OrcJIT
OrcShared
Support
TargetParser
)
@@ -15,6 +13,17 @@ add_clang_tool(clang-repl
EXPORT_SYMBOLS
)
if(EMSCRIPTEN)
# Incremental modules use dlopen and resolve symbols from the tool.
target_link_options(clang-repl PRIVATE
"-fwasm-exceptions"
"SHELL:-s MAIN_MODULE=1"
"LINKER:--export=__clang_Interpreter_SetValueWithAlloc"
"LINKER:--export=__clang_Interpreter_SetValueNoAlloc"
"LINKER:--export=_ZnwmPv26__clang_Interpreter_NewTag"
)
endif()
if(MSVC)
set_target_properties(clang-repl PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1)
+16 -24
View File
@@ -16,14 +16,12 @@
#include "clang/Config/config.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Interpreter/CodeCompletion.h"
#include "clang/Interpreter/IncrementalExecutor.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Sema.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ExecutionEngine/Orc/LLJIT.h"
#include "llvm/LineEditor/LineEditor.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
@@ -40,8 +38,6 @@
#include <string>
#include <vector>
#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h"
// Disable LSan for this test.
// FIXME: Re-enable once we can assume GCC 13.2 or higher.
// https://llvm.org/github.com/llvm/llvm-project/issues/67586.
@@ -110,7 +106,10 @@ static llvm::Error sanitizeOopArguments(const char *ArgV0) {
(OOPExecutor.getNumOccurrences() ||
OOPExecutorConnect.getNumOccurrences()))
return llvm::make_error<llvm::StringError>(
"Out-of-process execution is only supported on Unix platforms",
SystemTriple.isOSEmscripten()
? "Out-of-process execution is not supported by the WebAssembly "
"executor"
: "Out-of-process execution is only supported on Unix platforms",
llvm::inconvertibleErrorCode());
// If -slab-allocate is passed, check that we're not trying to use it in
@@ -275,18 +274,12 @@ int main(int argc, const char **argv) {
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
auto IEB = clang::IncrementalExecutorBuilder::createDefault();
if (OptHostSupportsJit) {
auto J = llvm::orc::LLJITBuilder().create();
if (J)
llvm::outs() << "true\n";
else {
llvm::consumeError(J.takeError());
llvm::outs() << "false\n";
}
llvm::outs() << (IEB->supportsJIT() ? "true\n" : "false\n");
return 0;
} else if (OptHostJitTriple) {
auto J = ExitOnErr(llvm::orc::LLJITBuilder().create());
auto T = J->getTargetTriple();
auto T = ExitOnErr(IEB->getHostJITTriple());
llvm::outs() << T.normalize() << '\n';
return 0;
}
@@ -296,21 +289,20 @@ int main(int argc, const char **argv) {
clang::IncrementalCompilerBuilder CB;
CB.SetCompilerArgs(ClangArgv);
auto IEB = std::make_unique<clang::IncrementalExecutorBuilder>();
IEB->IsOutOfProcess = !OOPExecutor.empty() || !OOPExecutorConnect.empty();
IEB->OOPExecutor = OOPExecutor;
if (!OrcRuntimePath.empty())
IEB->OrcRuntimePath = OrcRuntimePath;
else
CB.SetDriverCompilationCallback(IEB->UpdateOrcRuntimePathCB);
auto SizeOrErr = getSlabAllocSize(SlabAllocateSizeString);
if (!SizeOrErr) {
llvm::logAllUnhandledErrors(SizeOrErr.takeError(), llvm::errs(), "error: ");
return EXIT_FAILURE;
}
IEB->SlabAllocateSize = *SizeOrErr;
IEB->UseSharedMemory = UseSharedMemory;
clang::IncrementalExecutorBuilder::Options ExecutorOpts;
ExecutorOpts.IsOutOfProcess =
!OOPExecutor.empty() || !OOPExecutorConnect.empty();
ExecutorOpts.ExecutorPath = OOPExecutor;
ExecutorOpts.RuntimePath = OrcRuntimePath;
ExecutorOpts.SlabAllocateSize = *SizeOrErr;
ExecutorOpts.UseSharedMemory = UseSharedMemory;
ExitOnErr(IEB->configure(CB, ExecutorOpts));
// If both -cuda and -hip are passed, the one that appears later wins.
// FIXME: Support both CUDA and HIP setups at the same time, for example with
@@ -13,6 +13,7 @@
#include "InterpreterTestFixture.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Interpreter/OrcIncrementalExecutorBuilder.h"
#include "clang/AST/Expr.h"
#include "clang/Frontend/CompilerInstance.h"
@@ -125,7 +126,7 @@ TEST_F(InterpreterExtensionsTest, CustomCrossJIT) {
JIT = &J;
return llvm::Error::success();
});
auto IEB = std::make_unique<IncrementalExecutorBuilder>();
auto IEB = std::make_unique<OrcIncrementalExecutorBuilder>();
IEB->JITBuilder = std::move(JB);
llvm::Error ErrOut = llvm::Error::success();
CustomJBInterpreter Interp(std::move(CI), ErrOut, std::move(IEB));
@@ -181,7 +182,7 @@ TEST_F(InterpreterExtensionsTest, CustomIncrementalExecutor) {
};
// Prepare a builder that hands out our recording executor.
auto B = std::make_unique<IncrementalExecutorBuilder>();
auto B = IncrementalExecutorBuilder::createDefault();
B->IE = std::make_unique<RecordingIncrementalExecutor>();
IncrementalCompilerBuilder CB;
@@ -18,8 +18,8 @@
#include "clang/AST/Mangle.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Interpreter/IncrementalExecutor.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Interpreter/OrcIncrementalExecutorBuilder.h"
#include "clang/Interpreter/Value.h"
#include "clang/Sema/Lookup.h"
#include "clang/Sema/Sema.h"
@@ -115,7 +115,7 @@ createInterpreterWithRemoteExecution(std::shared_ptr<IOContext> io_ctx,
Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
llvm::append_range(ClangArgs, ExtraArgs);
auto Config = std::make_unique<IncrementalExecutorBuilder>();
auto Config = std::make_unique<OrcIncrementalExecutorBuilder>();
Config->IsOutOfProcess = true;
Config->OOPExecutor = getExecutorPath();