mirror of
https://github.com/coollabsio/coolify.git
synced 2026-10-28 09:34:36 +08:00
revert(ssh): send remote command batches in argv again
Sending SSH command batches on stdin changes how every remote command
reaches a server, which is too risky for a patch release. Move it to
next and restore the heredoc transport from before 1a2a5362a. The
shared test helpers stay because they work with both transports.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3c34bdd49b
commit
4edf987279
15 files changed
+46
-375
No files matched your search
@@ -113,8 +113,7 @@ class RunRemoteProcess
|
||||
$status = ProcessStatus::IN_PROGRESS;
|
||||
$timeout = config('constants.ssh.command_timeout');
|
||||
try {
|
||||
[$command, $input] = $this->getCommand();
|
||||
$process = Process::timeout($timeout)->input($input)->start($command, $this->handleOutput(...));
|
||||
$process = Process::timeout($timeout)->start($this->getCommand(), $this->handleOutput(...));
|
||||
// Start from the stored properties and save now, so later output writes only change the log and
|
||||
// never write back an old copy over a status that Coolify set meanwhile (for example a stopped import).
|
||||
$stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties ?? $this->activity->properties;
|
||||
@@ -215,19 +214,13 @@ class RunRemoteProcess
|
||||
event(new $eventClass($eventData));
|
||||
}
|
||||
|
||||
/**
|
||||
* The SSH command and its input. The command batch goes to ssh on stdin, so its secrets are
|
||||
* not in a command line on the Coolify host.
|
||||
*
|
||||
* @return array{0: string, 1: string}
|
||||
*/
|
||||
protected function getCommand(): array
|
||||
protected function getCommand(): string
|
||||
{
|
||||
$server_uuid = $this->activity->getExtraProperty('server_uuid');
|
||||
$command = RemoteProcessCommand::read($this->activity) ?? throw new \RuntimeException('The command of this task was already removed.');
|
||||
$server = Server::whereUuid($server_uuid)->firstOrFail();
|
||||
|
||||
return [SshMultiplexingHelper::generateSshBatchCommand($server), SshMultiplexingHelper::sshBatchInput($command)];
|
||||
return SshMultiplexingHelper::generateSshCommand($server, $command);
|
||||
}
|
||||
|
||||
protected function handleOutput(string $type, string $output)
|
||||
|
||||
@@ -114,7 +114,7 @@ class CheckProxy
|
||||
$results = Process::concurrently(function ($pool) use ($server, $ports, $proxyContainerName) {
|
||||
foreach ($ports as $port) {
|
||||
$commands = $this->buildPortCheckCommands($server, $port, $proxyContainerName);
|
||||
$pool->command($commands['ssh_command'])->input($commands['input'])->timeout(10);
|
||||
$pool->command($commands['ssh_command'])->timeout(10);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -152,10 +152,10 @@ class CheckProxy
|
||||
private function buildPortCheckCommands(Server $server, int $port, string $proxyContainerName): array
|
||||
{
|
||||
$portCheckScript = $this->buildPortCheckScript($port, $proxyContainerName);
|
||||
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $portCheckScript);
|
||||
|
||||
return [
|
||||
'ssh_command' => SshMultiplexingHelper::generateSshBatchCommand($server),
|
||||
'input' => SshMultiplexingHelper::sshBatchInput($portCheckScript),
|
||||
'ssh_command' => $sshCommand,
|
||||
'script' => $portCheckScript,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -253,25 +253,6 @@ class SshMultiplexingHelper
|
||||
.$delimiter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an SSH command that runs the command batch given to the process as input (see sshBatchInput()).
|
||||
* The batch is not part of any command line on the Coolify host, where `ps` would show its secrets
|
||||
* (database passwords, registry passwords, tokens), and it is not limited to 128 KiB.
|
||||
* Like the quoted heredoc of generateSshCommand(), the batch gets no local shell expansion.
|
||||
*/
|
||||
public static function generateSshBatchCommand(Server $server, bool $disableMultiplexing = false, ?int $commandTimeout = null): string
|
||||
{
|
||||
return self::generateSshStdinCommand($server, self::remoteShellCommand(), $disableMultiplexing, $commandTimeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* The process input for generateSshBatchCommand(). A heredoc ends its content with a newline, so does this.
|
||||
*/
|
||||
public static function sshBatchInput(string $command): string
|
||||
{
|
||||
return $command.PHP_EOL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an SSH command that runs one short remote command and leaves stdin free for data.
|
||||
* Large content must go through stdin: Linux limits one exec argument to 128 KiB, and
|
||||
|
||||
@@ -7,7 +7,6 @@ use App\Helpers\SshMultiplexingHelper;
|
||||
use App\Models\Server;
|
||||
use App\Services\ServerTransfer\ServerTransferClaimer;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Process\ProcessResult;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
@@ -226,7 +225,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
private function attemptConnection(): bool
|
||||
{
|
||||
try {
|
||||
$process = $this->runSsh(self::probeCommand(), self::SSH_TIMEOUT_SECONDS, disableMultiplexing: true);
|
||||
$process = Process::timeout(self::SSH_TIMEOUT_SECONDS)
|
||||
->run($this->sshCommand(self::probeCommand(), disableMultiplexing: true));
|
||||
if ($process->exitCode() !== 0) {
|
||||
$this->connectionError = 'ssh exit '.$process->exitCode().': '.Str::limit(trim($process->errorOutput()), 300);
|
||||
} else {
|
||||
@@ -308,7 +308,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
private function runDockerCommand(string $command): ?string
|
||||
{
|
||||
try {
|
||||
$process = $this->runSsh($command, self::DOCKER_TIMEOUT_SECONDS, disableMultiplexing: $this->disableMux);
|
||||
$process = Process::timeout(self::DOCKER_TIMEOUT_SECONDS)->run($this->sshCommand($command, disableMultiplexing: $this->disableMux));
|
||||
} catch (\Throwable $e) {
|
||||
Log::debug('ServerConnectionCheck: Docker command failed', [
|
||||
'server_id' => $this->server->id,
|
||||
@@ -322,15 +322,13 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
return $process->successful() ? sanitize_utf8_text(trim($process->output())) : null;
|
||||
}
|
||||
|
||||
private function runSsh(string $command, int $timeout, bool $disableMultiplexing = false): ProcessResult
|
||||
private function sshCommand(string $command, bool $disableMultiplexing = false): string
|
||||
{
|
||||
$commands = [$command];
|
||||
if ($this->server->isNonRoot()) {
|
||||
$commands = parseCommandsByLineForSudo(collect($commands), $this->server);
|
||||
}
|
||||
|
||||
return Process::timeout($timeout)
|
||||
->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
|
||||
->run(SshMultiplexingHelper::generateSshBatchCommand($this->server, $disableMultiplexing));
|
||||
return SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), $disableMultiplexing);
|
||||
}
|
||||
}
|
||||
@@ -496,11 +496,10 @@ class GetLogs extends Component
|
||||
*/
|
||||
private function executeLogCommand(Server $server, string $command, int $maxBytes): array
|
||||
{
|
||||
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $this->boundedLogCommand($command, $maxBytes));
|
||||
|
||||
// boundedLogCommand() caps the output at $maxBytes + 1, so one extra byte means truncation.
|
||||
$output = Process::timeout(config('constants.ssh.command_timeout'))
|
||||
->input(SshMultiplexingHelper::sshBatchInput($this->boundedLogCommand($command, $maxBytes)))
|
||||
->run(SshMultiplexingHelper::generateSshBatchCommand($server))
|
||||
->output();
|
||||
$output = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand)->output();
|
||||
$truncated = strlen($output) > $maxBytes;
|
||||
$logs = rtrim(removeAnsiColors($truncated ? substr($output, 0, $maxBytes) : $output), "\n");
|
||||
|
||||
|
||||
@@ -402,15 +402,12 @@ class ContainerFilesystemService
|
||||
$command = parseCommandsByLineForSudo(collect([$command]), $this->server)[0];
|
||||
}
|
||||
|
||||
$process = proc_open($this->sshCommand(), [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['file', '/dev/null', 'w']], $pipes);
|
||||
$process = proc_open($this->sshCommand($command), [1 => ['pipe', 'w'], 2 => ['file', '/dev/null', 'w']], $pipes);
|
||||
if (! is_resource($process)) {
|
||||
throw new \RuntimeException('Could not start the download.');
|
||||
}
|
||||
|
||||
try {
|
||||
// The command goes to ssh on stdin, so it is not in a command line on the Coolify host.
|
||||
fwrite($pipes[0], SshMultiplexingHelper::sshBatchInput($command));
|
||||
fclose($pipes[0]);
|
||||
while (! feof($pipes[1])) {
|
||||
$chunk = fread($pipes[1], 64 * 1024);
|
||||
if ($chunk === false || connection_aborted()) {
|
||||
@@ -453,9 +450,9 @@ class ContainerFilesystemService
|
||||
instant_remote_process([$this->buildS3UploadCommand($path, $isDir, $url, $headers)], $this->server, timeout: 3600);
|
||||
}
|
||||
|
||||
protected function sshCommand(): string
|
||||
protected function sshCommand(string $command): string
|
||||
{
|
||||
return SshMultiplexingHelper::generateSshBatchCommand($this->server);
|
||||
return SshMultiplexingHelper::generateSshCommand($this->server, $command);
|
||||
}
|
||||
|
||||
protected function escapePath(string $path, string $label): string
|
||||
|
||||
@@ -23,8 +23,7 @@ class DatabaseStartCommandExecutor
|
||||
$secrets = method_exists($database, 'resolvedSecretManagerValuesForRedaction')
|
||||
? $database->resolvedSecretManagerValuesForRedaction()
|
||||
: [];
|
||||
// The commands go to ssh on stdin, so the database passwords in them are not in a command line on the Coolify host.
|
||||
$remoteCommand = SshMultiplexingHelper::generateSshBatchCommand($server);
|
||||
$remoteCommand = SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands));
|
||||
|
||||
$activity->properties = $activity->properties->merge(['status' => ProcessStatus::IN_PROGRESS->value]);
|
||||
$activity->save();
|
||||
@@ -35,7 +34,6 @@ class DatabaseStartCommandExecutor
|
||||
try {
|
||||
$process = Process::timeout($timeout)
|
||||
->idleTimeout(3600)
|
||||
->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
|
||||
->start($remoteCommand, function (string $type, string $output) use ($activity, $secrets, &$pending): void {
|
||||
$pending[$type] = ($pending[$type] ?? '').$output;
|
||||
$end = max(strrpos($pending[$type], "\n"), strrpos($pending[$type], "\r"));
|
||||
|
||||
@@ -216,9 +216,10 @@ class DockerRegistryLogins
|
||||
/**
|
||||
* Run docker login on the server, writing to the same config file that deployments mount.
|
||||
*
|
||||
* The script goes to the server's shell on ssh stdin, and the shell builtin echo passes the secret to docker
|
||||
* on stdin, so it is not a process argument on the Coolify host or on the server. It is kept out of SSH error
|
||||
* logs and exception messages.
|
||||
* The SSH wrapper sends this script to the server's shell on stdin, and the shell builtin echo passes the
|
||||
* secret to docker on stdin, so it is not a process argument on the server. On the Coolify host it is still
|
||||
* in the command line of the local shell that runs ssh, because SshMultiplexingHelper::generateSshCommand()
|
||||
* embeds the script in a heredoc. It is kept out of SSH error logs and exception messages.
|
||||
*/
|
||||
public static function login(Server $server, string $registry, string $username, string $password): void
|
||||
{
|
||||
@@ -276,9 +277,7 @@ class DockerRegistryLogins
|
||||
{
|
||||
$commands = $server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $server) : [$command];
|
||||
try {
|
||||
$process = Process::timeout(60)
|
||||
->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
|
||||
->run(SshMultiplexingHelper::generateSshBatchCommand($server));
|
||||
$process = Process::timeout(60)->run(SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands)));
|
||||
} catch (ProcessTimedOutException|SymfonyProcessTimedOutException) {
|
||||
// The timeout message contains the full SSH command, including the registry password.
|
||||
throw new RuntimeException('The command on the server timed out after 60 seconds.', 124);
|
||||
|
||||
@@ -727,9 +727,9 @@ class SentinelTrafficClient
|
||||
$commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
|
||||
|
||||
try {
|
||||
$process = Process::timeout(self::REMOTE_TIMEOUT)
|
||||
->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
|
||||
->run(SshMultiplexingHelper::generateSshBatchCommand($this->server, commandTimeout: self::REMOTE_TIMEOUT));
|
||||
$process = Process::timeout(self::REMOTE_TIMEOUT)->run(
|
||||
SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
|
||||
);
|
||||
|
||||
if ($process->exitCode() !== 0) {
|
||||
excludeCertainErrors($process->errorOutput(), $process->exitCode());
|
||||
|
||||
@@ -159,9 +159,9 @@ trait ExecuteRemoteCommand
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the actual command with process handling. Without $input, the command goes to the remote shell
|
||||
* over SSH stdin, so its secrets are not in a command line on the Coolify host. With $input, the command
|
||||
* is the SSH argument and the content goes to it over SSH stdin.
|
||||
* Execute the actual command with process handling. With $input, the content goes to the command over
|
||||
* SSH stdin, because Linux limits one exec argument to 128 KiB and generateSshCommand() puts the whole
|
||||
* script into one argument.
|
||||
*/
|
||||
private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden = false, $skip_command_log = false, ?string $input = null)
|
||||
{
|
||||
@@ -169,12 +169,9 @@ trait ExecuteRemoteCommand
|
||||
$this->application_deployment_queue->addLogEntry('[CMD]: '.$this->redact_sensitive_info($command), hidden: true);
|
||||
}
|
||||
|
||||
if ($input === null) {
|
||||
$remote_command = SshMultiplexingHelper::generateSshBatchCommand($this->server);
|
||||
$input = SshMultiplexingHelper::sshBatchInput((string) $command);
|
||||
} else {
|
||||
$remote_command = SshMultiplexingHelper::generateSshStdinCommand($this->server, $command);
|
||||
}
|
||||
$remote_command = $input === null
|
||||
? SshMultiplexingHelper::generateSshCommand($this->server, $command)
|
||||
: SshMultiplexingHelper::generateSshStdinCommand($this->server, $command);
|
||||
$addLogEntry = function (string $type, string $output) use ($command, $hidden, $customType, $command_hidden, $skip_command_log): void {
|
||||
// Sanitize output to ensure valid UTF-8 encoding before JSON encoding
|
||||
$log_output = str(sanitize_utf8_text($output))->trim();
|
||||
|
||||
@@ -192,8 +192,8 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $
|
||||
|
||||
return SshRetryHandler::retry(
|
||||
function () use ($server, $command_string) {
|
||||
$sshCommand = SshMultiplexingHelper::generateSshBatchCommand($server);
|
||||
$process = runSshProcess($sshCommand, 30, SshMultiplexingHelper::sshBatchInput($command_string));
|
||||
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
|
||||
$process = runSshProcess($sshCommand, 30);
|
||||
|
||||
$output = trim($process->output());
|
||||
$exitCode = $process->exitCode();
|
||||
@@ -228,8 +228,8 @@ function instant_remote_process(Collection|array $command, Server $server, bool
|
||||
|
||||
return SshRetryHandler::retry(
|
||||
function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
|
||||
$sshCommand = SshMultiplexingHelper::generateSshBatchCommand($server, $disableMultiplexing, (int) $effectiveTimeout);
|
||||
$process = runSshProcess($sshCommand, (int) $effectiveTimeout, SshMultiplexingHelper::sshBatchInput($command_string));
|
||||
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
|
||||
$process = runSshProcess($sshCommand, (int) $effectiveTimeout);
|
||||
|
||||
$output = trim($process->output());
|
||||
$exitCode = $process->exitCode();
|
||||
|
||||
@@ -461,13 +461,13 @@ it('streams the remote command output to the writer in chunks', function () {
|
||||
$server = fsServer();
|
||||
$svc = new class($server, 'app-123') extends ContainerFilesystemService
|
||||
{
|
||||
public string $stdinFile;
|
||||
public ?string $remoteCommand = null;
|
||||
|
||||
protected function sshCommand(): string
|
||||
protected function sshCommand(string $command): string
|
||||
{
|
||||
$this->stdinFile = tempnam(sys_get_temp_dir(), 'fs-stream-');
|
||||
$this->remoteCommand = $command;
|
||||
|
||||
return 'cat > '.escapeshellarg($this->stdinFile).'; head -c 200000 /dev/zero';
|
||||
return 'head -c 200000 /dev/zero';
|
||||
}
|
||||
};
|
||||
$received = '';
|
||||
@@ -478,9 +478,7 @@ it('streams the remote command output to the writer in chunks', function () {
|
||||
$chunks++;
|
||||
});
|
||||
|
||||
$remoteCommand = file_get_contents($svc->stdinFile);
|
||||
unlink($svc->stdinFile);
|
||||
expect($remoteCommand)->toEndWith("docker exec 'app-123' cat -- '/app/a.bin'\n");
|
||||
expect($svc->remoteCommand)->toEndWith("docker exec 'app-123' cat -- '/app/a.bin'");
|
||||
expect(strlen($received))->toBe(200000);
|
||||
expect($chunks)->toBeGreaterThan(1);
|
||||
});
|
||||
@@ -141,7 +141,7 @@ it('redacts a database start secret that is split across output chunks', functio
|
||||
$activity = remote_process(['echo ok'], $this->server);
|
||||
// The process fake ends every output chunk with a newline, so the chunks are sent by hand.
|
||||
$invoked = Mockery::mock(InvokedProcess::class);
|
||||
Process::shouldReceive('timeout->idleTimeout->input->start')->andReturnUsing(function (string $command, Closure $output) use ($invoked) {
|
||||
Process::shouldReceive('timeout->idleTimeout->start')->andReturnUsing(function (string $command, Closure $output) use ($invoked) {
|
||||
$invoked->shouldReceive('wait')->andReturnUsing(function () use ($output) {
|
||||
foreach (['value is split-sec', "ret-value\nnext ", 'line split-secret-value'] as $chunk) {
|
||||
$output('out', $chunk);
|
||||
@@ -169,7 +169,7 @@ function runRemoteProcessWithChunks(Activity $activity, array $chunks, ?Throwabl
|
||||
// The process fake ends every output chunk with a newline, so the chunks are sent by hand.
|
||||
$invoked = Mockery::mock(InvokedProcess::class);
|
||||
$invoked->shouldReceive('id')->andReturn(123);
|
||||
Process::shouldReceive('timeout->input->start')->andReturnUsing(function (string $command, Closure $output) use ($invoked, $chunks, $failure) {
|
||||
Process::shouldReceive('timeout->start')->andReturnUsing(function (string $command, Closure $output) use ($invoked, $chunks, $failure) {
|
||||
$invoked->shouldReceive('wait')->andReturnUsing(function () use ($output, $chunks, $failure) {
|
||||
foreach ($chunks as [$type, $chunk]) {
|
||||
$output($type, $chunk);
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
<?php
|
||||
|
||||
use App\Actions\CoolifyTask\RunRemoteProcess;
|
||||
use App\Enums\ProcessStatus;
|
||||
use App\Exceptions\DeploymentException;
|
||||
use App\Models\ApplicationDeploymentQueue;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Server;
|
||||
use App\Models\User;
|
||||
use App\Services\DatabaseStartCommandExecutor;
|
||||
use App\Services\DockerRegistryLogins;
|
||||
use App\Support\RemoteProcessCommand;
|
||||
use App\Traits\ExecuteRemoteCommand;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\File;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Spatie\Activitylog\Models\Activity;
|
||||
|
||||
/**
|
||||
* Remote command batches must reach ssh on stdin. A batch inside the argv of a local process
|
||||
* (for example a heredoc in `sh -c`) shows its secrets to every user of `ps` on the Coolify host.
|
||||
*
|
||||
* These tests run the real generated command. A stub `ssh` on PATH records its argv, the argv of
|
||||
* every ancestor process up to PHP, and its stdin, then runs the remote shell command locally.
|
||||
*/
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
const SSH_STDIN_SECRET = 'rr-n5-secret-Zq81xLp';
|
||||
|
||||
beforeEach(function () {
|
||||
config()->set('constants.ssh.mux_enabled', false);
|
||||
config()->set('constants.ssh.max_retries', 3);
|
||||
config()->set('constants.ssh.retry_base_delay', 0);
|
||||
config()->set('constants.ssh.retry_max_delay', 0);
|
||||
Server::flushIdentityMap();
|
||||
|
||||
$this->stubDir = sys_get_temp_dir().'/ssh-stdin-stub-'.uniqid();
|
||||
File::ensureDirectoryExists($this->stubDir.'/bin');
|
||||
File::ensureDirectoryExists($this->stubDir.'/log');
|
||||
|
||||
$log = $this->stubDir.'/log';
|
||||
$phpPid = getmypid();
|
||||
File::put($this->stubDir.'/bin/ssh', <<<BASH
|
||||
#!/bin/bash
|
||||
id="\$(date +%s%N).\$\$"
|
||||
printf '%s\\0' "\$@" > "{$log}/argv.\$id"
|
||||
pid=\$\$
|
||||
while [ "\$pid" -gt 1 ] && [ "\$pid" != "{$phpPid}" ]; do
|
||||
tr '\\0' ' ' < /proc/\$pid/cmdline >> "{$log}/ancestors.\$id"
|
||||
echo >> "{$log}/ancestors.\$id"
|
||||
pid=\$(sed 's/.*) //' /proc/\$pid/stat | cut -d' ' -f2)
|
||||
done
|
||||
tee "{$log}/stdin.\$id" | sh -c "\${@: -1}"
|
||||
BASH);
|
||||
File::put($this->stubDir.'/bin/sudo', "#!/bin/sh\nexec \"\$@\"\n");
|
||||
chmod($this->stubDir.'/bin/ssh', 0755);
|
||||
chmod($this->stubDir.'/bin/sudo', 0755);
|
||||
File::put($this->stubDir.'/bin/docker', "#!/bin/sh\n[ \"\$3\" = login ] && { cat > \"{$log}/docker-login.stdin\"; echo 'Login Succeeded'; }\nexit 0\n");
|
||||
chmod($this->stubDir.'/bin/docker', 0755);
|
||||
|
||||
$this->originalPath = getenv('PATH');
|
||||
$path = $this->stubDir.'/bin:'.$this->originalPath;
|
||||
putenv("PATH={$path}");
|
||||
$_SERVER['PATH'] = $_ENV['PATH'] = $path;
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
putenv("PATH={$this->originalPath}");
|
||||
$_SERVER['PATH'] = $_ENV['PATH'] = $this->originalPath;
|
||||
File::deleteDirectory($this->stubDir);
|
||||
Server::flushIdentityMap();
|
||||
});
|
||||
|
||||
function sshStdinStubServer(string $user = 'root'): Server
|
||||
{
|
||||
$owner = User::factory()->create();
|
||||
$team = $owner->teams()->first();
|
||||
|
||||
Storage::fake('ssh-keys');
|
||||
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||
Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKey->private_key);
|
||||
|
||||
return Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'private_key_id' => $privateKey->id,
|
||||
'user' => $user,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{invocations: int, argv: string, stdin: string}
|
||||
*/
|
||||
function sshStdinStubRecords(string $stubDir): array
|
||||
{
|
||||
$files = fn (string $prefix) => glob("{$stubDir}/log/{$prefix}.*") ?: [];
|
||||
$read = fn (string $prefix) => collect($files($prefix))->map(fn (string $file) => (string) file_get_contents($file))->implode("\n");
|
||||
|
||||
return [
|
||||
'invocations' => count($files('argv')),
|
||||
'argv' => $read('argv')."\n".$read('ancestors'),
|
||||
'stdin' => $read('stdin'),
|
||||
];
|
||||
}
|
||||
|
||||
function sshStdinRemoteProcessActivity(Server $server, string $command): Activity
|
||||
{
|
||||
return activity()
|
||||
->withProperties([
|
||||
'server_uuid' => $server->uuid,
|
||||
...RemoteProcessCommand::properties($command),
|
||||
'type' => 'inline',
|
||||
'status' => ProcessStatus::QUEUED->value,
|
||||
'team_id' => $server->team_id,
|
||||
])
|
||||
->event('inline')
|
||||
->log('[]');
|
||||
}
|
||||
|
||||
function sshStdinDeploymentRunner(Server $server): object
|
||||
{
|
||||
$queue = ApplicationDeploymentQueue::create([
|
||||
'deployment_uuid' => 'ssh-stdin-deployment',
|
||||
'application_id' => 0,
|
||||
'server_id' => $server->id,
|
||||
'status' => 'in_progress',
|
||||
]);
|
||||
|
||||
return new class($server, $queue)
|
||||
{
|
||||
use ExecuteRemoteCommand;
|
||||
|
||||
public Collection $saved_outputs;
|
||||
|
||||
public function __construct(public Server $server, public ApplicationDeploymentQueue $application_deployment_queue)
|
||||
{
|
||||
$this->saved_outputs = collect();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
it('runs an instant remote batch with its secret only on ssh stdin', function (string $user) {
|
||||
$server = sshStdinStubServer($user);
|
||||
|
||||
$output = instant_remote_process([
|
||||
'export DB_PASSWORD='.SSH_STDIN_SECRET,
|
||||
'test -n "$DB_PASSWORD"',
|
||||
'echo "len=${#DB_PASSWORD} literal=\$HOME tick=`echo t`"',
|
||||
], $server);
|
||||
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($output)->toBe('len='.strlen(SSH_STDIN_SECRET).' literal=$HOME tick=t')
|
||||
->and($records['invocations'])->toBe(1)
|
||||
->and($records['stdin'])->toContain(SSH_STDIN_SECRET)
|
||||
->and($records['argv'])->toContain('bash -se')
|
||||
->not->toContain(SSH_STDIN_SECRET);
|
||||
if ($user !== 'root') {
|
||||
expect($records['stdin'])->toContain('sudo test -n');
|
||||
}
|
||||
})->with(['root' => 'root', 'non-root' => 'rr-deployer']);
|
||||
|
||||
it('does not retry a remote command that fails with exit code 1', function () {
|
||||
$server = sshStdinStubServer();
|
||||
|
||||
expect(fn () => instant_remote_process(['echo '.SSH_STDIN_SECRET.' > /dev/null', 'echo failed >&2', 'exit 1', 'echo never'], $server))
|
||||
->toThrow(RuntimeException::class, 'failed');
|
||||
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($records['invocations'])->toBe(1)
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET);
|
||||
});
|
||||
|
||||
it('retries the whole batch when ssh exits with the transport code 255', function () {
|
||||
$server = sshStdinStubServer();
|
||||
|
||||
expect(fn () => instant_remote_process(['echo '.SSH_STDIN_SECRET.' > /dev/null', 'echo "Connection refused" >&2', 'exit 255'], $server))
|
||||
->toThrow(RuntimeException::class);
|
||||
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($records['invocations'])->toBe(3)
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET);
|
||||
});
|
||||
|
||||
it('returns nothing for a failed batch with errors ignored', function () {
|
||||
$server = sshStdinStubServer();
|
||||
|
||||
expect(instant_remote_process(['exit 3'], $server, throwError: false))->toBeNull();
|
||||
});
|
||||
|
||||
it('runs batches larger than the 128 KiB argument limit and returns large output', function () {
|
||||
$server = sshStdinStubServer();
|
||||
$payload = str_repeat('a', 200 * 1024);
|
||||
|
||||
$output = instant_remote_process([
|
||||
"PAYLOAD='{$payload}'",
|
||||
'echo "${#PAYLOAD}"',
|
||||
'head -c 3000000 /dev/zero | tr "\0" "b"',
|
||||
], $server);
|
||||
|
||||
expect(strlen($output))->toBe(strlen((string) (200 * 1024)) + 1 + 3000000)
|
||||
->and($output)->toStartWith((200 * 1024)."\nbbb");
|
||||
});
|
||||
|
||||
it('streams remote process output into the activity log with the secret only on stdin', function (string $user) {
|
||||
$server = sshStdinStubServer($user);
|
||||
$activity = sshStdinRemoteProcessActivity($server, implode("\n", [
|
||||
'export TOKEN='.SSH_STDIN_SECRET,
|
||||
'echo first',
|
||||
'sleep 0.5',
|
||||
'echo second',
|
||||
]));
|
||||
|
||||
$result = (new RunRemoteProcess($activity))();
|
||||
|
||||
$entries = collect(json_decode($activity->fresh()->description, true));
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($result->exitCode())->toBe(0)
|
||||
->and($entries->pluck('output')->map(fn ($output) => trim($output))->all())->toBe(['first', 'second'])
|
||||
->and($activity->fresh()->getExtraProperty('status'))->toBe(ProcessStatus::FINISHED->value)
|
||||
->and($records['stdin'])->toContain(SSH_STDIN_SECRET)
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET);
|
||||
})->with(['root' => 'root', 'non-root' => 'rr-deployer']);
|
||||
|
||||
it('keeps the exit code of a failed remote process and honours ignore_errors', function () {
|
||||
$server = sshStdinStubServer();
|
||||
|
||||
$failing = sshStdinRemoteProcessActivity($server, "echo broken >&2\nexit 7");
|
||||
expect(fn () => (new RunRemoteProcess($failing))())->toThrow(RuntimeException::class, 'broken');
|
||||
expect($failing->fresh()->getExtraProperty('exitCode'))->toBe(7)
|
||||
->and($failing->fresh()->getExtraProperty('status'))->toBe(ProcessStatus::ERROR->value);
|
||||
|
||||
$ignored = sshStdinRemoteProcessActivity($server, 'exit 7');
|
||||
expect((new RunRemoteProcess($ignored, ignore_errors: true))()->exitCode())->toBe(7);
|
||||
});
|
||||
|
||||
it('runs deployment commands with the secret only on stdin and streams their output', function (string $user) {
|
||||
$server = sshStdinStubServer($user);
|
||||
$runner = sshStdinDeploymentRunner($server);
|
||||
|
||||
$runner->execute_remote_command(
|
||||
['command' => 'test -n '.SSH_STDIN_SECRET.' && echo deploy-42', 'hidden' => true],
|
||||
['command' => 'cat', 'input' => "from-input\n"],
|
||||
['command' => 'false', 'ignore_errors' => true],
|
||||
);
|
||||
|
||||
$logs = collect(json_decode($runner->application_deployment_queue->fresh()->logs, true))->pluck('output');
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($logs->all())->toContain('deploy-42', 'from-input')
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET)
|
||||
->and($records['stdin'])->toContain(SSH_STDIN_SECRET);
|
||||
})->with(['root' => 'root', 'non-root' => 'rr-deployer']);
|
||||
|
||||
it('fails a deployment command with its remote exit code', function () {
|
||||
$server = sshStdinStubServer();
|
||||
$runner = sshStdinDeploymentRunner($server);
|
||||
|
||||
expect(fn () => $runner->execute_remote_command(['command' => 'echo nope >&2; exit 4']))
|
||||
->toThrow(DeploymentException::class, 'exit code 4');
|
||||
expect(sshStdinStubRecords($this->stubDir)['invocations'])->toBe(1);
|
||||
});
|
||||
|
||||
it('starts a database with its password only on ssh stdin', function (string $user) {
|
||||
$server = sshStdinStubServer($user);
|
||||
$database = new class extends Model {};
|
||||
$database->setRelation('destination', (object) ['server' => $server]);
|
||||
$activity = activity()->withProperties(['status' => ProcessStatus::QUEUED->value])->event('inline')->log('[]');
|
||||
|
||||
app(DatabaseStartCommandExecutor::class)->execute([
|
||||
'export POSTGRES_PASSWORD='.SSH_STDIN_SECRET,
|
||||
'echo "started ${#POSTGRES_PASSWORD}"',
|
||||
], $database, $activity);
|
||||
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect($activity->fresh()->getExtraProperty('status'))->toBe(ProcessStatus::FINISHED->value)
|
||||
->and(collect(json_decode($activity->fresh()->description, true))->pluck('output')->implode(''))->toContain('started '.strlen(SSH_STDIN_SECRET))
|
||||
->and($records['stdin'])->toContain(SSH_STDIN_SECRET)
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET);
|
||||
})->with(['root' => 'root', 'non-root' => 'rr-deployer']);
|
||||
|
||||
it('logs in to a registry with the password only on ssh stdin', function (string $user) {
|
||||
$server = sshStdinStubServer($user);
|
||||
|
||||
DockerRegistryLogins::login($server, 'registry.example.com', 'deployer', SSH_STDIN_SECRET);
|
||||
|
||||
$records = sshStdinStubRecords($this->stubDir);
|
||||
expect(file_get_contents($this->stubDir.'/log/docker-login.stdin'))->toBe(SSH_STDIN_SECRET)
|
||||
->and($records['argv'])->not->toContain(SSH_STDIN_SECRET)
|
||||
->not->toContain(base64_encode(SSH_STDIN_SECRET));
|
||||
})->with(['root' => 'root', 'non-root' => 'rr-deployer']);
|
||||
@@ -71,8 +71,9 @@ function runSentSentinelScriptWithFakes(string $bin): array
|
||||
{
|
||||
$script = null;
|
||||
Process::assertRan(function ($process) use (&$script) {
|
||||
// The script goes to ssh on stdin.
|
||||
$script = rtrim((string) $process->input, "\n");
|
||||
// generateSshCommand sends the script as the body of a heredoc: first line opens it, last line closes it.
|
||||
$lines = explode("\n", $process->command);
|
||||
$script = implode("\n", array_slice($lines, 1, -1));
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user