mirror of
https://github.com/coollabsio/coolify.git
synced 2026-10-28 17:44:36 +08:00
Andras Bacsai
599dc1ca67
The SSH retry matched connection words in any error output. When `mc alias set` in a backup upload could not reach S3, its "connection refused" output caused a retry of the full batch, and the second `docker run --name backup-of-<uuid>` failed with a name conflict that hid the real error. - Retry only SSH transport failures: exit code 255, or errors without an exit code. - Keep the remote exit code on deployment command failures. - Remove a leftover backup helper container in the same upload batch. Closes #12059
141 lines
4.9 KiB
PHP
141 lines
4.9 KiB
PHP
<?php
|
|
|
|
use App\Helpers\SshMultiplexingHelper;
|
|
use App\Models\PrivateKey;
|
|
use App\Models\Server;
|
|
use App\Models\User;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Support\Facades\Process;
|
|
use Illuminate\Support\Facades\Storage;
|
|
|
|
/**
|
|
* Tests that per-call SSH command timeouts (e.g. a scheduled backup's configured
|
|
* timeout) reach the shell-level `timeout N ssh` wrapper instead of being capped
|
|
* by the global constants.ssh.command_timeout default.
|
|
*
|
|
* @see https://github.com/coollabsio/coolify DatabaseBackupJob/VolumeBackupJob pass
|
|
* a per-backup timeout to instant_remote_process()
|
|
*/
|
|
uses(RefreshDatabase::class);
|
|
|
|
function makeTimeoutTestServer(): Server
|
|
{
|
|
$user = User::factory()->create();
|
|
$team = $user->teams()->first();
|
|
|
|
$privateKeyContent = '-----BEGIN OPENSSH PRIVATE KEY-----
|
|
'.
|
|
'b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
|
'.
|
|
'QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk
|
|
'.
|
|
'hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA
|
|
'.
|
|
'AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
|
|
'.
|
|
'uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
|
|
'.
|
|
'-----END OPENSSH PRIVATE KEY-----';
|
|
|
|
$privateKey = PrivateKey::create([
|
|
'name' => 'timeout-test-key-'.uniqid(),
|
|
'private_key' => $privateKeyContent,
|
|
'team_id' => $team->id,
|
|
]);
|
|
|
|
Storage::fake('ssh-keys');
|
|
Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);
|
|
|
|
$server = Server::factory()->create([
|
|
'team_id' => $team->id,
|
|
'private_key_id' => $privateKey->id,
|
|
]);
|
|
|
|
Storage::disk('ssh-keys')->put("ssh_key@{$server->privateKey->uuid}", $server->privateKey->private_key);
|
|
|
|
return $server;
|
|
}
|
|
|
|
it('wraps ssh commands with an explicitly passed command timeout', function () {
|
|
config(['constants.ssh.mux_enabled' => false]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
$command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok', commandTimeout: 7200);
|
|
|
|
expect($command)->toStartWith('timeout 7200 ssh ');
|
|
});
|
|
|
|
it('wraps ssh commands with the configured default timeout when none is passed', function () {
|
|
config([
|
|
'constants.ssh.mux_enabled' => false,
|
|
'constants.ssh.command_timeout' => 1234,
|
|
]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
$command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok');
|
|
|
|
expect($command)->toStartWith('timeout 1234 ssh ');
|
|
});
|
|
|
|
it('forwards the per-call timeout of instant_remote_process to the ssh timeout wrapper', function () {
|
|
config(['constants.ssh.mux_enabled' => false]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
Process::fake();
|
|
|
|
instant_remote_process(['echo ok'], $server, timeout: 7200, disableMultiplexing: true);
|
|
|
|
Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 7200 ssh '));
|
|
});
|
|
|
|
it('uses the configured default timeout in instant_remote_process when no timeout is passed', function () {
|
|
config([
|
|
'constants.ssh.mux_enabled' => false,
|
|
'constants.ssh.command_timeout' => 1234,
|
|
]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
Process::fake();
|
|
|
|
instant_remote_process(['echo ok'], $server, disableMultiplexing: true);
|
|
|
|
Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 1234 ssh '));
|
|
});
|
|
|
|
it('does not replay a command batch when a remote command fails with connection text in its output', function () {
|
|
config([
|
|
'constants.ssh.mux_enabled' => false,
|
|
'constants.ssh.max_retries' => 3,
|
|
'constants.ssh.retry_base_delay' => 0,
|
|
]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
Process::fake([
|
|
'*' => Process::result(errorOutput: 'mc: <ERROR> Unable to initialize new alias. dial tcp 10.0.0.5:9000: connect: connection refused', exitCode: 1),
|
|
]);
|
|
|
|
expect(fn () => instant_remote_process(['docker run -d --name backup-of-x minio/mc', 'docker exec backup-of-x mc alias set temporary http://s3 k s'], $server, disableMultiplexing: true))
|
|
->toThrow(RuntimeException::class, 'connection refused');
|
|
|
|
Process::assertRanTimes(fn ($process) => str_contains($process->command, 'ssh '), 1);
|
|
});
|
|
|
|
it('retries a command batch when the ssh connection itself fails', function () {
|
|
config([
|
|
'constants.ssh.mux_enabled' => false,
|
|
'constants.ssh.max_retries' => 3,
|
|
'constants.ssh.retry_base_delay' => 0,
|
|
]);
|
|
$server = makeTimeoutTestServer();
|
|
|
|
Process::fake([
|
|
'*' => Process::sequence()
|
|
->push(Process::result(errorOutput: 'kex_exchange_identification: read: Connection reset by peer', exitCode: 255))
|
|
->push(Process::result(output: 'ok')),
|
|
]);
|
|
|
|
expect(instant_remote_process(['echo ok'], $server, disableMultiplexing: true))->toBe('ok');
|
|
|
|
Process::assertRanTimes(fn ($process) => str_contains($process->command, 'ssh '), 2);
|
|
});
|