fix(security): secure remote command secrets and clone volume data

.ai/lessons.md

@@ -21,6 +21,7 @@
 - Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
 - Browser `click()`/`assertSee()` wait forever when nothing matches, so a bad selector hangs the run instead of failing it. `click('Text')` can match a hidden element first (use `button:visible:has-text("Text")`); `:text-is()` on an element with child elements and `assertSee()` text with double quotes never match. Prefer plain CSS selectors. `click()` also hung on a button far down the long `/analytics` page; click it with `$page->script("document.querySelector('...').click()")` there.
 - A new worktree has no `vendor/` or `public/build/`. Run `composer install --ignore-platform-req=ext-zip` on the host and copy `public/build` from the main checkout before page-rendering tests; otherwise they fail with a Vite manifest error.
+- A running dev instance mounts the checkout, and its scheduled `CheckForUpdatesJob` writes the CDN data to `base_path('versions.json')`. Check `git diff versions.json` before you commit a version change.
 - Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only.
 - The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:<version>-dind` container); newer clients can reject data from older daemons.
 - Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree.
@@ -41,6 +42,8 @@
 - Horizon workers in a dev instance keep the code they loaded. Run `./scripts/dev exec php artisan horizon:terminate` after a code change and before a live test of queued jobs.
 - To prove that a server side effect is blocked, check the server itself after the request. A model hook or job can be one of several write paths (for example `ServerStorageSaveJob` and `ServerFilesFromServerJob` both write file storages).
 
+- When you add a gate that lets a user through only after a step (2FA, password reset, onboarding), test it with users who have no known password (OAuth, magic-link invites) and together with the other redirecting gates. Two gates that each allow only their own page make a redirect loop, and a step that needs password confirmation blocks users who cannot confirm one.
+
 ## Preserve product scope
 - Do not replace required SPA navigation with a full-page redirect to hide a lifecycle or ordering defect.
 - Do not add billing restrictions, live reconciliation, or fallback behavior unless the request includes them.

app/Actions/CoolifyTask/RunRemoteProcess.php

@@ -113,7 +113,8 @@ class RunRemoteProcess
         $status = ProcessStatus::IN_PROGRESS;
         $timeout = config('constants.ssh.command_timeout');
         try {
-            $process = Process::timeout($timeout)->start($this->getCommand(), $this->handleOutput(...));
+            [$command, $input] = $this->getCommand();
+            $process = Process::timeout($timeout)->input($input)->start($command, $this->handleOutput(...));
             // Start from the stored properties and save now, so later output writes only change the log and
             // never write back an old copy over a status that Coolify set meanwhile (for example a stopped import).
             $stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties ?? $this->activity->properties;
@@ -214,13 +215,19 @@ class RunRemoteProcess
         event(new $eventClass($eventData));
     }
 
-    protected function getCommand(): string
+    /**
+     * The SSH command and its input. The command batch goes to ssh on stdin, so its secrets are
+     * not in a command line on the Coolify host.
+     *
+     * @return array{0: string, 1: string}
+     */
+    protected function getCommand(): array
     {
         $server_uuid = $this->activity->getExtraProperty('server_uuid');
         $command = RemoteProcessCommand::read($this->activity) ?? throw new \RuntimeException('The command of this task was already removed.');
         $server = Server::whereUuid($server_uuid)->firstOrFail();
 
-        return SshMultiplexingHelper::generateSshCommand($server, $command);
+        return [SshMultiplexingHelper::generateSshBatchCommand($server), SshMultiplexingHelper::sshBatchInput($command)];
     }
 
     protected function handleOutput(string $type, string $output)

app/Actions/Database/StartDatabaseImport.php

@@ -2,6 +2,8 @@
 
 namespace App\Actions\Database;
 
+use App\Helpers\SshMultiplexingHelper;
+use App\Helpers\SshRetryHandler;
 use App\Models\S3Storage;
 use App\Models\Server;
 use App\Models\ServiceDatabase;
@@ -157,10 +159,9 @@ class StartDatabaseImport
                 throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.');
             }
             $helper = "s3-restore-{$operation}";
-            $this->startS3HelperWithEnv($storage, $server, $helper, $network);
+            $this->startS3Helper($storage, $server, $helper, $network);
             $sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}");
             $commandList = [
-                'docker exec '.escapeshellarg($helper).' sh -c '.escapeshellarg('mc alias set s3temp "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"'),
                 'docker exec '.escapeshellarg($helper).' mc cp '.$sourceArg.' /tmp/restore',
                 ...$this->streamFromHelper($helper, $container, $containerPath),
                 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
@@ -289,7 +290,12 @@ class StartDatabaseImport
         ];
     }
 
-    private function startS3HelperWithEnv(S3Storage $storage, Server $server, string $helper, string $network): void
+    /**
+     * Starts the S3 restore helper and sets its `s3temp` mc alias. The S3 keys go to mc through the
+     * stdin of a separate SSH process, so they are not in the arguments of any process on Coolify or
+     * on the server. The helper is removed when either step fails.
+     */
+    private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network): void
     {
         $image = escapeshellarg(coolifyHelperImage().':'.getHelperVersion());
 
@@ -298,16 +304,33 @@ class StartDatabaseImport
                 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
                 'docker run -d --network '.escapeshellarg($network)
                     .' --name '.escapeshellarg($helper)
-                    .' -e S3_ENDPOINT='.escapeshellarg((string) $storage->endpoint)
-                    .' -e S3_ACCESS_KEY='.escapeshellarg((string) $storage->key)
-                    .' -e S3_SECRET_KEY='.escapeshellarg((string) $storage->secret)
                     .' '.$storage->minioClientRegionOption()
                     .' '.$image.' sleep 86400',
             ], $server);
+            $this->setMinioClientAlias($storage, $server, $helper);
         } catch (Throwable) {
             instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false);
 
             throw new DatabaseImportException('Unable to start the S3 restore helper.');
         }
     }
+
+    private function setMinioClientAlias(S3Storage $storage, Server $server, string $helper): void
+    {
+        $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($helper)
+            .' mc alias set s3temp '.escapeshellarg((string) $storage->endpoint).' >/dev/null';
+        $timeout = (int) config('constants.ssh.command_timeout');
+
+        SshRetryHandler::retry(
+            function () use ($server, $remoteCommand, $storage, $timeout) {
+                $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
+                $process = runSshProcess($sshCommand, $timeout, $storage->key."\n".$storage->secret."\n");
+
+                if ($process->exitCode() !== 0) {
+                    excludeCertainErrors($process->errorOutput(), $process->exitCode());
+                }
+            },
+            ['server' => $server->ip, 'function' => 'setMinioClientAlias'],
+        );
+    }
 }

app/Actions/Proxy/CheckProxy.php

@@ -114,7 +114,7 @@ class CheckProxy
             $results = Process::concurrently(function ($pool) use ($server, $ports, $proxyContainerName) {
                 foreach ($ports as $port) {
                     $commands = $this->buildPortCheckCommands($server, $port, $proxyContainerName);
-                    $pool->command($commands['ssh_command'])->timeout(10);
+                    $pool->command($commands['ssh_command'])->input($commands['input'])->timeout(10);
                 }
             });
 
@@ -152,10 +152,10 @@ class CheckProxy
     private function buildPortCheckCommands(Server $server, int $port, string $proxyContainerName): array
     {
         $portCheckScript = $this->buildPortCheckScript($port, $proxyContainerName);
-        $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $portCheckScript);
 
         return [
-            'ssh_command' => $sshCommand,
+            'ssh_command' => SshMultiplexingHelper::generateSshBatchCommand($server),
+            'input' => SshMultiplexingHelper::sshBatchInput($portCheckScript),
             'script' => $portCheckScript,
         ];
     }

app/Actions/Server/CleanupDocker.php

@@ -43,6 +43,11 @@ class CleanupDocker implements ShouldBeUnique
      */
     public const STOP_CLEANUP_COOLDOWN = 3600;
 
+    /**
+     * Cleanup log command of the entry that records a skipped build cache prune.
+     */
+    public const BUILD_CACHE_KEPT_LOG_COMMAND = 'docker builder prune, docker buildx prune';
+
     public int $jobTries = 1;
 
     /**
@@ -87,8 +92,9 @@ class CleanupDocker implements ShouldBeUnique
 
     /**
      * The deployment that is queued or running on the server (as target or build server), if any.
-     * The cleanup prunes the build caches, which fails a build that uses them. A deployment that
-     * wrote no log for longer than the deployment timeout is stuck and does not count.
+     * Pruning the build caches fails a build that uses them, so the cleanup keeps them while such a
+     * deployment exists. A deployment that wrote no log for longer than the deployment timeout is
+     * stuck and does not count.
      */
     public static function activeDeployment(Server $server): ?ApplicationDeploymentQueue
     {
@@ -99,10 +105,17 @@ class CleanupDocker implements ShouldBeUnique
             ->first(['id', 'deployment_uuid']);
     }
 
+    /**
+     * The reason the build cache was kept, read from a cleanup log, if it was kept.
+     */
+    public static function buildCacheKeptReason(array $cleanupLog): ?string
+    {
+        return collect($cleanupLog)->firstWhere('command', self::BUILD_CACHE_KEPT_LOG_COMMAND)['output'] ?? null;
+    }
+
     /**
      * Cleanup after a resource stop: skipped when a cleanup ran recently, and the queued run
-     * only cleans when the disk usage is at or above the server cleanup threshold and no
-     * deployment is active on the server.
+     * only cleans when the disk usage is at or above the server cleanup threshold.
      */
     public static function dispatchAfterStop(Server $server): void
     {
@@ -137,14 +150,15 @@ class CleanupDocker implements ShouldBeUnique
         ];
     }
 
-    public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false, bool $onlyAboveThreshold = false)
+    /**
+     * @param  bool  $pruneBuildCacheDuringDeployment  Also prune the build caches while a deployment is active
+     *                                                 (a manual cleanup). Otherwise they stay until the next run.
+     */
+    public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false, bool $onlyAboveThreshold = false, bool $pruneBuildCacheDuringDeployment = false)
     {
         $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE;
 
         if ($onlyAboveThreshold) {
-            if (self::activeDeployment($server)) {
-                return [];
-            }
             $diskUsage = $server->getDiskUsage();
             if (is_numeric($diskUsage) && (int) $diskUsage < $server->settings->docker_cleanup_threshold) {
                 return [];
@@ -177,12 +191,22 @@ class CleanupDocker implements ShouldBeUnique
         $commands = [
             $this->buildContainerPruneCommand(),
             $imagePruneCmd,
-            'docker builder prune -af',
-            // The prune fails on a stopped builder, so start it first.
-            railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, 'docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; docker buildx prune --builder coolify-railpack -af'),
-            $this->buildCoolifyImageCleanupCommand($helperImageVersion),
         ];
 
+        $activeDeployment = $pruneBuildCacheDuringDeployment ? null : self::activeDeployment($server);
+        if (! $activeDeployment) {
+            $commands[] = 'docker builder prune -af';
+            // The prune fails on a stopped builder, so start it first.
+            $commands[] = railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, 'docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; docker buildx prune --builder coolify-railpack -af');
+        } else {
+            $cleanupLog[] = [
+                'command' => self::BUILD_CACHE_KEPT_LOG_COMMAND,
+                'output' => 'Skipped: the build cache was not pruned because deployment '.$activeDeployment->deployment_uuid.' is queued or running on this server.',
+            ];
+        }
+
+        $commands[] = $this->buildCoolifyImageCleanupCommand($helperImageVersion);
+
         if ($deleteUnusedVolumes) {
             $commands[] = 'docker volume prune -af';
         }

app/Actions/Shared/CloneVolumeDataWithSourceStopped.php

@@ -0,0 +1,333 @@
+<?php
+
+namespace App\Actions\Shared;
+
+use App\Actions\Application\StopApplication;
+use App\Actions\Database\StartDatabase;
+use App\Actions\Database\StopDatabase;
+use App\Actions\Service\StartService;
+use App\Actions\Service\StopService;
+use App\Enums\ApplicationDeploymentStatus;
+use App\Jobs\VolumeCloneJob;
+use App\Models\Application;
+use App\Models\Service;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\StandaloneSqlite;
+use App\Notifications\Internal\GeneralNotification;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use Lorisleiva\Actions\Concerns\AsAction;
+use RuntimeException;
+use Throwable;
+
+/**
+ * Copies the volume data of a clone in one queued chain. A running source is stopped before the
+ * copy and started again afterwards, also when a step fails. A stopped source stays stopped.
+ * When a step fails, the team gets a notification.
+ */
+class CloneVolumeDataWithSourceStopped
+{
+    use AsAction;
+
+    /** Exception code of a failed stop step: no volume data was copied. */
+    private const STOP_FAILED = 1001;
+
+    /** Exception code of a failed stop of an application that took down some, not all, of its containers. */
+    private const APPLICATION_PARTLY_STOPPED = 1002;
+
+    /**
+     * @param  array<int, VolumeCloneJob>  $volumeJobs
+     */
+    public function handle(
+        Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|StandaloneSqlite $source,
+        array $volumeJobs,
+    ): void {
+        if ($volumeJobs === []) {
+            return;
+        }
+
+        $wasRunning = filled($source->status) && ! $source->isExited();
+
+        // One closure per line: a serialized closure is found by its line, so two closures on one line run the same code.
+        $chain = $wasRunning ? [
+            static fn () => self::stopSource($source),
+            ...$volumeJobs,
+            static fn () => self::startSource($source),
+        ] : $volumeJobs;
+
+        // The jobs load the new volume rows, so they must not run before the transaction of the caller commits.
+        DB::afterCommit(fn () => Bus::chain($chain)
+            ->onQueue('high')
+            ->catch(static fn (Throwable $e) => self::recoverSource($source, $wasRunning, $e))
+            ->dispatch());
+    }
+
+    /**
+     * The policy ability that allows a user to stop and start the source.
+     */
+    public static function sourceAbility(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+    ): string {
+        return match (true) {
+            $source instanceof Application => 'deploy',
+            $source instanceof Service => 'stop',
+            default => 'manage',
+        };
+    }
+
+    /**
+     * Stops the source and fails the chain, so that no copy runs, when the stop reports an error
+     * or a container of the source still runs. The stop actions return their errors instead of throwing.
+     */
+    public static function stopSource(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+    ): void {
+        // A partly stopped application is redeployed, so remember which of its containers ran.
+        $runningBefore = $source instanceof Application ? self::runningContainersOrNull($source) : null;
+        $result = match (true) {
+            $source instanceof Application => StopApplication::run($source, false, false),
+            $source instanceof Service => StopService::run($source, dockerCleanup: false),
+            default => StopDatabase::run($source, dockerCleanup: false),
+        };
+        if (filled($result) && $result !== 'Database stopped successfully') {
+            throw new RuntimeException($result, self::failedStopCode($source, $runningBefore));
+        }
+
+        try {
+            $runningContainers = self::runningContainers($source);
+        } catch (Throwable $e) {
+            throw new RuntimeException('Could not check that the containers stopped: '.$e->getMessage(), self::STOP_FAILED, $e);
+        }
+        if ($runningContainers !== []) {
+            throw new RuntimeException('These containers still run: '.implode(', ', $runningContainers), self::failedStopCode($source, $runningBefore, $runningContainers));
+        }
+    }
+
+    /**
+     * An application whose stop took down some of its containers, while others still run, is
+     * partly stopped. When the containers before the stop are unknown, it may be partly stopped.
+     *
+     * @param  list<string>|null  $runningBefore
+     * @param  list<string>|null  $runningAfter
+     */
+    private static function failedStopCode(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+        ?array $runningBefore,
+        ?array $runningAfter = null,
+    ): int {
+        if (! $source instanceof Application) {
+            return self::STOP_FAILED;
+        }
+        $runningAfter ??= self::runningContainersOrNull($source);
+        if (blank($runningAfter)) {
+            return self::STOP_FAILED;
+        }
+
+        return $runningBefore === null || array_diff($runningBefore, $runningAfter) !== []
+            ? self::APPLICATION_PARTLY_STOPPED
+            : self::STOP_FAILED;
+    }
+
+    /**
+     * @return list<string>|null
+     */
+    private static function runningContainersOrNull(Application $application): ?array
+    {
+        try {
+            return self::runningContainers($application);
+        } catch (Throwable) {
+            return null;
+        }
+    }
+
+    /**
+     * The names of the running containers of the source on its main server. A swarm stack is
+     * removed asynchronously, so it is not checked.
+     *
+     * @return list<string>
+     */
+    private static function runningContainers(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+    ): array {
+        $server = $source->destination->server;
+        if ($server->isSwarm()) {
+            return [];
+        }
+        [$type, $filters] = match (true) {
+            $source instanceof Application => ['application', ['label=coolify.pullRequestId=0']],
+            $source instanceof Service => ['service', []],
+            default => ['database', []],
+        };
+        $output = instant_remote_process(dockerPsByOwnerCommands($type, $source->uuid, $filters, '{{.Names}}', all: false), $server);
+
+        return collect(explode("\n", (string) $output))->map(fn (string $name) => trim($name))->filter()->unique()->values()->all();
+    }
+
+    public static function startSource(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+    ): void {
+        match (true) {
+            $source instanceof Application => queue_application_deployment(
+                application: $source,
+                deployment_uuid: new_public_id(),
+                server: $source->destination->server,
+                destination: $source->destination,
+                no_questions_asked: true,
+            ),
+            $source instanceof Service => StartService::run($source),
+            default => StartDatabase::run($source),
+        };
+    }
+
+    public static function recoverSource(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+        bool $wasRunning,
+        Throwable $e,
+    ): void {
+        if (in_array($e->getCode(), [self::STOP_FAILED, self::APPLICATION_PARTLY_STOPPED], true)) {
+            self::recoverFromFailedStop($source, $e);
+
+            return;
+        }
+
+        $message = "Copying the volume data of '{$source->name}' to its clone failed. The clone may have no or partial data.";
+
+        if ($wasRunning) {
+            try {
+                self::startSource($source);
+                $message .= " Coolify started '{$source->name}' again.";
+            } catch (Throwable $startError) {
+                Log::error("Failed to start {$source->name} again after a failed volume clone: ".$startError->getMessage());
+                $message .= " Coolify could not start '{$source->name}' again. Start it manually.";
+            }
+        }
+
+        $source->team()?->notify(new GeneralNotification(
+            $message.' Error: '.remove_iip($e->getMessage()),
+            success: false,
+        ));
+    }
+
+    /**
+     * No data was copied. A source that the stop took down fully (or whose state is unknown) is
+     * started again. When containers of a service still run, only its stopped containers are
+     * started again; an application or a database that still runs is left alone.
+     */
+    private static function recoverFromFailedStop(
+        Application|Service|StandaloneClickhouse|StandaloneDragonfly|StandaloneKeydb|StandaloneMariadb|StandaloneMongodb|StandaloneMysql|StandalonePostgresql|StandaloneRedis|StandaloneSqlite $source,
+        Throwable $e,
+    ): void {
+        $message = "Coolify did not copy the volume data of '{$source->name}' to its clone, because stopping '{$source->name}' failed. The clone has no copied data.";
+
+        try {
+            $stillRunning = self::runningContainers($source) !== [];
+        } catch (Throwable) {
+            $stillRunning = false;
+        }
+
+        if ($stillRunning && $source instanceof Service) {
+            try {
+                self::startStoppedServiceContainers($source);
+                $message .= " Some containers of '{$source->name}' were still running, so Coolify started its stopped containers again.";
+            } catch (Throwable $startError) {
+                Log::error("Failed to start the stopped containers of {$source->name} after a failed stop for a volume clone: ".$startError->getMessage());
+                $message .= " Some containers of '{$source->name}' are still running. Coolify could not start its stopped containers again. Start '{$source->name}' manually.";
+            }
+        } elseif ($stillRunning && $source instanceof Application && $e->getCode() === self::APPLICATION_PARTLY_STOPPED) {
+            $message .= ' '.self::redeployPartlyStoppedApplication($source);
+        } elseif ($stillRunning) {
+            $message .= " '{$source->name}' is still running.";
+        } else {
+            try {
+                self::startSource($source);
+                $message .= " Coolify started '{$source->name}' again.";
+            } catch (Throwable $startError) {
+                Log::error("Failed to start {$source->name} again after a failed stop for a volume clone: ".$startError->getMessage());
+                $message .= " Coolify could not start '{$source->name}' again. Start it manually.";
+            }
+        }
+
+        $source->team()?->notify(new GeneralNotification(
+            $message.' Error: '.remove_iip($e->getMessage()),
+            success: false,
+        ));
+    }
+
+    /**
+     * Queues a normal deployment (a rolling update when the application supports it) of the commit
+     * of the last successful deployment, so the running containers are replaced only by the same
+     * version. Without such a commit, the configured commit is deployed.
+     *
+     * @return string The sentence for the notification.
+     */
+    private static function redeployPartlyStoppedApplication(Application $application): string
+    {
+        $commit = $application->deployment_queue()
+            ->where('status', ApplicationDeploymentStatus::FINISHED->value)
+            ->where('pull_request_id', 0)
+            ->whereNotNull('commit')
+            ->where('commit', '!=', 'HEAD')
+            ->orderByDesc('updated_at')
+            ->orderByDesc('id')
+            ->value('commit');
+
+        try {
+            $result = queue_application_deployment(
+                application: $application,
+                deployment_uuid: new_public_id(),
+                commit: $commit,
+                no_questions_asked: true,
+            );
+            if (($result['status'] ?? null) !== 'queued') {
+                throw new RuntimeException($result['message'] ?? 'The deployment was not queued.');
+            }
+        } catch (Throwable $deployError) {
+            Log::error("Failed to queue a redeploy of {$application->name} after a failed stop for a volume clone: ".$deployError->getMessage());
+
+            return "Some containers of '{$application->name}' stopped and others are still running. Coolify could not queue a redeploy of '{$application->name}' to bring back its stopped containers. Redeploy it manually.";
+        }
+
+        $sentence = "Some containers of '{$application->name}' stopped and others were still running, so Coolify started a redeploy of '{$application->name}' to bring back its stopped containers";
+
+        return $commit === null
+            ? $sentence.'. Coolify could not find the commit that ran before, so the redeploy uses the configured commit or image tag.'
+            : $sentence." with the commit that ran before ({$commit}).";
+    }
+
+    /**
+     * Creates and starts the containers that the stop removed, with the compose file of the last
+     * start. Running containers are not recreated, so `up --force-recreate` (StartService) is not
+     * used. `docker compose start` is not enough: StopService also removes the stopped containers.
+     */
+    private static function startStoppedServiceContainers(Service $service): void
+    {
+        $server = $service->destination->server;
+        $workdir = $service->workdir();
+        $compose = serverDockerConfigPrefix($server)."docker compose --project-directory {$workdir} -f {$workdir}/docker-compose.yml --project-name {$service->uuid}";
+
+        if (! data_get($service, 'connect_to_docker_network')) {
+            instant_remote_process(["{$compose} up -d --no-recreate < /dev/null"], $server);
+
+            return;
+        }
+
+        // Like StartService: connect new containers to the predefined network before they start.
+        $commands = ["{$compose} up -d --no-recreate --no-start < /dev/null"];
+        $composeServices = data_get(parseDockerComposeYaml(data_get($service, 'docker_compose', '')), 'services', []);
+        foreach ($composeServices as $serviceName => $serviceConfig) {
+            if ($command = StartService::predefinedNetworkConnectCommand($service, $serviceName, $serviceConfig)) {
+                $commands[] = $command;
+            }
+        }
+        $commands[] = "{$compose} start < /dev/null";
+        instant_remote_process($commands, $server);
+    }
+}

app/Helpers/SshMultiplexingHelper.php

@@ -253,6 +253,25 @@ class SshMultiplexingHelper
             .$delimiter;
     }
 
+    /**
+     * Build an SSH command that runs the command batch given to the process as input (see sshBatchInput()).
+     * The batch is not part of any command line on the Coolify host, where `ps` would show its secrets
+     * (database passwords, registry passwords, tokens), and it is not limited to 128 KiB.
+     * Like the quoted heredoc of generateSshCommand(), the batch gets no local shell expansion.
+     */
+    public static function generateSshBatchCommand(Server $server, bool $disableMultiplexing = false, ?int $commandTimeout = null): string
+    {
+        return self::generateSshStdinCommand($server, self::remoteShellCommand(), $disableMultiplexing, $commandTimeout);
+    }
+
+    /**
+     * The process input for generateSshBatchCommand(). A heredoc ends its content with a newline, so does this.
+     */
+    public static function sshBatchInput(string $command): string
+    {
+        return $command.PHP_EOL;
+    }
+
     /**
      * Build an SSH command that runs one short remote command and leaves stdin free for data.
      * Large content must go through stdin: Linux limits one exec argument to 128 KiB, and

app/Http/Controllers/Api/ApplicationsController.php

@@ -4070,7 +4070,9 @@ class ApplicationsController extends Controller
             ], 404);
         }
 
-        $env->value = $request->value;
+        if ($request->has('value')) {
+            $env->value = $request->value;
+        }
         $env->fill($request->only(['is_literal', 'is_multiline', 'is_shown_once', 'is_runtime', 'is_buildtime', 'comment']));
         $env->save();
 
@@ -4226,7 +4228,9 @@ class ApplicationsController extends Controller
             if ($is_preview) {
                 $env = $application->environment_variables_preview()->where('key', $key)->first();
                 if ($env) {
-                    $env->value = $item->get('value');
+                    if ($item->has('value')) {
+                        $env->value = $item->get('value');
+                    }
                     $env->fill($item->only(['is_literal', 'is_multiline', 'is_shown_once', 'is_runtime', 'is_buildtime', 'comment'])->all());
                     $env->save();
                 } else {
@@ -4247,7 +4251,9 @@ class ApplicationsController extends Controller
             } else {
                 $env = $application->environment_variables()->where('key', $key)->first();
                 if ($env) {
-                    $env->value = $item->get('value');
+                    if ($item->has('value')) {
+                        $env->value = $item->get('value');
+                    }
                     $env->fill($item->only(['is_literal', 'is_multiline', 'is_shown_once', 'is_runtime', 'is_buildtime', 'comment'])->all());
                     $env->save();
                 } else {
@@ -5177,7 +5183,7 @@ class ApplicationsController extends Controller
 
     #[OA\Patch(
         summary: 'Update Storage',
-        description: 'Update a persistent storage or file storage by application UUID.',
+        description: 'Update a persistent storage or file storage by application UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.',
         path: '/applications/{uuid}/storages',
         operationId: 'update-storage-by-application-uuid',
         security: [

app/Http/Controllers/Api/DatabasesController.php

@@ -7,6 +7,7 @@ use App\Actions\Database\StartDatabase;
 use App\Actions\Database\StartDatabaseProxy;
 use App\Actions\Database\StopDatabase;
 use App\Actions\Database\StopDatabaseProxy;
+use App\Actions\Shared\CloneVolumeDataWithSourceStopped;
 use App\Enums\NewDatabaseTypes;
 use App\Http\Controllers\Controller;
 use App\Jobs\DatabaseBackupJob;
@@ -36,7 +37,6 @@ use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
 use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\DB;
 use OpenApi\Attributes as OA;
 
@@ -4023,7 +4023,9 @@ class DatabasesController extends Controller
             return response()->json(['message' => 'Environment variable not found.'], 404);
         }
 
-        $env->value = $request->value;
+        if ($request->has('value')) {
+            $env->value = $request->value;
+        }
         if ($request->has('is_literal')) {
             $env->is_literal = $request->is_literal;
         }
@@ -4759,7 +4761,7 @@ class DatabasesController extends Controller
 
     #[OA\Patch(
         summary: 'Update Storage',
-        description: 'Update a persistent storage or file storage by database UUID.',
+        description: 'Update a persistent storage or file storage by database UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.',
         path: '/databases/{uuid}/storages',
         operationId: 'update-storage-by-database-uuid',
         security: [
@@ -5352,33 +5354,11 @@ class DatabasesController extends Controller
             }
         }
 
-        // Stop once, clone all volumes, then start once — avoids per-volume stop/start races.
-        if ($pendingVolumeClones !== [] && $sourceServer && $targetServer) {
-            try {
-                $chain = [
-                    function () use ($database) {
-                        StopDatabase::run($database);
-                    },
-                ];
-
-                foreach ($pendingVolumeClones as $clone) {
-                    $chain[] = new VolumeCloneJob(
-                        $clone['source'],
-                        $clone['target'],
-                        $sourceServer,
-                        $targetServer,
-                        $clone['model'],
-                    );
-                }
-
-                $chain[] = function () use ($database) {
-                    StartDatabase::run($database);
-                };
-
-                Bus::chain($chain)->onQueue('high')->dispatch();
-            } catch (\Exception $e) {
-                \Log::error('Failed to queue database volume clone for '.$database->uuid.': '.$e->getMessage());
-            }
+        if ($sourceServer && $targetServer) {
+            CloneVolumeDataWithSourceStopped::run($database, array_map(
+                fn (array $clone) => new VolumeCloneJob($clone['source'], $clone['target'], $sourceServer, $targetServer, $clone['model']),
+                $pendingVolumeClones,
+            ));
         }
 
         foreach ($database->fileStorages()->get() as $storage) {

app/Http/Controllers/Api/ServersController.php

@@ -776,8 +776,8 @@ class ServersController extends Controller
             'connection_timeout' => 'integer|min:1|max:300',
             'is_terminal_enabled' => 'boolean|nullable',
             'wildcard_domain' => 'nullable|url',
-            'server_timezone' => ['string', function (string $attribute, mixed $value, \Closure $fail): void {
-                if (! validate_timezone($value)) {
+            'server_timezone' => ['bail', 'filled', 'string', function (string $attribute, mixed $value, \Closure $fail): void {
+                if (! is_string($value) || ! validate_timezone($value)) {
                     $fail('The server_timezone must be a valid timezone.');
                 }
             }],

app/Http/Controllers/Api/ServicesController.php

@@ -1573,7 +1573,9 @@ class ServicesController extends Controller
             return response()->json(['message' => 'Environment variable not found.'], 404);
         }
 
-        $env->value = $request->value;
+        if ($request->has('value')) {
+            $env->value = $request->value;
+        }
         if ($request->has('is_literal')) {
             $env->is_literal = $request->is_literal;
         }
@@ -2760,7 +2762,7 @@ class ServicesController extends Controller
 
     #[OA\Patch(
         summary: 'Update Storage',
-        description: 'Update a persistent storage or file storage by service UUID.',
+        description: 'Update a persistent storage or file storage by service UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.',
         path: '/services/{uuid}/storages',
         operationId: 'update-storage-by-service-uuid',
         security: [

app/Http/Middleware/RequireTeamTwoFactor.php

@@ -17,7 +17,9 @@ class RequireTeamTwoFactor
     public function handle(Request $request, Closure $next): Response
     {
         $user = $request->user();
-        if (! $user || filled($user->two_factor_confirmed_at)) {
+        // Users who must set a password first (magic link invitees) stay on the password page
+        // (CheckForcePasswordReset); they need a known password to confirm the 2FA setup.
+        if (! $user || filled($user->two_factor_confirmed_at) || $user->force_password_reset) {
             return $next($request);
         }
 

app/Jobs/DatabaseBackupJob.php

@@ -3,6 +3,8 @@
 namespace App\Jobs;
 
 use App\Events\BackupCreated;
+use App\Helpers\SshMultiplexingHelper;
+use App\Helpers\SshRetryHandler;
 use App\Models\S3Storage;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledDatabaseBackupExecution;
@@ -921,8 +923,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
     protected function upload_to_s3(S3Storage $s3): void
     {
         try {
-            $key = $s3->key;
-            $secret = $s3->secret;
             $bucket = $s3->bucket;
             $endpoint = $s3->endpoint;
             $s3->testConnection(shouldSave: true);
@@ -940,10 +940,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $mount = escapeshellarg($this->backupMountSource().':'.$this->backup_location.':ro');
             $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm {$s3->minioClientRegionOption()} -v {$mount} {$fullImageName}";
 
-            // Escape S3 credentials to prevent command injection
-            $escapedEndpoint = escapeshellarg($endpoint);
-            $escapedKey = escapeshellarg($key);
-            $escapedSecret = escapeshellarg($secret);
             $escapedBackupLocation = escapeshellarg($this->backup_location);
             $escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
             $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $s3->trustedInternalHosts()))
@@ -951,9 +947,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                 ->implode(' ');
             $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
 
-            $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set{$resolveOptions} temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
-            $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}";
             instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true);
+            $this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions);
+            instant_remote_process(
+                ["docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}"],
+                $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true,
+            );
 
         } catch (Throwable $e) {
             $this->add_to_error_output($e->getMessage());
@@ -964,6 +963,32 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
         }
     }
 
+    /**
+     * Sets the `temporary` mc alias in a helper container. The S3 keys go to mc through the stdin of the
+     * SSH process, so they are not in the arguments of any process on Coolify or on the server.
+     */
+    private function setMinioClientAlias(Server $server, string $container, S3Storage $s3, string $resolveOptions): void
+    {
+        $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container)
+            .' mc alias set'.$resolveOptions.' temporary '.escapeshellarg($s3->endpoint).' >/dev/null';
+        $timeout = (int) config('constants.ssh.command_timeout');
+
+        SshRetryHandler::retry(
+            function () use ($server, $remoteCommand, $s3, $timeout) {
+                $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
+                $process = runSshProcess($sshCommand, $timeout, $s3->key."\n".$s3->secret."\n");
+
+                if ($process->exitCode() !== 0) {
+                    excludeCertainErrors($process->errorOutput(), $process->exitCode());
+                }
+            },
+            [
+                'server' => $server->ip,
+                'function' => 'setMinioClientAlias',
+            ],
+        );
+    }
+
     private function getFullImageName(): string
     {
         $helperImage = coolifyHelperImage();

app/Jobs/DockerCleanupJob.php

@@ -110,28 +110,19 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
                 return;
             }
 
-            $activeDeployment = $this->manualCleanup ? null : CleanupDocker::activeDeployment($this->server);
-            if ($activeDeployment) {
-                $this->execution_log->update([
-                    'status' => 'success',
-                    'message' => 'Docker cleanup skipped: deployment '.$activeDeployment->deployment_uuid.' is queued or running on this server. The next scheduled cleanup runs as usual.',
-                ]);
-                event(new DockerCleanupDone($this->execution_log));
-
-                return;
-            }
-
             $this->usageBefore = $this->server->getDiskUsage();
 
             if ($this->manualCleanup || $this->server->settings->force_docker_cleanup) {
                 $cleanup_log = CleanupDocker::run(
                     server: $this->server,
                     deleteUnusedVolumes: $this->deleteUnusedVolumes,
-                    deleteUnusedNetworks: $this->deleteUnusedNetworks
+                    deleteUnusedNetworks: $this->deleteUnusedNetworks,
+                    pruneBuildCacheDuringDeployment: $this->manualCleanup
                 );
                 $usageAfter = $this->server->getDiskUsage();
                 $message = ($this->manualCleanup ? 'Manual' : 'Forced').' Docker cleanup job executed successfully. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.';
 
+                $message = $this->withBuildCacheKeptReason($message, $cleanup_log);
                 $this->execution_log->update([
                     'status' => 'success',
                     'message' => $message,
@@ -148,10 +139,12 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
                 $cleanup_log = CleanupDocker::run(
                     server: $this->server,
                     deleteUnusedVolumes: $this->deleteUnusedVolumes,
-                    deleteUnusedNetworks: $this->deleteUnusedNetworks
+                    deleteUnusedNetworks: $this->deleteUnusedNetworks,
+                    pruneBuildCacheDuringDeployment: $this->manualCleanup
                 );
                 $message = 'Docker cleanup job executed successfully, but no disk usage could be determined.';
 
+                $message = $this->withBuildCacheKeptReason($message, $cleanup_log);
                 $this->execution_log->update([
                     'status' => 'success',
                     'message' => $message,
@@ -168,7 +161,8 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
                 $cleanup_log = CleanupDocker::run(
                     server: $this->server,
                     deleteUnusedVolumes: $this->deleteUnusedVolumes,
-                    deleteUnusedNetworks: $this->deleteUnusedNetworks
+                    deleteUnusedNetworks: $this->deleteUnusedNetworks,
+                    pruneBuildCacheDuringDeployment: $this->manualCleanup
                 );
                 $usageAfter = $this->server->getDiskUsage();
                 $diskSaved = $this->usageBefore - $usageAfter;
@@ -179,6 +173,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
                     $message = 'Docker cleanup job executed successfully, but no disk space was saved. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.';
                 }
 
+                $message = $this->withBuildCacheKeptReason($message, $cleanup_log);
                 $this->execution_log->update([
                     'status' => 'success',
                     'message' => $message,
@@ -227,6 +222,13 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
         }
     }
 
+    private function withBuildCacheKeptReason(string $message, array $cleanupLog): string
+    {
+        $reason = CleanupDocker::buildCacheKeptReason($cleanupLog);
+
+        return $reason ? $message.' '.$reason : $message;
+    }
+
     /**
      * failed() runs on a fresh job copy built from the queue payload (for example after a worker timeout),
      * so the execution this run created is kept under the queue job uuid instead of on the job.

app/Jobs/ServerConnectionCheckJob.php

@@ -7,6 +7,7 @@ use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
 use App\Services\ServerTransfer\ServerTransferClaimer;
 use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Process\ProcessResult;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -225,8 +226,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
     private function attemptConnection(): bool
     {
         try {
-            $process = Process::timeout(self::SSH_TIMEOUT_SECONDS)
-                ->run($this->sshCommand(self::probeCommand(), disableMultiplexing: true));
+            $process = $this->runSsh(self::probeCommand(), self::SSH_TIMEOUT_SECONDS, disableMultiplexing: true);
             if ($process->exitCode() !== 0) {
                 $this->connectionError = 'ssh exit '.$process->exitCode().': '.Str::limit(trim($process->errorOutput()), 300);
             } else {
@@ -308,7 +308,7 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
     private function runDockerCommand(string $command): ?string
     {
         try {
-            $process = Process::timeout(self::DOCKER_TIMEOUT_SECONDS)->run($this->sshCommand($command, disableMultiplexing: $this->disableMux));
+            $process = $this->runSsh($command, self::DOCKER_TIMEOUT_SECONDS, disableMultiplexing: $this->disableMux);
         } catch (\Throwable $e) {
             Log::debug('ServerConnectionCheck: Docker command failed', [
                 'server_id' => $this->server->id,
@@ -322,13 +322,15 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
         return $process->successful() ? sanitize_utf8_text(trim($process->output())) : null;
     }
 
-    private function sshCommand(string $command, bool $disableMultiplexing = false): string
+    private function runSsh(string $command, int $timeout, bool $disableMultiplexing = false): ProcessResult
     {
         $commands = [$command];
         if ($this->server->isNonRoot()) {
             $commands = parseCommandsByLineForSudo(collect($commands), $this->server);
         }
 
-        return SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), $disableMultiplexing);
+        return Process::timeout($timeout)
+            ->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
+            ->run(SshMultiplexingHelper::generateSshBatchCommand($this->server, $disableMultiplexing));
     }
 }

app/Jobs/VolumeBackupJob.php

@@ -3,6 +3,8 @@
 namespace App\Jobs;
 
 use App\Events\BackupCreated;
+use App\Helpers\SshMultiplexingHelper;
+use App\Helpers\SshRetryHandler;
 use App\Models\LocalPersistentVolume;
 use App\Models\S3Storage;
 use App\Models\ScheduledVolumeBackup;
@@ -102,7 +104,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
             $archiveScript = self::archiveScript($compressionCpuPercentage);
             if ($streamToS3) {
                 $this->execution->update(['local_storage_deleted' => true]);
-                $archiveCommand = $this->streamToS3Command($destinations->first(), $archiveScript, $backupLocation, $source, $containerName, $image);
+                $archiveCommand = $this->streamToS3Command($destinations->first(), $archiveScript, $backupLocation, $containerName);
                 $this->execution->update(['s3_cleanup_pending' => true]);
             } else {
                 $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
@@ -124,11 +126,20 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                 }
             }
 
-            $archiveOutput = instant_remote_process(array_filter([
-                $verifySourceCommand,
-                $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory),
-                $archiveCommand,
-            ]), $server, timeout: $this->timeout, disableMultiplexing: true);
+            try {
+                if ($streamToS3) {
+                    $this->startStreamHelper($destinations->first(), $verifySourceCommand, $source, $containerName, $image, $server);
+                }
+                $archiveOutput = instant_remote_process(array_filter([
+                    $verifySourceCommand,
+                    $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory),
+                    $archiveCommand,
+                ]), $server, timeout: $this->timeout, disableMultiplexing: true);
+            } finally {
+                if ($streamToS3) {
+                    instant_remote_process(['docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true'], $server, throwError: false, disableMultiplexing: true);
+                }
+            }
             $this->execution->update([
                 'stop_container_ids' => null,
                 'stop_recovery_pending' => false,
@@ -429,18 +440,16 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
         $s3->testConnection(shouldSave: true);
         $containerName = 'volume-upload-'.$this->execution->uuid;
         $image = coolifyHelperImage().':'.getHelperVersion();
-        $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))
-            ->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
-            ->implode(' ');
-        $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
+        $resolveOptions = $this->minioClientResolveOptions($s3);
 
         try {
             instant_remote_process([
                 'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true',
                 'docker run -d --name '.escapeshellarg($containerName).' --rm '.$s3->minioClientRegionOption().' -v '
                     .escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image),
-                'docker exec '.escapeshellarg($containerName).' mc alias set'.$resolveOptions.' temporary '
-                    .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret),
+            ], $server, timeout: $this->timeout, disableMultiplexing: true);
+            $this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions);
+            instant_remote_process([
                 'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' '
                     .escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'),
             ], $server, timeout: $this->timeout, disableMultiplexing: true);
@@ -454,32 +463,75 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
         }
     }
 
-    private function streamToS3Command(S3Storage $s3, string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string
+    /**
+     * Starts the helper container that streams the volume to S3 and sets its mc alias. The archive then
+     * runs in it with `docker exec`, so the S3 keys are never part of a command line.
+     */
+    private function startStreamHelper(S3Storage $s3, string $verifySourceCommand, string $source, string $containerName, string $image, Server $server): void
+    {
+        instant_remote_process([
+            $verifySourceCommand,
+            'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true',
+            'docker run -d --rm --name '.escapeshellarg($containerName)
+                .' '.$s3->minioClientRegionOption()
+                .' -v '.escapeshellarg($source.':/volume:ro')
+                .' '.escapeshellarg($image).' >/dev/null',
+        ], $server, timeout: $this->timeout, disableMultiplexing: true);
+        $this->setMinioClientAlias($server, $containerName, $s3, $this->minioClientResolveOptions($s3));
+    }
+
+    /**
+     * Sets the `temporary` mc alias in a helper container. The S3 keys go to mc through the stdin of the
+     * SSH process, so they are not in the arguments of any process on Coolify or on the server.
+     */
+    private function setMinioClientAlias(Server $server, string $container, S3Storage $s3, string $resolveOptions): void
+    {
+        $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container)
+            .' mc alias set'.$resolveOptions.' temporary '.escapeshellarg($s3->endpoint).' >/dev/null';
+        $timeout = (int) config('constants.ssh.command_timeout');
+
+        SshRetryHandler::retry(
+            function () use ($server, $remoteCommand, $s3, $timeout) {
+                $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
+                $process = runSshProcess($sshCommand, $timeout, $s3->key."\n".$s3->secret."\n");
+
+                if ($process->exitCode() !== 0) {
+                    excludeCertainErrors($process->errorOutput(), $process->exitCode());
+                }
+            },
+            [
+                'server' => $server->ip,
+                'function' => 'setMinioClientAlias',
+            ],
+        );
+    }
+
+    private function minioClientResolveOptions(S3Storage $s3): string
     {
-        $s3->testConnection(shouldSave: true);
         $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))
             ->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
             ->implode(' ');
-        $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
+
+        return $resolveOptions === '' ? '' : ' '.$resolveOptions;
+    }
+
+    private function streamToS3Command(S3Storage $s3, string $archiveScript, string $backupLocation, string $containerName): string
+    {
+        $s3->testConnection(shouldSave: true);
+        $resolveOptions = $this->minioClientResolveOptions($s3);
         $destination = 'temporary/'.$s3->bucket.$backupLocation;
         // `mc pipe --quiet` panics in recent mc releases, so keep its progress bar out of the error output instead.
         // Each progress frame starts with "\r", so printing the last field per line keeps only the final frame and real errors.
         $mcPipeLog = '/tmp/mc-pipe.log';
         // Object Lock buckets need Content-MD5 on each upload. `mc cp` adds it when the bucket lock is enabled, `mc pipe` only with `--checksum=MD5`.
-        $streamScript = 'set -o pipefail; mc alias set'.$resolveOptions.' temporary '
-            .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret)
-            .' >/dev/null && lock=$(mc retention info --default --json'.$resolveOptions.' '.escapeshellarg('temporary/'.$s3->bucket).' 2>/dev/null || true)'
+        $streamScript = 'set -o pipefail; lock=$(mc retention info --default --json'.$resolveOptions.' '.escapeshellarg('temporary/'.$s3->bucket).' 2>/dev/null || true)'
             .' && case "$lock" in *\'"enabled":"Enabled"\'*) checksum=--checksum=MD5 ;; *) checksum= ;; esac'
             .' && ('.$archiveScript.' | mc pipe'.$resolveOptions.' $checksum '.escapeshellarg($destination).' >/dev/null 2>'.$mcPipeLog
             .' || { status=$?; awk -F '.escapeshellarg('\r').' '.escapeshellarg('{ print $NF }').' '.$mcPipeLog.' >&2; exit $status; })'
             .' && mc stat --json'.$resolveOptions.' '.escapeshellarg($destination)
             .' | sed -n '.escapeshellarg('s/.*"size":\([0-9][0-9]*\).*/\1/p');
 
-        return 'docker run --rm --name '.escapeshellarg($containerName)
-            .' '.$s3->minioClientRegionOption()
-            .' -v '.escapeshellarg($source.':/volume:ro')
-            .' '.escapeshellarg($image)
-            .' sh -c '.escapeshellarg($streamScript);
+        return 'docker exec '.escapeshellarg($containerName).' sh -c '.escapeshellarg($streamScript);
     }
 
     private function logCompressorInDevelopment(string $image, Server $server, int $compressionCpuPercentage): void

app/Jobs/VolumeCloneJob.php

@@ -30,6 +30,22 @@ class VolumeCloneJob implements ShouldBeEncrypted, ShouldQueue
         $this->onQueue('high');
     }
 
+    /**
+     * Explicit tags, so Horizon does not compare the models of the job. In a chain, the job is restored
+     * with two copies of one server for a clone on the same server, and comparing them fails on PHP 8.4
+     * ("Nesting level too deep"), which stops the chain before the copy.
+     *
+     * @return array<int, string>
+     */
+    public function tags(): array
+    {
+        return array_values(array_unique([
+            Server::class.':'.$this->sourceServer->id,
+            ...($this->targetServer ? [Server::class.':'.$this->targetServer->id] : []),
+            LocalPersistentVolume::class.':'.$this->persistentVolume->id,
+        ]));
+    }
+
     public function handle(): void
     {
         try {

app/Livewire/Fleet/Metrics.php

@@ -12,6 +12,7 @@ use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Concurrency;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\RateLimiter;
 use Livewire\Attributes\Lazy;
 use Livewire\Attributes\Url;
 use Livewire\Component;
@@ -194,16 +195,45 @@ class Metrics extends Component
     public function setRange(string $range): void
     {
         $this->range = in_array($range, ['24h', '7d', '30d'], true) ? $range : '24h';
+        // Every refresh fetches the full range from each server over SSH, so a new range starts paused.
+        $this->live = false;
         $this->loadData();
     }
 
     public function updatedLive(): void
     {
         if ($this->live) {
-            $this->loadData();
+            $this->pollData();
         }
     }
 
+    /**
+     * The poll target. The server decides the refresh rate, so a client cannot fetch faster
+     * than once a minute or while live refresh is off.
+     */
+    public function pollData(): void
+    {
+        $seconds = $this->pollSeconds();
+        if ($seconds === null) {
+            return;
+        }
+
+        // One second of slack, so a poll that fires slightly early is not dropped.
+        if (RateLimiter::attempt('metrics-poll:'.$this->getId(), 1, fn () => true, $seconds - 1) === false) {
+            return;
+        }
+
+        $this->loadData();
+    }
+
+    /**
+     * Seconds between live refreshes, or null when live refresh is off.
+     */
+    public function pollSeconds(): ?int
+    {
+        return $this->live ? 60 : null;
+    }
+
     /**
      * @return Collection<int, Server>
      */

app/Livewire/Project/CloneMe.php

@@ -2,8 +2,7 @@
 
 namespace App\Livewire\Project;
 
-use App\Actions\Database\StartDatabase;
-use App\Actions\Database\StopDatabase;
+use App\Actions\Shared\CloneVolumeDataWithSourceStopped;
 use App\Jobs\VolumeCloneJob;
 use App\Models\Environment;
 use App\Models\Project;
@@ -103,6 +102,11 @@ class CloneMe extends Component
             if (! $this->environment || (int) $this->environment->project_id !== (int) $this->project->id) {
                 throw new \Exception('Environment not found.');
             }
+            if ($this->cloneVolumeData) {
+                foreach ([...$this->environment->applications, ...$this->environment->databases(), ...$this->environment->services] as $source) {
+                    $this->authorize(CloneVolumeDataWithSourceStopped::sourceAbility($source), $source);
+                }
+            }
             $selectedDestination = $this->findProjectTeamDestination($this->selectedDestination);
             if (! $selectedDestination) {
                 throw new \Exception('Destination not found.');
@@ -192,6 +196,7 @@ class CloneMe extends Component
             }
 
             $newDatabase->persistentStorages()->delete();
+            $pendingVolumeClones = [];
             $persistentVolumes = $database->persistentStorages()->get();
             foreach ($persistentVolumes as $volume) {
                 $originalName = $volume->name;
@@ -235,22 +240,18 @@ class CloneMe extends Component
                 $newPersistentVolume->save();
 
                 if ($this->cloneVolumeData) {
-                    try {
-                        StopDatabase::dispatch($database);
-                        $sourceVolume = $volume->name;
-                        $targetVolume = $newPersistentVolume->name;
-                        $sourceServer = $database->destination->server;
-                        $targetServer = $newDatabase->destination->server;
-
-                        VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume);
-
-                        StartDatabase::dispatch($database);
-                    } catch (\Exception $e) {
-                        \Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage());
-                    }
+                    $pendingVolumeClones[] = new VolumeCloneJob(
+                        $volume->name,
+                        $newPersistentVolume->name,
+                        $database->destination->server,
+                        $newDatabase->destination->server,
+                        $newPersistentVolume,
+                    );
                 }
             }
 
+            CloneVolumeDataWithSourceStopped::run($database, $pendingVolumeClones);
+
             $fileStorages = $database->fileStorages()->get();
             foreach ($fileStorages as $storage) {
                 $newStorage = $storage->replicate([

app/Livewire/Project/Shared/GetLogs.php

@@ -496,10 +496,11 @@ class GetLogs extends Component
      */
     private function executeLogCommand(Server $server, string $command, int $maxBytes): array
     {
-        $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $this->boundedLogCommand($command, $maxBytes));
-
         // boundedLogCommand() caps the output at $maxBytes + 1, so one extra byte means truncation.
-        $output = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand)->output();
+        $output = Process::timeout(config('constants.ssh.command_timeout'))
+            ->input(SshMultiplexingHelper::sshBatchInput($this->boundedLogCommand($command, $maxBytes)))
+            ->run(SshMultiplexingHelper::generateSshBatchCommand($server))
+            ->output();
         $truncated = strlen($output) > $maxBytes;
         $logs = rtrim(removeAnsiColors($truncated ? substr($output, 0, $maxBytes) : $output), "\n");
 

app/Livewire/Project/Shared/Metrics.php

@@ -2,6 +2,7 @@
 
 namespace App\Livewire\Project\Shared;
 
+use Illuminate\Support\Facades\RateLimiter;
 use Livewire\Component;
 
 class Metrics extends Component
@@ -40,18 +41,51 @@ class Metrics extends Component
         $this->loadData();
     }
 
+    /**
+     * Ranges up to one hour refresh live by default. Longer ranges load once, because each
+     * refresh fetches the full range from Sentinel over SSH; the user can still turn live on.
+     */
+    public function updatedInterval(): void
+    {
+        $this->live = $this->interval <= 60;
+    }
+
     public function updatedLive(): void
     {
         if ($this->live) {
-            $this->loadData();
+            $this->pollData();
         }
     }
 
     /**
-     * Short ranges refresh every 5 seconds; longer ranges change slowly, so they refresh every minute.
+     * The poll target. The server decides the refresh rate, so a client cannot fetch faster
+     * than the range allows or while live refresh is off.
      */
-    public function pollSeconds(): int
+    public function pollData(): void
     {
+        $seconds = $this->pollSeconds();
+        if ($seconds === null) {
+            return;
+        }
+
+        // One second of slack, so a poll that fires slightly early is not dropped.
+        if (RateLimiter::attempt('metrics-poll:'.$this->getId(), 1, fn () => true, $seconds - 1) === false) {
+            return;
+        }
+
+        $this->loadData();
+    }
+
+    /**
+     * Seconds between live refreshes, or null when live refresh is off. Short ranges refresh
+     * every 5 seconds; longer ranges change slowly, so they refresh every minute.
+     */
+    public function pollSeconds(): ?int
+    {
+        if (! $this->live) {
+            return null;
+        }
+
         return $this->interval <= 60 ? 5 : 60;
     }
 

app/Livewire/Project/Shared/ResourceOperations.php

@@ -2,8 +2,7 @@
 
 namespace App\Livewire\Project\Shared;
 
-use App\Actions\Database\StartDatabase;
-use App\Actions\Database\StopDatabase;
+use App\Actions\Shared\CloneVolumeDataWithSourceStopped;
 use App\Actions\Shared\MigrateResourceToDestination;
 use App\Jobs\VolumeCloneJob;
 use App\Models\Application;
@@ -72,6 +71,9 @@ class ResourceOperations extends Component
     {
         try {
             $this->authorize('update', $this->resource);
+            if ($this->cloneVolumeData) {
+                $this->authorize(CloneVolumeDataWithSourceStopped::sourceAbility($this->resource), $this->resource);
+            }
 
             $new_environment = $this->resource->environment;
             if ($environment_id !== null) {
@@ -145,6 +147,7 @@ class ResourceOperations extends Component
                 }
 
                 $new_resource->persistentStorages()->delete();
+                $pendingVolumeClones = [];
                 $persistentVolumes = $this->resource->persistentStorages()->get();
                 foreach ($persistentVolumes as $volume) {
                     $originalName = $volume->name;
@@ -188,22 +191,18 @@ class ResourceOperations extends Component
                     $newPersistentVolume->save();
 
                     if ($this->cloneVolumeData) {
-                        try {
-                            StopDatabase::dispatch($this->resource);
-                            $sourceVolume = $volume->name;
-                            $targetVolume = $newPersistentVolume->name;
-                            $sourceServer = $this->resource->destination->server;
-                            $targetServer = $new_resource->destination->server;
-
-                            VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume);
-
-                            StartDatabase::dispatch($this->resource);
-                        } catch (\Exception $e) {
-                            \Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage());
-                        }
+                        $pendingVolumeClones[] = new VolumeCloneJob(
+                            $volume->name,
+                            $newPersistentVolume->name,
+                            $this->resource->destination->server,
+                            $new_resource->destination->server,
+                            $newPersistentVolume,
+                        );
                     }
                 }
 
+                CloneVolumeDataWithSourceStopped::run($this->resource, $pendingVolumeClones);
+
                 $fileStorages = $this->resource->fileStorages()->get();
                 foreach ($fileStorages as $storage) {
                     $newStorage = $storage->replicate([

app/Livewire/Server/Charts.php

@@ -5,6 +5,7 @@ namespace App\Livewire\Server;
 use App\Actions\Server\StartSentinel;
 use App\Models\Server;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Illuminate\Support\Facades\RateLimiter;
 use Livewire\Attributes\Validate;
 use Livewire\Component;
 
@@ -150,18 +151,51 @@ class Charts extends Component
         $this->loadData();
     }
 
+    /**
+     * Ranges up to one hour refresh live by default. Longer ranges load once, because each
+     * refresh fetches the full range from Sentinel over SSH; the user can still turn live on.
+     */
+    public function updatedInterval(): void
+    {
+        $this->live = $this->interval <= 60;
+    }
+
     public function updatedLive(): void
     {
         if ($this->live) {
-            $this->loadData();
+            $this->pollData();
         }
     }
 
     /**
-     * Short ranges refresh every 5 seconds; longer ranges change slowly, so they refresh every minute.
+     * The poll target. The server decides the refresh rate, so a client cannot fetch faster
+     * than the range allows or while live refresh is off.
      */
-    public function pollSeconds(): int
+    public function pollData(): void
     {
+        $seconds = $this->pollSeconds();
+        if ($seconds === null) {
+            return;
+        }
+
+        // One second of slack, so a poll that fires slightly early is not dropped.
+        if (RateLimiter::attempt('metrics-poll:'.$this->getId(), 1, fn () => true, $seconds - 1) === false) {
+            return;
+        }
+
+        $this->loadData();
+    }
+
+    /**
+     * Seconds between live refreshes, or null when live refresh is off. Short ranges refresh
+     * every 5 seconds; longer ranges change slowly, so they refresh every minute.
+     */
+    public function pollSeconds(): ?int
+    {
+        if (! $this->live) {
+            return null;
+        }
+
         return $this->interval <= 60 ? 5 : 60;
     }
 }

app/Livewire/TwoFactorRequired.php

@@ -17,6 +17,7 @@ class TwoFactorRequired extends Component
     {
         return view('livewire.two-factor-required', [
             'teamName' => data_get(session('currentTeam'), 'name'),
+            'teamsWithoutTwoFactor' => auth()->user()->teams()->where('is_two_factor_required', false)->get(),
         ])->layout('layouts.simple');
     }
 }

app/Mcp/Tools/ManageApplication.php

@@ -16,7 +16,7 @@ class ManageApplication extends RestActionTool
     protected function actions(): array
     {
         return [
-            'update' => ['method' => 'PATCH', 'path' => 'applications/{uuid}', 'ability' => 'write', 'description' => 'update application settings (data: same fields as the REST update, e.g. name, domains, git_branch, ports_exposes, build_command; instant_deploy and manual_webhook_secret_* need deploy ability)'],
+            'update' => ['method' => 'PATCH', 'path' => 'applications/{uuid}', 'ability' => 'write', 'description' => 'update application settings (data: same fields as the REST update, e.g. name, domains, git_branch, ports_exposes, build_command, is_container_label_readonly_enabled, include_coolify_default_variables; instant_deploy and manual_webhook_secret_* need deploy ability)'],
             'delete' => ['method' => 'DELETE', 'path' => 'applications/{uuid}', 'ability' => 'deploy', 'confirm' => true, 'description' => 'delete the application and its containers (data: delete_volumes, delete_connected_networks, delete_configurations, docker_cleanup)'],
             'clone' => ['method' => 'POST', 'path' => 'applications/{uuid}/clone', 'ability' => 'write', 'description' => 'clone the application to a destination of the team (data: destination_uuid, name, clone_volumes; clone_volumes needs deploy ability)'],
             'move' => ['method' => 'POST', 'path' => 'applications/{uuid}/move', 'ability' => 'write', 'confirm' => true, 'description' => 'move the application to another environment of the team (data: environment_uuid)'],

app/Mcp/Tools/ManageDatabase.php

@@ -16,7 +16,7 @@ class ManageDatabase extends RestActionTool
     protected function actions(): array
     {
         return [
-            'update' => ['method' => 'PATCH', 'path' => 'databases/{uuid}', 'ability' => 'write', 'description' => 'update database settings (data: name, description, image, is_public, public_port, public_port_timeout, limits_*, type credentials and *_conf fields, redact_all_env_values_in_logs, instant_deploy; instant_deploy and changing is_public need deploy ability)'],
+            'update' => ['method' => 'PATCH', 'path' => 'databases/{uuid}', 'ability' => 'write', 'description' => 'update database settings (data: name, description, image, is_public, public_port, public_port_timeout, limits_*, type credentials and *_conf fields, redact_all_env_values_in_logs, is_log_drain_enabled, include_coolify_default_variables, instant_deploy; instant_deploy and changing is_public need deploy ability)'],
             'delete' => ['method' => 'DELETE', 'path' => 'databases/{uuid}', 'ability' => 'deploy', 'confirm' => true, 'description' => 'delete the database (data: delete_volumes, delete_connected_networks, delete_configurations, docker_cleanup; all default true)'],
             'clone' => ['method' => 'POST', 'path' => 'databases/{uuid}/clone', 'ability' => 'write', 'confirm' => true, 'description' => 'clone the database to a destination of the team (data: destination_uuid, name, clone_volumes; clone_volumes stops the source database while volumes are copied and needs deploy ability)'],
             'move' => ['method' => 'POST', 'path' => 'databases/{uuid}/move', 'ability' => 'write', 'confirm' => true, 'description' => 'move the database to another environment of the team (data: environment_uuid)'],

app/Mcp/Tools/ManageServer.php

@@ -17,7 +17,7 @@ class ManageServer extends RestActionTool
     {
         return [
             'create' => ['method' => 'POST', 'path' => 'servers', 'ability' => 'write', 'description' => 'add a server (data: ip, private_key_uuid, name, description, port, user, server_role, proxy_type, instant_validate)'],
-            'update' => ['method' => 'PATCH', 'path' => 'servers/{uuid}', 'ability' => 'write', 'description' => 'update a server (data: name, description, ip, port, user, private_key_uuid, server_role, proxy_type, concurrent_builds, dynamic_timeout, deployment_queue_limit, connection_timeout, is_terminal_enabled, server_disk_usage_*; changing proxy_type needs deploy ability)'],
+            'update' => ['method' => 'PATCH', 'path' => 'servers/{uuid}', 'ability' => 'write', 'description' => 'update a server (data: name, description, ip, port, user, private_key_uuid, server_role, proxy_type, concurrent_builds, dynamic_timeout, deployment_queue_limit, connection_timeout, is_terminal_enabled, server_disk_usage_*, wildcard_domain, server_timezone, backup_compression_cpu_percentage; changing proxy_type needs deploy ability)'],
             'delete' => ['method' => 'DELETE', 'path' => 'servers/{uuid}', 'ability' => 'write', 'confirm' => true, 'description' => 'delete a server without resources (the Coolify host cannot be deleted)'],
             'validate' => ['method' => 'POST', 'path' => 'servers/{uuid}/validate', 'ability' => 'write', 'description' => 'start a server validation (data: install=true also installs Docker and needs deploy ability)'],
         ];

app/Mcp/Tools/ManageService.php

@@ -16,11 +16,11 @@ class ManageService extends RestActionTool
     protected function actions(): array
     {
         return [
-            'update' => ['method' => 'PATCH', 'path' => 'services/{uuid}', 'ability' => 'write', 'description' => 'update the service (data: name, description, docker_compose_raw base64, connect_to_docker_network, urls, force_domain_override, is_container_label_escape_enabled, redact_all_env_values_in_logs, instant_deploy; instant_deploy needs deploy ability)'],
+            'update' => ['method' => 'PATCH', 'path' => 'services/{uuid}', 'ability' => 'write', 'description' => 'update the service (data: name, description, docker_compose_raw base64, connect_to_docker_network, urls, force_domain_override, is_container_label_escape_enabled, include_coolify_default_variables, redact_all_env_values_in_logs, instant_deploy; instant_deploy needs deploy ability)'],
             'delete' => ['method' => 'DELETE', 'path' => 'services/{uuid}', 'ability' => 'deploy', 'confirm' => true, 'description' => 'delete the service (data: delete_volumes, delete_connected_networks, delete_configurations, docker_cleanup)'],
             'clone' => ['method' => 'POST', 'path' => 'services/{uuid}/clone', 'ability' => 'write', 'confirm' => true, 'description' => 'clone the service to a destination of the team (data: destination_uuid, name, clone_volumes; clone_volumes stops the source service while volumes are copied and needs deploy ability)'],
             'move' => ['method' => 'POST', 'path' => 'services/{uuid}/move', 'ability' => 'write', 'confirm' => true, 'description' => 'move the service to another environment of the team (data: environment_uuid)'],
-            'update_application' => ['method' => 'PATCH', 'path' => 'services/{uuid}/applications/{app_uuid}', 'ability' => 'write', 'description' => 'update a service application (data: url, noindex_domains, human_name, description, image, exclude_from_status, is_log_drain_enabled, is_gzip_enabled, traefik_certificate_resolver, is_stripprefix_enabled, is_force_https_enabled, max_restart_count)'],
+            'update_application' => ['method' => 'PATCH', 'path' => 'services/{uuid}/applications/{app_uuid}', 'ability' => 'write', 'description' => 'update a service application (data: url, redirect, noindex_domains, human_name, description, image, exclude_from_status, is_log_drain_enabled, is_gzip_enabled, traefik_certificate_resolver, is_stripprefix_enabled, is_force_https_enabled, max_restart_count)'],
             'update_database' => ['method' => 'PATCH', 'path' => 'services/{uuid}/databases/{database_uuid}', 'ability' => 'write', 'description' => 'update a service database (data: human_name, description, image, exclude_from_status, is_log_drain_enabled, is_public, public_port, public_port_timeout; changing is_public needs deploy ability)'],
             'import_database' => ['method' => 'POST', 'path' => 'services/{uuid}/databases/{database_uuid}/imports', 'ability' => 'deploy', 'confirm' => true, 'description' => 'import a backup into a service database, can overwrite data (data: source s3 | server, s3_storage_uuid, path, dump_all, replace_existing, keep_owners, restore_mysql_users); check progress with get_database_import'],
         ];

app/Mcp/Tools/ManageStorage.php

@@ -17,7 +17,7 @@ class ManageStorage extends RestActionTool
     {
         return [
             'create' => ['method' => 'POST', 'paths' => $this->paths('storages'), 'ability' => 'write', 'description' => 'create a storage (data: type persistent|file, name, mount_path, content, is_directory, is_host_file, fs_path; services also resource_uuid; file content and an fs_path outside the resource directory need deploy ability)'],
-            'update' => ['method' => 'PATCH', 'paths' => $this->paths('storages'), 'ability' => 'write', 'description' => 'update a storage (data: uuid of the storage, type, name, mount_path, content, is_preview_suffix_enabled; content of a file outside the resource directory needs deploy ability)'],
+            'update' => ['method' => 'PATCH', 'paths' => $this->paths('storages'), 'ability' => 'write', 'description' => 'update a storage (data: uuid of the storage, type, name, mount_path, content, is_preview_suffix_enabled; a content change needs deploy ability; the content is saved right away and written to the server in the background, so a failed write does not change the response)'],
             'delete' => ['method' => 'DELETE', 'paths' => $this->paths('storages/{storage_uuid}'), 'ability' => 'write', 'confirm' => true, 'description' => 'delete a storage'],
             'set_backup' => ['method' => 'PUT', 'paths' => $this->paths('storages/{storage_uuid}/backups'), 'ability' => 'write', 'description' => 'create or replace the volume backup schedule (data: frequency, enabled, save_s3, s3_storage_uuid(s), stop_during_backup, retention fields ...; stop_during_backup needs deploy ability)'],
             'remove_backup' => ['method' => 'DELETE', 'paths' => $this->paths('storages/{storage_uuid}/backups'), 'ability' => 'write', 'confirm' => true, 'description' => 'delete the volume backup schedule and its local and S3 archives'],

app/Models/Application.php

@@ -1885,9 +1885,7 @@ class Application extends BaseModel
                     $fullRepoUrl = "{$this->source->html_url}/{$customRepository}";
                     $escapedRepoUrl = escapeshellarg("{$this->source->html_url}/{$customRepository}");
                     $git_clone_command = "{$git_clone_command} {$escapedRepoUrl} {$escapedBaseDir}";
-                    if (! $only_checkout) {
-                        $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout);
-                    }
+                    $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout);
                     if ($exec_in_docker) {
                         $commands->push($this->gitCommand(executeInDocker($deployment_uuid, $git_clone_command)));
                     } else {
@@ -1912,9 +1910,7 @@ class Application extends BaseModel
                         $git_clone_command = "{$git_clone_command} {$escapedRepoUrl} {$escapedBaseDir}";
                         $fullRepoUrl = $repoUrl;
                     }
-                    if (! $only_checkout) {
-                        $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: false, commit: $commit, gitConfigOptions: $gitConfigOption, baseDir: $baseDir, onlyCheckout: $only_checkout);
-                    }
+                    $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: false, commit: $commit, gitConfigOptions: $gitConfigOption, baseDir: $baseDir, onlyCheckout: $only_checkout);
                     if ($exec_in_docker) {
                         $commands->push($this->gitCommand(executeInDocker($deployment_uuid, $git_clone_command)));
                     } else {
@@ -1960,11 +1956,7 @@ class Application extends BaseModel
                     $escapedRepoUrl = escapeshellarg($repoUrl);
                     $fullRepoUrl = $repoUrl;
                     $git_clone_command_base = "{$git_clone_command} {$escapedRepoUrl} {$escapedBaseDir}";
-                    if ($only_checkout) {
-                        $git_clone_command = $git_clone_command_base;
-                    } else {
-                        $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitConfigOptions: $gitConfigOption, baseDir: $baseDir, onlyCheckout: $only_checkout);
-                    }
+                    $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitConfigOptions: $gitConfigOption, baseDir: $baseDir, onlyCheckout: $only_checkout);
 
                     if ($pull_request_id !== 0) {
                         $branch = "{$pullRequestRef}:{$pr_branch_name}";
@@ -2002,11 +1994,7 @@ class Application extends BaseModel
                     $gitlabSshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);
                     $gitlabGitSshCommand = "GIT_SSH_COMMAND=\"{$gitlabSshCommand}\"";
                     $git_clone_command_base = "{$gitlabGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
-                    if ($only_checkout) {
-                        $git_clone_command = $git_clone_command_base;
-                    } else {
-                        $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitSshCommand: $gitlabSshCommand, baseDir: $baseDir, onlyCheckout: $only_checkout);
-                    }
+                    $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitSshCommand: $gitlabSshCommand, baseDir: $baseDir, onlyCheckout: $only_checkout);
                     $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
 
                     if ($pull_request_id !== 0) {
@@ -2063,11 +2051,7 @@ class Application extends BaseModel
             $deployKeySshCommand = gitSshCommand($customPort, $customSshKeyLocation);
             $deployKeyGitSshCommand = "GIT_SSH_COMMAND=\"{$deployKeySshCommand}\"";
             $git_clone_command_base = "{$deployKeyGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
-            if ($only_checkout) {
-                $git_clone_command = $git_clone_command_base;
-            } else {
-                $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitSshCommand: $deployKeySshCommand, baseDir: $baseDir, onlyCheckout: $only_checkout);
-            }
+            $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base, commit: $commit, gitSshCommand: $deployKeySshCommand, baseDir: $baseDir, onlyCheckout: $only_checkout);
             $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
             if ($pull_request_id !== 0) {
                 if ($git_type === 'gitlab') {
@@ -2270,8 +2254,9 @@ class Application extends BaseModel
     {
         ['commands' => $cloneCommand, 'git' => $git] = $this->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: $checkoutDir);
 
+        // Callers read data from stdout after the checkout, so progress lines of the checkout go to stderr.
         return [
-            'command' => str_replace(' clone ', ' clone --quiet ', $this->gitCommandsAsShellCommand($cloneCommand)),
+            'command' => 'exec >&2 && '.str_replace(' clone ', ' clone --quiet ', $this->gitCommandsAsShellCommand($cloneCommand)),
             'git' => $git,
         ];
     }
@@ -2290,9 +2275,9 @@ class Application extends BaseModel
         $checkoutDir = "/tmp/{$uuid}/checkout";
         ['command' => $cloneCommand, 'git' => $git] = $this->serverCheckout($uuid, $checkoutDir);
         $composeFile = ltrim(rtrim($this->base_directory, '/').$this->docker_compose_location, '/');
-        $submoduleCommands = $this->composeFileSubmoduleCommands($git, $composeFile);
+        $submoduleCommand = $this->composeFileSubmoduleCommand($git, $composeFile, $checkoutDir);
         // Go into the submodule that contains the Compose file, and read the file from the objects of that submodule.
-        $enterSubmodule = $submoduleCommands === [] ? '' : implode(' ', [
+        $enterSubmodule = $submoduleCommand === null ? '' : implode(' ', [
             'while sub=$(git ls-tree -r HEAD | while read -r mode type object path; do',
             'if [ "$mode" = 160000 ]; then case "$file" in "$path"/*) echo "$path"; break ;; esac; fi; done) && [ -n "$sub" ];',
             'do cd "$sub" || exit 1; file="${file#"$sub"/}"; done;',
@@ -2317,9 +2302,9 @@ class Application extends BaseModel
             "mkdir -p /tmp/{$uuid}",
             "cd /tmp/{$uuid}",
             // One sh -c line: the non-root sudo parser would turn `&& if ...; then` into invalid `&& sudo if`.
-            'sh -c '.escapeshellarg($cloneCommand),
+            // The submodule step runs in the same script, because a deploy key is removed when the script exits.
+            'sh -c '.escapeshellarg($submoduleCommand === null ? $cloneCommand : "{$cloneCommand} && {$submoduleCommand}"),
             "cd {$checkoutDir}",
-            ...$submoduleCommands,
             'sh -c '.escapeshellarg($readComposeFile),
         ]);
     }
@@ -2328,23 +2313,21 @@ class Application extends BaseModel
      * The clone has no checkout and no submodules. If the Compose file is inside a submodule, this fills the
      * index (without a working tree) and clones only that submodule, with the credentials of the clone.
      *
-     * @return list<string>
+     * The command is appended to the clone script, so it must not be a separate sh -c line.
      */
-    private function composeFileSubmoduleCommands(string $git, string $composeFile): array
+    private function composeFileSubmoduleCommand(string $git, string $composeFile, string $checkoutDir): ?string
     {
         if (! $this->settings?->is_git_submodules_enabled) {
-            return [];
+            return null;
         }
 
         $depthFlag = $this->settings->is_git_shallow_clone_enabled ? ' --depth=1' : '';
         $escapedComposeFile = escapeshellarg($composeFile);
 
-        // One sh -c line: the non-root sudo parser cannot change commands inside a loop.
-        return ['git read-tree HEAD', 'sh -c '.escapeshellarg(
-            'git ls-files -s | while read -r mode object stage path; do '
+        return 'cd '.escapeshellarg($checkoutDir).' && git read-tree HEAD && '
+            .'git ls-files -s | while read -r mode object stage path; do '
             ."if [ \"\$mode\" = 160000 ]; then case {$escapedComposeFile} in \"\$path\"/*) "
-            ."{$git} submodule update --init --recursive{$depthFlag} -- \"\$path\" >/dev/null || exit 1 ;; esac; fi; done"
-        )];
+            ."{$git} submodule update --init --recursive{$depthFlag} -- \"\$path\" >/dev/null || exit 1 ;; esac; fi; done";
     }
 
     /**

app/Models/EnvironmentVariable.php

@@ -55,6 +55,17 @@ class EnvironmentVariable extends BaseModel
         'RAILPACK_SKIP_MIGRATIONS',
     ];
 
+    /**
+     * Magic variable commands (SERVICE_<COMMAND>_<NAME>) whose value generateEnvValue() generates.
+     */
+    public const GENERATED_MAGIC_VARIABLE_COMMANDS = [
+        'PASSWORD', 'PASSWORD_64', 'PASSWORDWITHSYMBOLS', 'PASSWORDWITHSYMBOLS_64',
+        'BASE64', 'BASE64_32', 'BASE64_64', 'BASE64_128',
+        'REALBASE64', 'REALBASE64_32', 'REALBASE64_64', 'REALBASE64_128',
+        'HEX_32', 'HEX_64', 'HEX_128',
+        'USER', 'LOWERCASEUSER', 'SUPABASEANON', 'SUPABASESERVICE',
+    ];
+
     protected $attributes = [
         'is_runtime' => true,
         'is_buildtime' => true,
@@ -125,7 +136,8 @@ class EnvironmentVariable extends BaseModel
 
                 if (! $found) {
                     $application = Application::find($environment_variable->resourceable_id);
-                    if ($application) {
+                    // A compose preview generates its own magic secrets on the first pull request parse.
+                    if ($application && ! ($application->build_pack === 'dockercompose' && $environment_variable->isGeneratedMagicVariable())) {
                         ModelsEnvironmentVariable::create([
                             'key' => $environment_variable->key,
                             'value' => $environment_variable->value,
@@ -156,6 +168,12 @@ class EnvironmentVariable extends BaseModel
         });
     }
 
+    public function isGeneratedMagicVariable(): bool
+    {
+        return str_starts_with($this->key, 'SERVICE_')
+            && in_array(parseCommandFromMagicEnvVariable($this->key)->value(), self::GENERATED_MAGIC_VARIABLE_COMMANDS, true);
+    }
+
     public function preservesLegacyComposeNull(): bool
     {
         return $this->preserve_compose_null && $this->value === null;

app/Services/ContainerFilesystemService.php

@@ -402,12 +402,15 @@ class ContainerFilesystemService
             $command = parseCommandsByLineForSudo(collect([$command]), $this->server)[0];
         }
 
-        $process = proc_open($this->sshCommand($command), [1 => ['pipe', 'w'], 2 => ['file', '/dev/null', 'w']], $pipes);
+        $process = proc_open($this->sshCommand(), [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['file', '/dev/null', 'w']], $pipes);
         if (! is_resource($process)) {
             throw new \RuntimeException('Could not start the download.');
         }
 
         try {
+            // The command goes to ssh on stdin, so it is not in a command line on the Coolify host.
+            fwrite($pipes[0], SshMultiplexingHelper::sshBatchInput($command));
+            fclose($pipes[0]);
             while (! feof($pipes[1])) {
                 $chunk = fread($pipes[1], 64 * 1024);
                 if ($chunk === false || connection_aborted()) {
@@ -450,9 +453,9 @@ class ContainerFilesystemService
         instant_remote_process([$this->buildS3UploadCommand($path, $isDir, $url, $headers)], $this->server, timeout: 3600);
     }
 
-    protected function sshCommand(string $command): string
+    protected function sshCommand(): string
     {
-        return SshMultiplexingHelper::generateSshCommand($this->server, $command);
+        return SshMultiplexingHelper::generateSshBatchCommand($this->server);
     }
 
     protected function escapePath(string $path, string $label): string

app/Services/DatabaseStartCommandExecutor.php

@@ -23,7 +23,8 @@ class DatabaseStartCommandExecutor
         $secrets = method_exists($database, 'resolvedSecretManagerValuesForRedaction')
             ? $database->resolvedSecretManagerValuesForRedaction()
             : [];
-        $remoteCommand = SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands));
+        // The commands go to ssh on stdin, so the database passwords in them are not in a command line on the Coolify host.
+        $remoteCommand = SshMultiplexingHelper::generateSshBatchCommand($server);
 
         $activity->properties = $activity->properties->merge(['status' => ProcessStatus::IN_PROGRESS->value]);
         $activity->save();
@@ -34,6 +35,7 @@ class DatabaseStartCommandExecutor
         try {
             $process = Process::timeout($timeout)
                 ->idleTimeout(3600)
+                ->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
                 ->start($remoteCommand, function (string $type, string $output) use ($activity, $secrets, &$pending): void {
                     $pending[$type] = ($pending[$type] ?? '').$output;
                     $end = max(strrpos($pending[$type], "\n"), strrpos($pending[$type], "\r"));

app/Services/DockerRegistryLogins.php

@@ -216,10 +216,9 @@ class DockerRegistryLogins
     /**
      * Run docker login on the server, writing to the same config file that deployments mount.
      *
-     * The SSH wrapper sends this script to the server's shell on stdin, and the shell builtin echo passes the
-     * secret to docker on stdin, so it is not a process argument on the server. On the Coolify host it is still
-     * in the command line of the local shell that runs ssh, because SshMultiplexingHelper::generateSshCommand()
-     * embeds the script in a heredoc. It is kept out of SSH error logs and exception messages.
+     * The script goes to the server's shell on ssh stdin, and the shell builtin echo passes the secret to docker
+     * on stdin, so it is not a process argument on the Coolify host or on the server. It is kept out of SSH error
+     * logs and exception messages.
      */
     public static function login(Server $server, string $registry, string $username, string $password): void
     {
@@ -277,7 +276,9 @@ class DockerRegistryLogins
     {
         $commands = $server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $server) : [$command];
         try {
-            $process = Process::timeout(60)->run(SshMultiplexingHelper::generateSshCommand($server, implode("\n", $commands)));
+            $process = Process::timeout(60)
+                ->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
+                ->run(SshMultiplexingHelper::generateSshBatchCommand($server));
         } catch (ProcessTimedOutException|SymfonyProcessTimedOutException) {
             // The timeout message contains the full SSH command, including the registry password.
             throw new RuntimeException('The command on the server timed out after 60 seconds.', 124);

app/Services/SentinelTrafficClient.php

@@ -727,9 +727,9 @@ class SentinelTrafficClient
         $commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
 
         try {
-            $process = Process::timeout(self::REMOTE_TIMEOUT)->run(
-                SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
-            );
+            $process = Process::timeout(self::REMOTE_TIMEOUT)
+                ->input(SshMultiplexingHelper::sshBatchInput(implode("\n", $commands)))
+                ->run(SshMultiplexingHelper::generateSshBatchCommand($this->server, commandTimeout: self::REMOTE_TIMEOUT));
 
             if ($process->exitCode() !== 0) {
                 excludeCertainErrors($process->errorOutput(), $process->exitCode());

app/Services/ServerTransfer/ServerTransferBundle.php

@@ -213,6 +213,7 @@ class ServerTransferBundle
                 }
 
                 foreach (self::items($environment['databases'] ?? null) as $d => $database) {
+                    self::checkDatabasePorts($errors, "{$at}.databases.{$d}", (array) ($database['attributes'] ?? []));
                     self::checkStorages($errors, "{$at}.databases.{$d}", $database);
                     self::checkScheduledBackups($errors, "{$at}.databases.{$d}", $database);
                 }
@@ -395,4 +396,38 @@ class ServerTransferBundle
             self::checkPath($errors, "{$at}.scheduled_backups.{$i}.databases_to_backup", $backup['databases_to_backup'] ?? null);
         }
     }
+
+    /**
+     * Checks the port values of a database. Older databases were saved before the UI checked them
+     * (March 2026), so this accepts any public port in 1-65535, spaces around mappings, empty entries,
+     * and container-only ports. It refuses values that are not ports, because they reach the compose file.
+     *
+     * @param  list<string>  $errors
+     * @param  array<string, mixed>  $attributes
+     */
+    private static function checkDatabasePorts(array &$errors, string $at, array $attributes): void
+    {
+        if (array_key_exists('public_port', $attributes)) {
+            self::check($errors, "{$at}.public_port", $attributes['public_port'], ['nullable', 'integer', 'between:1,65535']);
+        }
+
+        if (! array_key_exists('ports_mappings', $attributes) || $attributes['ports_mappings'] === null) {
+            return;
+        }
+        $mappings = $attributes['ports_mappings'];
+        if (! is_string($mappings)) {
+            $errors[] = "{$at}.ports_mappings must be a string.";
+
+            return;
+        }
+        foreach (explode(',', $mappings) as $mapping) {
+            $mapping = trim($mapping, " \t");
+            if ($mapping === '' || preg_match('/^\d+(?:-\d+)?(?:\/(?:tcp|udp|sctp))?$/', $mapping) === 1 || preg_match(ValidationPatterns::PORT_MAPPINGS_PATTERN, $mapping) === 1) {
+                continue;
+            }
+            $errors[] = "{$at}.ports_mappings must be a comma-separated list of port mappings (e.g. 5432:5432,127.0.0.1:8080:80/udp).";
+
+            return;
+        }
+    }
 }

app/Support/DatabaseBackupFileValidator.php

@@ -100,6 +100,8 @@ class DatabaseBackupFileValidator
             return false;
         }
 
+        $sql = self::withoutCopyData($sql);
+
         // Token separator PostgreSQL treats as whitespace: real whitespace, a block comment
         // (the lazy match can extend to the outer end of a nested comment), or a line
         // comment. Comment forms also match inside string literals, which can only block
@@ -120,6 +122,29 @@ class DatabaseBackupFileValidator
         return preg_match('~(?:from|to)'.$sep.'+program|lo_(?:import|export)'.$sep.'*\(|alter'.$sep.'+system'.$sep.'+(?:set|reset)~is', $sql) !== 0;
     }
 
+    /**
+     * Removes the table data of COPY ... FROM stdin blocks: rows are data, not SQL, and ordinary
+     * text such as "how to program" matched the patterns. Same rule as the restore scan in
+     * DatabaseImportCommandBuilder: a block starts at the line "COPY ... FROM stdin;" as pg_dump
+     * writes it and ends at the line "\.".
+     */
+    private static function withoutCopyData(string $sql): string
+    {
+        $kept = '';
+        $offset = 0;
+
+        while (preg_match('/^COPY .* FROM stdin;$/m', $sql, $match, PREG_OFFSET_CAPTURE, $offset) === 1) {
+            $dataStart = $match[0][1] + strlen($match[0][0]);
+            $kept .= substr($sql, $offset, $dataStart - $offset);
+            $end = preg_match('/^\\\\\\.$/m', $sql, $terminator, PREG_OFFSET_CAPTURE, $dataStart) === 1
+                ? $terminator[0][1] + 2
+                : strlen($sql);
+            $offset = $end;
+        }
+
+        return $kept.substr($sql, $offset);
+    }
+
     private static function extensionFor(string $name): ?string
     {
         $lower = strtolower($name);
@@ -152,14 +177,14 @@ class DatabaseBackupFileValidator
         $sample = (string) file_get_contents($path, false, null, 0, 4096);
 
         return match ($extension) {
-            'sql' => self::looksLikeText($sample) && ! self::containsPostgresqlProgramExecution($sample),
+            'sql' => self::looksLikeText($sample),
             'sql.gz', 'gz', 'tar.gz', 'tgz', 'bson.gz', 'archive.gz', 'db.gz', 'sqlite.gz', 'sqlite3.gz' => str_starts_with($sample, "\x1f\x8b"),
             'zip' => str_starts_with($sample, "PK\x03\x04") || str_starts_with($sample, "PK\x05\x06") || str_starts_with($sample, "PK\x07\x08"),
             'tar' => substr($sample, 257, 5) === 'ustar',
             'bz2' => str_starts_with($sample, 'BZh'),
             'xz' => str_starts_with($sample, "\xfd7zXZ\x00"),
             'dump', 'bak', 'archive', 'dmp' => str_starts_with($sample, 'PGDMP')
-                || (self::looksLikeText($sample) && ! self::containsPostgresqlProgramExecution($sample)),
+                || self::looksLikeText($sample),
             'bson' => self::looksLikeBson($path, $sample),
             'db', 'sqlite', 'sqlite3' => str_starts_with($sample, "SQLite format 3\0"),
             default => false,

app/Support/DatabaseImport/DatabaseImportCommandBuilder.php

@@ -54,6 +54,11 @@ tolower($0) ~ /^[ \t]*create[ \t]+(database|schema)[ \t]/ { if (is_system(databa
 !skip { print }
 AWK;
 
+    /**
+     * Prints a PostgreSQL SQL dump without the table data of its COPY ... FROM stdin blocks.
+     */
+    private const POSTGRES_COPY_DATA_FILTER = 'data { if ($0 == "\\\\.") data = 0; next } { print } /^COPY .* FROM stdin;$/ { data = 1 }';
+
     /**
      * @param  bool  $keepOwners  PostgreSQL archives: restore object owners and privileges. Off by default,
      *                            because roles from another host (for example RDS) usually do not exist here.
@@ -135,13 +140,19 @@ SH;
         // Comments are no longer stripped: deleting the rest of a line on "--" also cut
         // inside string literals, which let "SELECT '--'; COPY ... PROGRAM" slip through.
         // A commented-out PROGRAM statement is now blocked too, which is the safer default.
-        $scan = static fn (string $source): string => "{$source} | LC_ALL=C grep -Eiq {$escapedPsqlPattern} || {$source} | tr -d '~' | tr '\\n\\r' '~~' | LC_ALL=C grep -Eiq {$escapedSqlPattern}";
+        // Table data of COPY ... FROM stdin blocks is not scanned: it is data, not SQL, and ordinary
+        // text such as "how to program" matched the patterns. pg_dump writes each block as the line
+        // "COPY ... FROM stdin;", the rows, and the line "\.". When awk is missing or fails, the
+        // scan receives a line that matches, so the restore is blocked.
+        $withoutCopyData = static fn (string $source): string => "{ {$source} | LC_ALL=C awk ".escapeshellarg(self::POSTGRES_COPY_DATA_FILTER)." || echo '\\! from program'; }";
+        $scan = static fn (string $source): string => $withoutCopyData($source)." | LC_ALL=C grep -Eiq {$escapedPsqlPattern} || ".$withoutCopyData($source)." | tr -d '~' | tr '\\n\\r' '~~' | LC_ALL=C grep -Eiq {$escapedSqlPattern}";
         $customScan = $scan('pg_restore -f - "$inspect" 2>/dev/null');
         $sqlScan = $scan($contents);
         $blockedProgram = 'echo \'Blocked PostgreSQL restore: the backup contains a COPY ... PROGRAM, psql shell command, lo_import/lo_export, or ALTER SYSTEM pattern. If you trust this dump, restore it from the database container terminal.\'; exit 1';
         $blockedInspect = 'echo \'Blocked PostgreSQL restore: unable to inspect custom archive.\'; exit 1';
 
         return <<<SH
+command -v awk >/dev/null 2>&1 || { echo 'Blocked PostgreSQL restore: awk is required to scan the backup.'; exit 1; }
 header=\$({$contents} | head -c 5)
 tar_magic=\$({$contents} | head -c 262 | tail -c 5)
 if [ "\$header" = 'PGDMP' ] || [ "\$tar_magic" = 'ustar' ]; then
@@ -270,6 +281,12 @@ SH;
     /**
      * Checks the backup format before any database is dropped, then recreates the
      * target database and restores archives with pg_restore and SQL with psql.
+     * The restore must not stop on the first error: a pg_dumpall dump always creates the
+     * superuser role (and creates the target database) that already exist here, and an
+     * archive with owners references roles this server may not have. Stopping there would
+     * leave the server without the databases that were just dropped.
+     * pg_dumpall does not create the postgres database (it expects it to exist), so it is
+     * recreated too when POSTGRES_DB names another database.
      */
     private function postgresqlDumpAll(): string
     {
@@ -287,10 +304,11 @@ check_superuser
 psql -U "$pg_user" -d template1 -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" || exit 1
 psql -U "$pg_user" -d template1 -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U "$pg_user" --maintenance-db=template1 --if-exists {} || exit 1
 createdb -U "$pg_user" "$db" || exit 1
+[ "$db" = postgres ] || createdb -U "$pg_user" postgres || exit 1
 if [ "$kind" = archive ]; then
-  stream | pg_restore --exit-on-error -U "$pg_user" -d "$db"
+  stream | pg_restore -U "$pg_user" -d "$db"
 else
-  stream | psql -v ON_ERROR_STOP=1 -U "$pg_user" -d "$db"
+  stream | psql -U "$pg_user" -d "$db"
 fi
 SH;
     }

app/Traits/ExecuteRemoteCommand.php

@@ -159,9 +159,9 @@ trait ExecuteRemoteCommand
     }
 
     /**
-     * Execute the actual command with process handling. With $input, the content goes to the command over
-     * SSH stdin, because Linux limits one exec argument to 128 KiB and generateSshCommand() puts the whole
-     * script into one argument.
+     * Execute the actual command with process handling. Without $input, the command goes to the remote shell
+     * over SSH stdin, so its secrets are not in a command line on the Coolify host. With $input, the command
+     * is the SSH argument and the content goes to it over SSH stdin.
      */
     private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden = false, $skip_command_log = false, ?string $input = null)
     {
@@ -169,9 +169,12 @@ trait ExecuteRemoteCommand
             $this->application_deployment_queue->addLogEntry('[CMD]: '.$this->redact_sensitive_info($command), hidden: true);
         }
 
-        $remote_command = $input === null
-            ? SshMultiplexingHelper::generateSshCommand($this->server, $command)
-            : SshMultiplexingHelper::generateSshStdinCommand($this->server, $command);
+        if ($input === null) {
+            $remote_command = SshMultiplexingHelper::generateSshBatchCommand($this->server);
+            $input = SshMultiplexingHelper::sshBatchInput((string) $command);
+        } else {
+            $remote_command = SshMultiplexingHelper::generateSshStdinCommand($this->server, $command);
+        }
         $addLogEntry = function (string $type, string $output) use ($command, $hidden, $customType, $command_hidden, $skip_command_log): void {
             // Sanitize output to ensure valid UTF-8 encoding before JSON encoding
             $log_output = str(sanitize_utf8_text($output))->trim();

bootstrap/helpers/applications.php

@@ -1,6 +1,6 @@
 <?php
 
-use App\Actions\Application\StopApplication;
+use App\Actions\Shared\CloneVolumeDataWithSourceStopped;
 use App\Enums\ApplicationDeploymentStatus;
 use App\Jobs\ApplicationDeploymentJob;
 use App\Jobs\VolumeCloneJob;
@@ -447,6 +447,7 @@ function clone_application(Application $source, $destination, array $overrides =
     }
 
     // Clone persistent volumes
+    $pendingVolumeClones = [];
     $persistentVolumes = $source->persistentStorages()->get();
     foreach ($persistentVolumes as $volume) {
         $newName = '';
@@ -472,28 +473,18 @@ function clone_application(Application $source, $destination, array $overrides =
         $newPersistentVolume->save();
 
         if ($cloneVolumeData) {
-            try {
-                StopApplication::dispatch($source, false, false);
-                $sourceVolume = $volume->name;
-                $targetVolume = $newPersistentVolume->name;
-                $sourceServer = $source->destination->server;
-                $targetServer = $newApplication->destination->server;
-
-                VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume);
-
-                queue_application_deployment(
-                    deployment_uuid: new_public_id(),
-                    application: $source,
-                    server: $sourceServer,
-                    destination: $source->destination,
-                    no_questions_asked: true
-                );
-            } catch (Exception $e) {
-                Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage());
-            }
+            $pendingVolumeClones[] = new VolumeCloneJob(
+                $volume->name,
+                $newPersistentVolume->name,
+                $source->destination->server,
+                $newApplication->destination->server,
+                $newPersistentVolume,
+            );
         }
     }
 
+    CloneVolumeDataWithSourceStopped::run($source, $pendingVolumeClones);
+
     // Clone file storages
     $fileStorages = $source->fileStorages()->get();
     foreach ($fileStorages as $storage) {

bootstrap/helpers/auth.php

@@ -3,19 +3,49 @@
 use Illuminate\Http\Request;
 
 /**
- * The client IP for rate limits, allow-lists, and audit logs. Behind Cloudflare, $request->ip()
- * is the Cloudflare edge, so a valid CF-Connecting-IP is used: on cloud, and on self-hosted
- * instances when the request comes from a Cloudflare IP. A client that connects directly
- * cannot set the client IP with the header.
+ * The client IP for rate limits, allow-lists, and audit logs.
+ *
+ * TrustProxies trusts every peer, so a client that connects directly to the dashboard port could
+ * set $request->ip() with X-Forwarded-For. The TCP peer (REMOTE_ADDR) decides what to trust:
+ * - A public peer is the client. Forwarded headers are ignored, except CF-Connecting-IP when the
+ *   peer is a Cloudflare edge.
+ * - A private or loopback peer is a local reverse proxy (coolify-proxy, cloudflared, a Docker
+ *   network), so the address that proxy forwarded is used. CF-Connecting-IP is used only when
+ *   that address is a Cloudflare edge.
+ * On cloud, which runs behind Cloudflare, a valid CF-Connecting-IP is always used.
  */
 function client_ip(Request $request): string
 {
-    $ip = (string) $request->ip();
     $cloudflareIp = $request->header('CF-Connecting-IP');
+    $cloudflareIp = is_string($cloudflareIp) && filter_var($cloudflareIp, FILTER_VALIDATE_IP) !== false ? $cloudflareIp : null;
 
-    if (is_string($cloudflareIp) && filter_var($cloudflareIp, FILTER_VALIDATE_IP) !== false && (isCloud() || isCloudflareIp($ip))) {
+    $peer = (string) $request->server('REMOTE_ADDR');
+    $hop = isLocalProxyIp($peer) ? (string) $request->ip() : $peer;
+
+    if ($cloudflareIp !== null && (isCloud() || isCloudflareIp($hop))) {
         return $cloudflareIp;
     }
 
-    return $ip;
+    return $hop;
+}
+
+/**
+ * Whether an address is loopback, private, or link-local (IPv4, IPv6, or IPv4-mapped IPv6): a
+ * reverse proxy on the same host or Docker network rather than a client on the internet.
+ */
+function isLocalProxyIp(string $ip): bool
+{
+    if (filter_var($ip, FILTER_VALIDATE_IP) === false) {
+        return false;
+    }
+
+    $packed = inet_pton($ip);
+    if (strlen($packed) === 16 && str_starts_with($packed, str_repeat("\0", 10)."\xff\xff")) {
+        $ip = inet_ntop(substr($packed, 12));
+    }
+
+    return ipMatch($ip, [
+        '127.0.0.0/8', '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '169.254.0.0/16',
+        '::1/128', 'fc00::/7', 'fe80::/10',
+    ]);
 }

bootstrap/helpers/parsers.php

@@ -1151,6 +1151,8 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
 
     $pullRequestId = $pull_request_id;
     $isPullRequest = $pullRequestId == 0 ? false : true;
+    // A public preview can run the compose file of a fork pull request.
+    $isPublicPullRequest = $isPullRequest && (bool) data_get($resource, 'settings.is_pr_deployments_public_enabled');
     $server = data_get($resource, 'destination.server');
     try {
         $yaml = parseDockerComposeYaml($compose);
@@ -1216,12 +1218,6 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
         // convert environment variables to one format
         $environment = convertToKeyValueCollection($environment);
 
-        // Add Coolify defined environments
-        $allEnvironments = $resource->environment_variables()->get(['key', 'value']);
-
-        $allEnvironments = $allEnvironments->mapWithKeys(function ($item) {
-            return [$item['key'] => $item['value']];
-        });
         // filter and add magic environments
         foreach ($environment as $key => $value) {
             // Get all SERVICE_ variables from keys and values
@@ -1269,7 +1265,8 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                     $resource->save();
                 }
 
-                if (! $parsed['has_port']) {
+                // The stored variables belong to the main application; a preview deployment gets its own domains.
+                if (! $parsed['has_port'] && ! $isPullRequest) {
                     $resource->environment_variables()->updateOrCreate([
                         'key' => $key->value(),
                         'resourceable_type' => get_class($resource),
@@ -1279,7 +1276,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                         'is_preview' => false,
                     ]);
                 }
-                if ($parsed['has_port']) {
+                if ($parsed['has_port'] && ! $isPullRequest) {
 
                     $newKey = str($key)->beforeLast('_');
                     $resource->environment_variables()->updateOrCreate([
@@ -1375,45 +1372,48 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                         $fqdnValueForEnvWithPort = "$fqdnValueForEnv:$port";
                     }
 
-                    // ALWAYS create base SERVICE_FQDN variable (host only, no scheme)
-                    $resource->environment_variables()->firstOrCreate([
-                        'key' => "SERVICE_FQDN_{$serviceNamePreserved}",
-                        'resourceable_type' => get_class($resource),
-                        'resourceable_id' => $resource->id,
-                    ], [
-                        'value' => $fqdnValueForEnv,
-                        'is_preview' => false,
-                    ]);
-
-                    // ALWAYS create base SERVICE_URL variable (with scheme)
-                    $resource->environment_variables()->firstOrCreate([
-                        'key' => "SERVICE_URL_{$serviceNamePreserved}",
-                        'resourceable_type' => get_class($resource),
-                        'resourceable_id' => $resource->id,
-                    ], [
-                        'value' => $url,
-                        'is_preview' => false,
-                    ]);
-
-                    // If port-specific, ALSO create port-specific pairs
-                    if ($parsed['has_port'] && $port) {
+                    // The stored variables belong to the main application; a preview deployment gets its own domains.
+                    if (! $isPullRequest) {
+                        // ALWAYS create base SERVICE_FQDN variable (host only, no scheme)
                         $resource->environment_variables()->firstOrCreate([
-                            'key' => "SERVICE_FQDN_{$serviceNamePreserved}_{$port}",
+                            'key' => "SERVICE_FQDN_{$serviceNamePreserved}",
                             'resourceable_type' => get_class($resource),
                             'resourceable_id' => $resource->id,
                         ], [
-                            'value' => $fqdnValueForEnvWithPort,
+                            'value' => $fqdnValueForEnv,
                             'is_preview' => false,
                         ]);
 
+                        // ALWAYS create base SERVICE_URL variable (with scheme)
                         $resource->environment_variables()->firstOrCreate([
-                            'key' => "SERVICE_URL_{$serviceNamePreserved}_{$port}",
+                            'key' => "SERVICE_URL_{$serviceNamePreserved}",
                             'resourceable_type' => get_class($resource),
                             'resourceable_id' => $resource->id,
                         ], [
-                            'value' => $urlWithPort,
+                            'value' => $url,
                             'is_preview' => false,
                         ]);
+
+                        // If port-specific, ALSO create port-specific pairs
+                        if ($parsed['has_port'] && $port) {
+                            $resource->environment_variables()->firstOrCreate([
+                                'key' => "SERVICE_FQDN_{$serviceNamePreserved}_{$port}",
+                                'resourceable_type' => get_class($resource),
+                                'resourceable_id' => $resource->id,
+                            ], [
+                                'value' => $fqdnValueForEnvWithPort,
+                                'is_preview' => false,
+                            ]);
+
+                            $resource->environment_variables()->firstOrCreate([
+                                'key' => "SERVICE_URL_{$serviceNamePreserved}_{$port}",
+                                'resourceable_type' => get_class($resource),
+                                'resourceable_id' => $resource->id,
+                            ], [
+                                'value' => $urlWithPort,
+                                'is_preview' => false,
+                            ]);
+                        }
                     }
 
                     if ($resource->build_pack === 'dockercompose') {
@@ -1437,6 +1437,21 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                             }
                         }
                     }
+                } elseif ($isPullRequest) {
+                    // A preview deployment reads its preview variables, so a pull request parse generates the value there.
+                    $previewVariable = $resource->environment_variables_preview()->firstOrCreate([
+                        'key' => $key->value(),
+                        'resourceable_type' => get_class($resource),
+                        'resourceable_id' => $resource->id,
+                    ], [
+                        'value' => generateEnvValue($command, $resource),
+                        'is_preview' => true,
+                    ]);
+                    // A public preview does not get the production secret; it gets its own value.
+                    if ($isPublicPullRequest && ! $previewVariable->wasRecentlyCreated
+                        && $previewVariable->value === $resource->environment_variables()->where('key', $key->value())->first()?->value) {
+                        $previewVariable->update(['value' => generateEnvValue($command, $resource)]);
+                    }
                 } else {
                     $value = generateEnvValue($command, $resource);
                     $resource->environment_variables()->firstOrCreate([
@@ -1452,6 +1467,9 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
         }
     }
 
+    // A pull request parse keeps its magic variables in the preview variables (see above), not in the production variables.
+    $isPullRequestMagicVariable = fn (string $key): bool => $isPullRequest && $allMagicEnvironments->has($key);
+
     // generate SERVICE_NAME variables for docker compose services
     $serviceNameEnvironments = collect([]);
     if ($resource->build_pack === 'dockercompose') {
@@ -1863,15 +1881,17 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                             $isRequired = str_contains($operator, '?');
 
                             // Create the primary variable with its default (only if it doesn't exist)
-                            $envVar = $resource->environment_variables()->firstOrCreate([
-                                'key' => $varName,
-                                'resourceable_type' => get_class($resource),
-                                'resourceable_id' => $resource->id,
-                            ], [
-                                'value' => $isRequired ? null : $defaultValue,
-                                'is_preview' => false,
-                                'is_required' => $isRequired,
-                            ]);
+                            if (! $isPullRequestMagicVariable($varName)) {
+                                $envVar = $resource->environment_variables()->firstOrCreate([
+                                    'key' => $varName,
+                                    'resourceable_type' => get_class($resource),
+                                    'resourceable_id' => $resource->id,
+                                ], [
+                                    'value' => $isRequired ? null : $defaultValue,
+                                    'is_preview' => false,
+                                    'is_required' => $isRequired,
+                                ]);
+                            }
 
                             // Recursively process nested variables in default value
                             if (str_contains($defaultValue, '${')) {
@@ -1887,7 +1907,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                                     // Skip SERVICE_URL_* and SERVICE_FQDN_* variables - they are handled by magic variable system
                                     $isMagicVariable = str_starts_with($nestedVarName, 'SERVICE_URL_') || str_starts_with($nestedVarName, 'SERVICE_FQDN_');
 
-                                    if (! $isMagicVariable) {
+                                    if (! $isMagicVariable && ! $isPullRequestMagicVariable($nestedVarName)) {
                                         if ($nestedSplit !== null) {
                                             $nestedEnvVar = $resource->environment_variables()->firstOrCreate([
                                                 'key' => $nestedSplit['variable'],
@@ -1918,14 +1938,16 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                         } else {
                             // Simple variable reference without default
                             $parsedKeyValue = replaceVariables($value);
-                            $envVar = $resource->environment_variables()->firstOrCreate([
-                                'key' => $content,
-                                'resourceable_type' => get_class($resource),
-                                'resourceable_id' => $resource->id,
-                            ], [
-                                'is_preview' => false,
-                                'is_required' => $isRequired,
-                            ]);
+                            if (! $isPullRequestMagicVariable($content)) {
+                                $envVar = $resource->environment_variables()->firstOrCreate([
+                                    'key' => $content,
+                                    'resourceable_type' => get_class($resource),
+                                    'resourceable_id' => $resource->id,
+                                ], [
+                                    'is_preview' => false,
+                                    'is_required' => $isRequired,
+                                ]);
+                            }
                         }
                     } else {
                         // Fallback to old behavior for malformed input (backward compatibility)
@@ -2108,7 +2130,8 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
         if ($environment->count() > 0) {
             $environment = $environment->filter(function ($value, $key) {
                 return ! str($key)->startsWith('SERVICE_FQDN_');
-            })->map(function ($value, $key) use ($resource, $isPullRequest) {
+            })->map(function ($value, $key) use ($resource, $isPullRequest, $isPublicPullRequest) {
+                $isValueFromDatabase = false;
                 // Preserve empty strings and null values with correct Docker Compose semantics:
                 // - Empty string: Variable is set to "" (e.g., HTTP_PROXY="" means "no proxy")
                 // - Null: Variable is unset/removed from container environment (may inherit from host)
@@ -2119,10 +2142,18 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                 } elseif ($value === '' && ! ($isPullRequest && str($key)->startsWith('SERVICE_URL_'))) {
                     // Empty string - allow database override for backward compatibility.
                     // The stored service URLs belong to the main application, so a preview does not use them.
-                    $dbEnv = $resource->environment_variables()->where('key', $key)->first();
+                    // A preview uses its preview variables, like ApplicationDeploymentJob does.
+                    $dbEnv = $isPullRequest
+                        ? $resource->environment_variables_preview()->where('key', $key)->first()
+                        : $resource->environment_variables()->where('key', $key)->first();
+                    // A public preview does not get preview variables that still have the production value.
+                    if ($dbEnv && $isPublicPullRequest && $resource->environment_variables()->where('key', $key)->first()?->value === $dbEnv->value) {
+                        $dbEnv = null;
+                    }
                     // Only use database override if it exists AND has a non-empty value
                     if ($dbEnv && str($dbEnv->value)->isNotEmpty()) {
                         $value = $dbEnv->value;
+                        $isValueFromDatabase = true;
                     }
                     // Otherwise keep empty string as-is
                 }
@@ -2130,7 +2161,8 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
                 // Resolve shared variable patterns like {{environment.VAR}}, {{project.VAR}}, {{team.VAR}}
                 // Without this, literal {{...}} strings end up in the compose environment: section,
                 // which takes precedence over the resolved values in the .env file (env_file:)
-                if (is_string($value) && str_contains($value, '{{')) {
+                // A public preview resolves them only in its own variables, not in the compose file of the pull request.
+                if (is_string($value) && str_contains($value, '{{') && ($isValueFromDatabase || ! $isPublicPullRequest)) {
                     $value = resolveSharedEnvironmentVariables($value, $resource);
                 }
 

bootstrap/helpers/remoteProcess.php

@@ -192,8 +192,8 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $
 
     return SshRetryHandler::retry(
         function () use ($server, $command_string) {
-            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
-            $process = runSshProcess($sshCommand, 30);
+            $sshCommand = SshMultiplexingHelper::generateSshBatchCommand($server);
+            $process = runSshProcess($sshCommand, 30, SshMultiplexingHelper::sshBatchInput($command_string));
 
             $output = trim($process->output());
             $exitCode = $process->exitCode();
@@ -228,8 +228,8 @@ function instant_remote_process(Collection|array $command, Server $server, bool
 
     return SshRetryHandler::retry(
         function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
-            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
-            $process = runSshProcess($sshCommand, (int) $effectiveTimeout);
+            $sshCommand = SshMultiplexingHelper::generateSshBatchCommand($server, $disableMultiplexing, (int) $effectiveTimeout);
+            $process = runSshProcess($sshCommand, (int) $effectiveTimeout, SshMultiplexingHelper::sshBatchInput($command_string));
 
             $output = trim($process->output());
             $exitCode = $process->exitCode();

bootstrap/helpers/services.php

@@ -25,8 +25,7 @@ function service_logo_urls(mixed $logo): array
     ];
 }
 
-use App\Actions\Service\StartService;
-use App\Actions\Service\StopService;
+use App\Actions\Shared\CloneVolumeDataWithSourceStopped;
 use App\Jobs\VolumeCloneJob;
 use App\Models\Application;
 use App\Models\GithubApp;
@@ -38,8 +37,6 @@ use App\Models\ServiceDatabase;
 use App\Models\StandaloneDocker;
 use App\Models\SwarmDocker;
 use Illuminate\Database\Eloquent\Model;
-use Illuminate\Support\Facades\Bus;
-use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Stringable;
 use Spatie\Url\Url;
@@ -778,22 +775,11 @@ function clone_service(Service $source, StandaloneDocker|SwarmDocker $destinatio
 
     $sourceServer = $source->destination?->server;
     $targetServer = $newService->destination?->server;
-    if ($pendingVolumeClones !== [] && $sourceServer && $targetServer) {
-        $chain = [
-            function () use ($source) {
-                StopService::run($source);
-            },
-            ...array_map(
-                fn (array $clone) => new VolumeCloneJob($clone[0], $clone[1]->name, $sourceServer, $targetServer, $clone[1]),
-                $pendingVolumeClones,
-            ),
-            function () use ($source) {
-                StartService::run($source);
-            },
-        ];
-
-        // The jobs load the new volume rows, so they must not run before the transaction of the caller commits.
-        DB::afterCommit(fn () => Bus::chain($chain)->onQueue('high')->dispatch());
+    if ($sourceServer && $targetServer) {
+        CloneVolumeDataWithSourceStopped::run($source, array_map(
+            fn (array $clone) => new VolumeCloneJob($clone[0], $clone[1]->name, $sourceServer, $targetServer, $clone[1]),
+            $pendingVolumeClones,
+        ));
     }
 
     return $newService;

bootstrap/helpers/subscriptions.php

@@ -122,10 +122,13 @@ function allowedPathsForTwoFactorSetup(): array
 {
     return [
         'two-factor-required',
+        'two-factor-required/switch-team',
         'two-factor-challenge',
         'logout',
         'user/confirm-password',
         'user/confirmed-password-status',
+        'passkeys/confirm',
+        'passkeys/confirm/options',
         'user/two-factor-authentication',
         'user/confirmed-two-factor-authentication',
         'user/two-factor-qr-code',

docker-compose.prod.yml

@@ -3,8 +3,9 @@ services:
   # Realtime runs inside the coolify container now. Without this entry, such an override is an
   # incomplete service and every upgrade fails. Zero replicas means it never starts, and
   # "docker compose up" removes the old coolify-realtime container (an inactive profile keeps it).
+  # It uses the coolify image, so install and upgrade do not pull another image.
   soketi:
-    image: busybox:1.37
+    image: "${REGISTRY_URL:-docker.io}/coollabsio/coolify:${LATEST_IMAGE:-latest}"
     deploy:
       replicas: 0
   coolify:

openapi.json

@@ -5395,7 +5395,7 @@
                     "Applications"
                 ],
                 "summary": "Update Storage",
-                "description": "Update a persistent storage or file storage by application UUID.",
+                "description": "Update a persistent storage or file storage by application UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.",
                 "operationId": "update-storage-by-application-uuid",
                 "parameters": [
                     {
@@ -10286,7 +10286,7 @@
                     "Databases"
                 ],
                 "summary": "Update Storage",
-                "description": "Update a persistent storage or file storage by database UUID.",
+                "description": "Update a persistent storage or file storage by database UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.",
                 "operationId": "update-storage-by-database-uuid",
                 "parameters": [
                     {
@@ -17932,7 +17932,7 @@
                     "Servers"
                 ],
                 "summary": "Migrate server to another Coolify instance",
-                "description": "Start a transfer in the background: export this server, import and claim it on the target instance (with the given token), then disable automations here. Follow the progress with GET \/servers\/{uuid}\/transfer. The target URL must use https. Requires read:sensitive and write.",
+                "description": "Start a transfer in the background: export this server, import and claim it on the target instance (with the given token), then disable automations here. All secrets of the server, also locked environment variables, are sent to the target. Follow the progress with GET \/servers\/{uuid}\/transfer. The target URL must use https. Requires read:sensitive and write, and the owner role of the server team.",
                 "operationId": "migrate-server-between-instances",
                 "parameters": [
                     {
@@ -17973,7 +17973,7 @@
                         "description": "Transfer started"
                     },
                     "403": {
-                        "description": "Missing sensitive permission"
+                        "description": "Missing sensitive permission or not an owner of the server team"
                     },
                     "404": {
                         "$ref": "#\/components\/responses\/404"
@@ -21735,7 +21735,7 @@
                     "Services"
                 ],
                 "summary": "Update Storage",
-                "description": "Update a persistent storage or file storage by service UUID.",
+                "description": "Update a persistent storage or file storage by service UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.",
                 "operationId": "update-storage-by-service-uuid",
                 "parameters": [
                     {

openapi.yaml

@@ -3545,7 +3545,7 @@ paths:
       tags:
         - Applications
       summary: 'Update Storage'
-      description: 'Update a persistent storage or file storage by application UUID.'
+      description: 'Update a persistent storage or file storage by application UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.'
       operationId: update-storage-by-application-uuid
       parameters:
         -
@@ -6738,7 +6738,7 @@ paths:
       tags:
         - Databases
       summary: 'Update Storage'
-      description: 'Update a persistent storage or file storage by database UUID.'
+      description: 'Update a persistent storage or file storage by database UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.'
       operationId: update-storage-by-database-uuid
       parameters:
         -
@@ -11335,7 +11335,7 @@ paths:
       tags:
         - Servers
       summary: 'Migrate server to another Coolify instance'
-      description: 'Start a transfer in the background: export this server, import and claim it on the target instance (with the given token), then disable automations here. Follow the progress with GET /servers/{uuid}/transfer. The target URL must use https. Requires read:sensitive and write.'
+      description: 'Start a transfer in the background: export this server, import and claim it on the target instance (with the given token), then disable automations here. All secrets of the server, also locked environment variables, are sent to the target. Follow the progress with GET /servers/{uuid}/transfer. The target URL must use https. Requires read:sensitive and write, and the owner role of the server team.'
       operationId: migrate-server-between-instances
       parameters:
         -
@@ -11364,7 +11364,7 @@ paths:
         '202':
           description: 'Transfer started'
         '403':
-          description: 'Missing sensitive permission'
+          description: 'Missing sensitive permission or not an owner of the server team'
         '404':
           $ref: '#/components/responses/404'
         '422':
@@ -13712,7 +13712,7 @@ paths:
       tags:
         - Services
       summary: 'Update Storage'
-      description: 'Update a persistent storage or file storage by service UUID.'
+      description: 'Update a persistent storage or file storage by service UUID. Changed file content is saved right away and written to the server in the background, so a failed write does not change the response.'
       operationId: update-storage-by-service-uuid
       parameters:
         -

other/nightly/docker-compose.prod.yml

@@ -3,8 +3,9 @@ services:
   # Realtime runs inside the coolify container now. Without this entry, such an override is an
   # incomplete service and every upgrade fails. Zero replicas means it never starts, and
   # "docker compose up" removes the old coolify-realtime container (an inactive profile keeps it).
+  # It uses the coolify image, so install and upgrade do not pull another image.
   soketi:
-    image: busybox:1.37
+    image: "${REGISTRY_URL:-docker.io}/coollabsio/coolify:${LATEST_IMAGE:-latest}"
     deploy:
       replicas: 0
   coolify:

other/nightly/install.sh

@@ -1028,8 +1028,8 @@ if [ "$IS_COOLIFY_VOLUME_EXISTS" -eq 0 ]; then
     test -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub && rm -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub
     ssh-keygen -t ed25519 -a 100 -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal -q -N "" -C coolify
     chown 9999 /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal
-    # Remove only the key from a previous install (generated with the comment "coolify"), not other keys that contain "coolify"
-    sed -i -E "/[[:space:]]coolify[[:space:]]*$/d" ~/.ssh/authorized_keys
+    # Remove only the key from a previous install (comment "coolify", or "root@coolify" before September 2024), not other keys that contain "coolify"
+    sed -i -E "/[[:space:]](root@)?coolify[[:space:]]*$/d" ~/.ssh/authorized_keys
     cat /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub >>~/.ssh/authorized_keys
     rm -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub
 fi

resources/views/components/live-toggle.blade.php

@@ -1,11 +1,14 @@
 {{--
     Live refresh toggle. Entangles the `live` bool of the host Livewire component, so it works
     with any time range. The button shows "Live" with a pulsing dot when refresh is on and
-    "Paused" with a play icon when it is off. `storageKey` keeps the choice per browser.
+    "Paused" with a play icon when it is off. `storageKey` keeps the last choice the user clicked per
+    browser; changes made by the component (for example a range that starts paused) are not stored.
+    A stored "off" always applies on page load; a stored "on" applies only when `restoreLive` is true.
 --}}
 @props([
     'active' => false,
     'storageKey' => null,
+    'restoreLive' => true,
     'title' => 'Turn live refresh on or off',
 ])
 
@@ -13,10 +16,9 @@
     @if ($storageKey) x-init="
         const stored = localStorage.getItem(@js($storageKey));
         if (stored === '0') { live = false; }
-        if (stored === '1') { live = true; }
-        $watch('live', value => localStorage.setItem(@js($storageKey), value ? '1' : '0'));
+        if (stored === '1' && @js((bool) $restoreLive)) { live = true; }
     " @endif>
-    <button type="button" @click="live = !live" :aria-pressed="live ? 'true' : 'false'"
+    <button type="button" @click="live = !live; @if ($storageKey) localStorage.setItem(@js($storageKey), live ? '1' : '0'); @endif" :aria-pressed="live ? 'true' : 'false'"
         aria-pressed="{{ $active ? 'true' : 'false' }}" title="{{ $title }}"
         class="inline-flex h-7 items-center gap-1.5 rounded-md px-2.5 text-[12px] font-medium ring-1 ring-neutral-200 transition-colors dark:ring-white/[0.08]"
         :class="live

resources/views/livewire/fleet/metrics.blade.php

@@ -52,14 +52,14 @@ $serverListboxOptions = array_merge(
                             <x-loading compact aria-label="Loading metrics" />
                         </div>
                     </div>
-                    <x-live-toggle :active="$live" storageKey="metrics-live" title="Turn live refresh on or off (updates every 60 seconds)" />
+                    <x-live-toggle :active="$live" storageKey="metrics-live" :restoreLive="$range === '24h'" title="Turn live refresh on or off (updates every 60 seconds)" />
                 </div>
             </div>
         @endif
     </div>
 
-    @if ($live)
-        <div wire:poll.60s="loadData" class="hidden"></div>
+    @if ($this->pollSeconds())
+        <div wire:poll.{{ $this->pollSeconds() }}s="pollData" class="hidden"></div>
     @endif
 
     @if ($servers->isEmpty())

resources/views/livewire/project/application/previews.blade.php

@@ -86,7 +86,7 @@
 
             <div class="w-full">
                 <x-forms.listbox id="isPrDeploymentsPublicEnabled" label="PR deployment access" onChange="savePreviewSettings"
-                    helper="When public, anyone can trigger PR deployments. Otherwise only PRs from branches in the same repository are deployed and fork PRs are blocked.<br><br><span class='font-bold'>Secrets:</span> the code of a fork PR can read all environment variables of its preview. When public, previews do not get preview variables that still have the production value. Give previews their own values, and do not use production secrets in them."
+                    helper="When public, anyone can trigger PR deployments. Otherwise only PRs from branches in the same repository are deployed and fork PRs are blocked.<br><br><span class='font-bold'>Secrets:</span> the code of a fork PR can read all environment variables of its preview. When public, previews do not get preview variables that still have the production value, and shared variables are resolved only in preview variables, not in the compose file of the PR. This is not full isolation: the preview runs on the same server as production, and with the Docker Compose build pack Coolify deploys the compose file of the PR, which can, for example, mount host paths. Give previews their own values, and do not use production secrets in them."
                     :options="[
                         ['value' => false, 'label' => 'Repository members only'],
                         ['value' => true, 'label' => 'Public (fork PRs allowed)'],

resources/views/livewire/project/clone-me.blade.php

@@ -20,6 +20,10 @@
                 <div class="max-w-md">
                     <x-forms.input required id="newName" label="New name" />
                 </div>
+                <div class="mt-4 max-w-md">
+                    <x-forms.checkbox id="cloneVolumeData" live label="Clone volume data"
+                        helper="Copies the data of the persistent volumes to the clones. Running source resources are stopped while the data is copied and started again afterwards." />
+                </div>
             </div>
         </section>
 
@@ -138,14 +142,35 @@
                 </div>
                 <div
                     class="flex flex-col gap-2 border-t border-neutral-200 p-4 sm:flex-row sm:justify-end dark:border-white/[0.06]">
-                    <x-forms.button isHighlighted wire:click="clone('environment')"
-                        :disabled="! filled($selectedDestination)">
-                        Clone to environment
-                    </x-forms.button>
-                    <x-forms.button isHighlighted wire:click="clone('project')"
-                        :disabled="! filled($selectedDestination)">
-                        Clone to project
-                    </x-forms.button>
+                    @if ($cloneVolumeData)
+                        @foreach (['environment' => 'Clone to environment', 'project' => 'Clone to project'] as $cloneType => $cloneLabel)
+                            <div wire:key="clone-with-volume-data-{{ $cloneType }}">
+                                <x-modal-confirmation title="Clone with volume data?" submitAction="clone('{{ $cloneType }}')"
+                                    :ignoreWire="false" :confirmWithText="false" :confirmWithPassword="false"
+                                    :step2ButtonText="$cloneLabel"
+                                    warningMessage="Running source resources are unavailable while their volume data is copied."
+                                    :actions="[
+                                        'Running source resources are stopped while their volume data is copied.',
+                                        'They are started again after the copy, also when the copy fails.',
+                                    ]">
+                                    <x-slot:trigger>
+                                        <x-forms.button isHighlighted :disabled="! filled($selectedDestination)">
+                                            {{ $cloneLabel }}
+                                        </x-forms.button>
+                                    </x-slot:trigger>
+                                </x-modal-confirmation>
+                            </div>
+                        @endforeach
+                    @else
+                        <x-forms.button isHighlighted wire:click="clone('environment')"
+                            :disabled="! filled($selectedDestination)">
+                            Clone to environment
+                        </x-forms.button>
+                        <x-forms.button isHighlighted wire:click="clone('project')"
+                            :disabled="! filled($selectedDestination)">
+                            Clone to project
+                        </x-forms.button>
+                    @endif
                 </div>
             </div>
         </section>

resources/views/livewire/project/shared/metrics.blade.php

@@ -38,8 +38,8 @@
                 icon-name="dashboard" />
         </x-application.settings-section>
     @else
-        @if ($live)
-            <div wire:key="metrics-poll-{{ $this->pollSeconds() }}" wire:poll.{{ $this->pollSeconds() }}s="loadData" class="hidden"></div>
+        @if ($this->pollSeconds())
+            <div wire:key="metrics-poll-{{ $this->pollSeconds() }}" wire:poll.{{ $this->pollSeconds() }}s="pollData" class="hidden"></div>
         @endif
         <x-application.settings-section id="metrics-overview-section" title="Metrics"
             helper="Inspect recent CPU and memory usage reported by Sentinel.">
@@ -58,7 +58,7 @@
                 ]" />
             </div>
             <p class="mt-3 text-xs leading-5 text-neutral-500 dark:text-fg-dim">
-                While live, ranges up to one hour refresh every five seconds. Longer ranges refresh every minute.
+                Ranges up to one hour refresh live every five seconds. Longer ranges load once; turn on live refresh to update them every minute.
             </p>
         </x-application.settings-section>
 

resources/views/livewire/project/shared/resource-operations.blade.php

@@ -1,4 +1,5 @@
-<div x-on:migrate-resource-confirmed.window="$wire.migrateTo(selectedMigrateDestination)" x-data="{
+<div x-on:migrate-resource-confirmed.window="$wire.migrateTo(selectedMigrateDestination)"
+    x-on:clone-resource-confirmed.window="$wire.cloneTo(selectedCloneDestination, selectedCloneEnvironment)" x-data="{
     selectedCloneServer: null,
     selectedCloneDestination: null,
     selectedCloneProject: null,
@@ -153,9 +154,9 @@
     @can('update', $resource)
         <x-application.settings-section id="clone-resource-section" title="Clone resource"
             helper="Create a copy of this resource. Choose where it runs and where it belongs. The current values are selected by default.">
-            <x-callout type="info" title="Configuration only">
+            <x-callout type="info" title="What is copied">
                 Cloning copies settings, environment variables, and resource configuration. Stored files,
-                database records, and other persistent data are not copied.
+                database records, and other persistent data are copied only when you select Clone volume data.
             </x-callout>
 
             <div class="mt-4 grid gap-4 md:grid-cols-2">
@@ -179,15 +180,41 @@
                     placeholder="Choose an environment…" />
             </div>
 
+            <div class="mt-4">
+                <x-forms.checkbox id="cloneVolumeData" live label="Clone volume data"
+                    helper="Copies the data of the persistent volumes to the clone. A running source resource is stopped while the data is copied and started again afterwards." />
+            </div>
+
             <div
                 class="mt-4 flex flex-col gap-3 border-t border-neutral-200 pt-4 sm:flex-row sm:items-center sm:justify-between dark:border-white/[0.07]">
-                <p class="text-[13px] text-neutral-500 dark:text-fg-dim">
-                    The running resource will not be changed.
-                </p>
-                <x-forms.button x-bind:disabled="!selectedCloneDestination || !selectedCloneEnvironment"
-                    @click="$wire.cloneTo(selectedCloneDestination, selectedCloneEnvironment)">
-                    Clone resource
-                </x-forms.button>
+                @if ($cloneVolumeData)
+                    <p class="text-[13px] text-neutral-500 dark:text-fg-dim">
+                        A running resource is stopped while its volume data is copied.
+                    </p>
+                    <x-modal-confirmation title="Clone this resource with its volume data?" submitAction="clone-resource-confirmed"
+                        dispatchAction :confirmWithText="false" :confirmWithPassword="false"
+                        step2ButtonText="Clone resource"
+                        warningMessage="A running resource is unavailable while its volume data is copied."
+                        :actions="[
+                            'If the resource is running, it is stopped while its volume data is copied.',
+                            'The resource is started again after the copy, also when the copy fails.',
+                        ]">
+                        <x-slot:trigger>
+                            <x-forms.button x-bind:disabled="!selectedCloneDestination || !selectedCloneEnvironment">
+                                Clone resource
+                            </x-forms.button>
+                        </x-slot:trigger>
+                    </x-modal-confirmation>
+                @else
+                    <p class="text-[13px] text-neutral-500 dark:text-fg-dim">
+                        The running resource will not be changed.
+                    </p>
+                    <x-forms.button wire:key="clone-resource-without-volume-data"
+                        x-bind:disabled="!selectedCloneDestination || !selectedCloneEnvironment"
+                        @click="$wire.cloneTo(selectedCloneDestination, selectedCloneEnvironment)">
+                        Clone resource
+                    </x-forms.button>
+                @endif
             </div>
         </x-application.settings-section>
 

resources/views/livewire/server/charts.blade.php

@@ -12,8 +12,8 @@
         <div class="application-settings-form flex w-full flex-col gap-6"
             @if ($server->isMetricsEnabled()) x-init="$wire.loadData()" @endif>
             @if ($server->isMetricsEnabled())
-                @if ($live)
-                    <div wire:key="metrics-poll-{{ $this->pollSeconds() }}" wire:poll.{{ $this->pollSeconds() }}s="loadData" class="hidden"></div>
+                @if ($this->pollSeconds())
+                    <div wire:key="metrics-poll-{{ $this->pollSeconds() }}" wire:poll.{{ $this->pollSeconds() }}s="pollData" class="hidden"></div>
                 @endif
                 <form wire:submit.prevent="saveMetricsSettings" class="contents">
                     <x-unsaved-bar action="saveMetricsSettings"
@@ -54,7 +54,7 @@
                             ]" />
                         </div>
                         <p class="mt-3 text-xs leading-5 text-neutral-500 dark:text-fg-dim">
-                            While live, ranges up to one hour refresh every five seconds. Longer ranges refresh every minute.
+                            Ranges up to one hour refresh live every five seconds. Longer ranges load once; turn on live refresh to update them every minute.
                         </p>
                     </x-application.settings-section>
                 </form>

resources/views/livewire/two-factor-required.blade.php

@@ -46,6 +46,18 @@
                 <x-forms.button class="w-full justify-center" type="submit" isHighlighted>Configure 2FA</x-forms.button>
             </form>
         @endif
+        @if (!request()->user()->two_factor_confirmed_at && $teamsWithoutTwoFactor->isNotEmpty())
+            <div class="flex flex-col gap-2 border-t border-neutral-200 pt-4 dark:border-white/[0.07]">
+                <p class="text-sm text-neutral-500 dark:text-fg-dim">Or switch to a team that does not require 2FA:</p>
+                @foreach ($teamsWithoutTwoFactor as $teamWithoutTwoFactor)
+                    <form action="{{ route('two-factor.required.switch-team') }}" method="POST">
+                        @csrf
+                        <input type="hidden" name="team_id" value="{{ $teamWithoutTwoFactor->id }}">
+                        <x-forms.button class="w-full justify-center" type="submit">{{ $teamWithoutTwoFactor->name }}</x-forms.button>
+                    </form>
+                @endforeach
+            </div>
+        @endif
     </div>
     @unless (request()->user()->two_factor_confirmed_at)
         <x-slot:footer>

routes/web.php

@@ -193,6 +193,19 @@ Route::middleware(['auth', 'verified'])->group(function () {
         Route::get('/force-password-reset', ForcePasswordReset::class)->name('auth.force-password-reset');
     });
     Route::get('/two-factor-required', TwoFactorRequired::class)->name('two-factor.required');
+    // A member without 2FA can leave a team that requires it, but only for a team that does not.
+    Route::post('/two-factor-required/switch-team', function (Request $request) {
+        $team = $request->user()->teams()
+            ->whereKey($request->integer('team_id'))
+            ->where('is_two_factor_required', false)
+            ->first();
+        if (! $team) {
+            return redirect()->route('two-factor.required');
+        }
+        refreshSession($team);
+
+        return redirect()->route('dashboard');
+    })->name('two-factor.required.switch-team');
 
     Route::get('/', Dashboard::class)->name('dashboard');
     Route::get('/dashboard/canvas', DashboardCanvas::class)->name('dashboard.canvas');

scripts/install.sh

@@ -1028,8 +1028,8 @@ if [ "$IS_COOLIFY_VOLUME_EXISTS" -eq 0 ]; then
     test -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub && rm -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub
     ssh-keygen -t ed25519 -a 100 -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal -q -N "" -C coolify
     chown 9999 /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal
-    # Remove only the key from a previous install (generated with the comment "coolify"), not other keys that contain "coolify"
-    sed -i -E "/[[:space:]]coolify[[:space:]]*$/d" ~/.ssh/authorized_keys
+    # Remove only the key from a previous install (comment "coolify", or "root@coolify" before September 2024), not other keys that contain "coolify"
+    sed -i -E "/[[:space:]](root@)?coolify[[:space:]]*$/d" ~/.ssh/authorized_keys
     cat /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub >>~/.ssh/authorized_keys
     rm -f /data/coolify/ssh/keys/id.$CURRENT_USER@host.docker.internal.pub
 fi

tests/Feature/Api/EnvironmentVariableUpdateApiTest.php

@@ -8,6 +8,7 @@ use App\Models\Project;
 use App\Models\Server;
 use App\Models\Service;
 use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -415,3 +416,69 @@ describe('environment variable key validation for app and service APIs', functio
             ->assertStatus(422);
     });
 });
+
+describe('env PATCH keeps the value when value is omitted', function () {
+    beforeEach(function () {
+        $this->headers = [
+            'Authorization' => 'Bearer '.$this->bearerToken,
+            'Content-Type' => 'application/json',
+        ];
+    });
+
+    test('keeps or updates the value', function (string $resourceType, string $endpoint, bool $bulk) {
+        $resource = match ($resourceType) {
+            'application' => Application::factory()->create([
+                'environment_id' => $this->environment->id,
+                'destination_id' => $this->destination->id,
+                'destination_type' => $this->destination->getMorphClass(),
+            ]),
+            'service' => Service::factory()->create([
+                'server_id' => $this->server->id,
+                'destination_id' => $this->destination->id,
+                'destination_type' => $this->destination->getMorphClass(),
+                'environment_id' => $this->environment->id,
+            ]),
+            'database' => StandalonePostgresql::create([
+                'name' => 'db',
+                'postgres_user' => 'postgres',
+                'postgres_password' => 'password',
+                'postgres_db' => 'app',
+                'image' => 'postgres:16',
+                'environment_id' => $this->environment->id,
+                'destination_id' => $this->destination->id,
+                'destination_type' => $this->destination->getMorphClass(),
+            ]),
+        };
+
+        $env = EnvironmentVariable::create([
+            'key' => 'KEEP_ME',
+            'value' => 'secret-value',
+            'resourceable_type' => $resource->getMorphClass(),
+            'resourceable_id' => $resource->id,
+            'is_preview' => false,
+            'is_literal' => false,
+        ]);
+
+        $url = "/api/v1/{$endpoint}/{$resource->uuid}/envs".($bulk ? '/bulk' : '');
+        $send = fn (array $payload) => $this->withHeaders($this->headers)
+            ->patchJson($url, $bulk ? ['data' => [$payload]] : $payload)
+            ->assertStatus(201);
+
+        $send(['key' => 'KEEP_ME', 'is_literal' => true]);
+
+        $env->refresh();
+        expect($env->value)->toBe('secret-value')
+            ->and((bool) $env->is_literal)->toBeTrue();
+
+        $send(['key' => 'KEEP_ME', 'value' => 'new-value']);
+
+        expect($env->fresh()->value)->toBe('new-value');
+    })->with([
+        'application single' => ['application', 'applications', false],
+        'application bulk' => ['application', 'applications', true],
+        'service single' => ['service', 'services', false],
+        'service bulk' => ['service', 'services', true],
+        'database single' => ['database', 'databases', false],
+        'database bulk' => ['database', 'databases', true],
+    ]);
+});

tests/Feature/Api/ServerDeployAbilityTest.php

@@ -210,7 +210,7 @@ describe('destination create', function () {
             ->assertJsonPath('errors.network.0', 'A Docker network with this name already exists on the server. Use a new network name.');
 
         expect(StandaloneDocker::query()->count())->toBe($countBefore);
-        Process::assertRan(fn ($process) => str_contains($process->command, 'docker network ls'));
+        Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'docker network ls'));
     });
 
     test('rejects the destination when the server networks cannot be read', function () {

tests/Feature/Api/ServerDockerRegistriesApiTest.php

@@ -75,7 +75,7 @@ function serverRegistriesApiHeaders(string $bearerToken): array
 function fakeServerRegistriesSsh(array &$commands, string $config = '', string $error = ''): void
 {
     Process::fake(function ($process) use (&$commands, $config, $error) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands[] = $command;
 
         if (str_contains($command, '.docker/config.json')) {

tests/Feature/Api/UiSettingsApiParityTest.php

@@ -271,6 +271,9 @@ describe('servers', function () {
     })->with([
         'wildcard domain' => ['wildcard_domain', 'not a url'],
         'timezone' => ['server_timezone', 'Mars/Olympus'],
+        'null timezone' => ['server_timezone', null],
+        'empty timezone' => ['server_timezone', ''],
+        'array timezone' => ['server_timezone', ['Europe/Berlin']],
         'compression' => ['backup_compression_cpu_percentage', 30],
     ]);
 

tests/Feature/ApiRateLimitClientIpTest.php

@@ -15,11 +15,11 @@ beforeEach(function () {
     InstanceSettings::forceCreate(['id' => 0]);
 });
 
-function pushSentinelFrom(string $clientIp): TestResponse
+function pushSentinelFrom(string $clientIp, string $peer = '172.70.0.1'): TestResponse
 {
     return test()
         ->withHeader('CF-Connecting-IP', $clientIp)
-        ->withServerVariables(['REMOTE_ADDR' => '172.70.0.1'])
+        ->withServerVariables(['REMOTE_ADDR' => $peer])
         ->postJson('/api/v1/sentinel/push');
 }
 
@@ -33,13 +33,13 @@ it('gives each Cloudflare client ip its own sentinel push bucket on cloud', func
     pushSentinelFrom('203.0.113.20')->assertUnauthorized();
 });
 
-it('ignores the Cloudflare client ip for sentinel pushes on self-hosted instances', function () {
+it('ignores the Cloudflare client ip for sentinel pushes on self-hosted instances when the peer is not Cloudflare', function () {
     config()->set('constants.coolify.self_hosted', true);
 
-    pushSentinelFrom('203.0.113.10')->assertUnauthorized();
-    pushSentinelFrom('203.0.113.10')->assertUnauthorized();
+    pushSentinelFrom('203.0.113.10', peer: '198.51.100.7')->assertUnauthorized();
+    pushSentinelFrom('203.0.113.10', peer: '198.51.100.7')->assertUnauthorized();
 
-    pushSentinelFrom('203.0.113.20')->assertStatus(429);
+    pushSentinelFrom('203.0.113.20', peer: '198.51.100.7')->assertStatus(429);
 });
 
 it('keys unauthenticated limiters by the Cloudflare client ip on cloud', function (string $limiter) {

tests/Feature/ApplicationComposeInjectionValidationTest.php

@@ -75,7 +75,7 @@ function composeInjectionPayloads(): array
 function fakeRepositoryCompose(string $compose): void
 {
     Process::fake(function ($process) use ($compose) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         if (str_contains($command, 'git cat-file blob')) {
             return Process::result(output: $compose);
         }
@@ -310,7 +310,7 @@ test('a deployment stops with a visible log line when the repository Compose fil
         ->not->toContain('#0 ')
         ->not->toContain('.php');
     expect($this->application->refresh()->docker_compose_raw)->toBe(SAFE_APPLICATION_COMPOSE);
-    Process::assertNotRan(fn ($process) => str_contains(is_array($process->command) ? implode(' ', $process->command) : $process->command, 'base64 -d'));
+    Process::assertNotRan(fn ($process) => str_contains(is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process), 'base64 -d'));
 })->with(['production' => 0, 'pull request preview' => 42]);
 
 test('the deployment log shows the unsafe Compose file explanation once and a short failure line', function () {

tests/Feature/ApplicationDeploymentFailedKeepsHealthyContainerTest.php

@@ -67,8 +67,8 @@ function failQueuedCopy(string $queuedCopy): void
 
 function removesContainer(string $containerName): Closure
 {
-    return fn (PendingProcess $process) => str_contains((string) $process->command, 'docker rm -f')
-        && str_contains((string) $process->command, $containerName);
+    return fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'docker rm -f')
+        && str_contains(remoteProcessScript($process), $containerName);
 }
 
 it('keeps the new container when a deployment fails after the new container became healthy', function () {

tests/Feature/ApplicationPreviewApiTest.php

@@ -142,17 +142,17 @@ describe('GET /api/v1/applications/{uuid}/previews/{pull_request_id}/logs', func
         $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
         $this->server->update(['private_key_id' => $privateKey->id]);
         Process::fake(function ($process) {
-            if (str_contains($process->command, 'docker ps -a')) {
+            if (str_contains(remoteProcessScript($process), 'docker ps -a')) {
                 return Process::result(output: json_encode([
                     'ID' => 'preview-container',
                     'Names' => "{$this->application->uuid}-pr-42",
                     'Labels' => "coolify.applicationId={$this->application->id},coolify.pullRequestId=42",
                 ]));
             }
-            if (str_contains($process->command, 'docker inspect')) {
+            if (str_contains(remoteProcessScript($process), 'docker inspect')) {
                 return Process::result(output: json_encode(['State' => ['Status' => 'running']]));
             }
-            if (str_contains($process->command, 'docker logs')) {
+            if (str_contains(remoteProcessScript($process), 'docker logs')) {
                 return Process::result(output: 'preview runtime log');
             }
 
@@ -455,7 +455,7 @@ describe('GET /api/v1/applications/{uuid}/logs service_name', function () {
         $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
         $this->server->update(['private_key_id' => $privateKey->id]);
         Process::fake(function ($process) {
-            if (str_contains($process->command, 'docker ps -a')) {
+            if (str_contains(remoteProcessScript($process), 'docker ps -a')) {
                 return Process::result(output: implode(PHP_EOL, [
                     json_encode([
                         'ID' => 'web-container',
@@ -469,11 +469,11 @@ describe('GET /api/v1/applications/{uuid}/logs service_name', function () {
                     ]),
                 ]));
             }
-            if (str_contains($process->command, 'docker inspect')) {
+            if (str_contains(remoteProcessScript($process), 'docker inspect')) {
                 return Process::result(output: json_encode(['State' => ['Status' => 'running']]));
             }
-            if (str_contains($process->command, 'docker logs')) {
-                return Process::result(output: str_contains($process->command, 'worker-container') ? 'worker log' : 'web log');
+            if (str_contains(remoteProcessScript($process), 'docker logs')) {
+                return Process::result(output: str_contains(remoteProcessScript($process), 'worker-container') ? 'worker log' : 'web log');
             }
 
             return Process::result();

tests/Feature/ApplicationPreviewComposeCleanupTest.php

@@ -91,7 +91,7 @@ beforeEach(function () {
 function fakePreviewCleanupServer(array &$commands, string $previewNetwork, ?array $previewNetworkContainers = [], string $dockerPs = ''): void
 {
     Process::fake(function ($process) use (&$commands, $previewNetwork, $previewNetworkContainers, $dockerPs) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands[] = $command;
 
         if (str_contains($command, 'docker network inspect') && str_contains($command, "'{$previewNetwork}'")) {

tests/Feature/ApplicationPreviewVolumeCleanupTest.php

@@ -67,11 +67,11 @@ it('deletes only named volumes that have the preview suffix enabled', function (
 
     $this->preview->forceDelete();
 
-    Process::assertRanTimes(fn ($process) => str_contains($process->command, 'docker volume rm'), 1);
-    Process::assertRan(fn ($process) => str_contains($process->command, "docker volume rm -f 'app-data-pr-42'"));
-    Process::assertRan(fn ($process) => str_contains($process->command, "sudo docker volume rm -f 'app-data-pr-42'"));
-    Process::assertNotRan(fn ($process) => str_contains($process->command, 'shared-cache'));
-    Process::assertNotRan(fn ($process) => str_contains($process->command, 'seed-data'));
+    Process::assertRanTimes(fn ($process) => str_contains(remoteProcessScript($process), 'docker volume rm'), 1);
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker volume rm -f 'app-data-pr-42'"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "sudo docker volume rm -f 'app-data-pr-42'"));
+    Process::assertNotRan(fn ($process) => str_contains(remoteProcessScript($process), 'shared-cache'));
+    Process::assertNotRan(fn ($process) => str_contains(remoteProcessScript($process), 'seed-data'));
 });
 
 it('reports Docker volume removal failures and keeps the preview record', function () {
@@ -103,7 +103,7 @@ it('continues removing preview volumes when an earlier volume is already absent'
         'is_preview_suffix_enabled' => true,
     ]);
     Process::fake(function ($process) {
-        if (str_contains($process->command, "docker volume rm -f 'already-removed-pr-42'")) {
+        if (str_contains(remoteProcessScript($process), "docker volume rm -f 'already-removed-pr-42'")) {
             return Process::result(
                 errorOutput: 'Error response from daemon: volume already-removed-pr-42 not found',
                 exitCode: 1,
@@ -115,8 +115,8 @@ it('continues removing preview volumes when an earlier volume is already absent'
 
     $this->preview->forceDelete();
 
-    Process::assertRan(fn ($process) => str_contains($process->command, "docker volume rm -f 'already-removed-pr-42'"));
-    Process::assertRan(fn ($process) => str_contains($process->command, "docker volume rm -f 'app-data-pr-42'"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker volume rm -f 'already-removed-pr-42'"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker volume rm -f 'app-data-pr-42'"));
     expect(ApplicationPreview::find($this->preview->id))->toBeNull();
 });
 
@@ -156,7 +156,7 @@ YAML,
 
     $this->preview->forceDelete();
 
-    Process::assertRan(fn ($process) => str_contains($process->command, "docker volume rm -f '{$uuid}_db-pr-42'"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker volume rm -f '{$uuid}_db-pr-42'"));
     expect($this->application->persistentStorages()->pluck('name')->sort()->values()->all())->toBe([
         "{$uuid}_db",
         "{$uuid}_db-pr-43",

tests/Feature/BackupAndNotificationRedactionTest.php

@@ -99,7 +99,7 @@ it('redacts the database password from the stored execution and the failure noti
     $backup = redactionMysqlBackup($this->team, $this->environment, $this->destination);
     $password = $backup->database->mysql_root_password;
     Process::fake(function ($process) use ($password) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         if (str_contains($command, 'mysqldump')) {
             return Process::result(errorOutput: "mysqldump: Got error: 1045: Access denied for user 'root' with password {$password}", exitCode: 1);
         }

tests/Feature/BackupDirectoryPermissionsTest.php

@@ -52,9 +52,9 @@ function backupDirectoryDumpScript(string $sshUser): string
     ]);
     $commands = new Collection;
     Process::fake(function ($process) use ($commands) {
-        $commands->push($process->command);
+        $commands->push(remoteProcessScript($process));
 
-        return Process::result(output: str_contains($process->command, 'du -b') ? '100' : '');
+        return Process::result(output: str_contains(remoteProcessScript($process), 'du -b') ? '100' : '');
     });
 
     (new DatabaseBackupJob($backup))->handle();

tests/Feature/BackupFailureExitCodeTest.php

@@ -57,7 +57,7 @@ beforeEach(function () {
 function interruptedMongodump(int $exitCode): void
 {
     Process::fake(function ($process) use ($exitCode) {
-        if (str_contains($process->command, 'mongodump')) {
+        if (str_contains(remoteProcessScript($process), 'mongodump')) {
             return Process::result(
                 errorOutput: "2026-08-29T15:44:00.555+0000\tdone dumping `evolution.leaverequests` (0 documents)\n2026-08-29T15:44:00.555+0000\twriting `evolution.printagents` to `archive on stdout`",
                 exitCode: $exitCode,
@@ -86,7 +86,7 @@ it('tells why a dump stopped without an error message', function (int $exitCode,
 
 it('keeps the error of a dump that reports its own failure', function () {
     Process::fake(function ($process) {
-        if (str_contains($process->command, 'mongodump')) {
+        if (str_contains(remoteProcessScript($process), 'mongodump')) {
             return Process::result(errorOutput: 'Failed: error connecting to db server: server returned error on SASL authentication step', exitCode: 1);
         }
 

tests/Feature/BackupRunSafetyTest.php

@@ -65,7 +65,7 @@ function backupRunSafetyFakeRemoteCommands(array $outputs = []): Collection
 {
     $commands = collect();
     Process::fake(function ($process) use ($commands, $outputs) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands->push($command);
         foreach ($outputs as $needle => $output) {
             if (str_contains($command, $needle)) {
@@ -403,7 +403,7 @@ it('fails the running backup execution and notifies the team when the worker tim
     $team = $this->team;
     $atTimeout = null;
     Process::fake(function ($process) use ($queuedCopy, $team, &$atTimeout) {
-        if (str_contains($process->command, 'pg_dump')) {
+        if (str_contains(remoteProcessScript($process), 'pg_dump')) {
             $queuedCopy->failed(new TimeoutExceededException('App\Jobs\DatabaseBackupJob has timed out.'));
             $execution = ScheduledDatabaseBackupExecution::query()->sole();
             $atTimeout = [

tests/Feature/BackupS3CredentialsArgvTest.php

@@ -0,0 +1,209 @@
+<?php
+
+use App\Jobs\DatabaseBackupJob;
+use App\Jobs\VolumeBackupJob;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Filesystem\FilesystemAdapter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+use Symfony\Component\Process\Process as SymfonyProcess;
+
+uses(RefreshDatabase::class);
+
+const S3_ARGV_TEST_KEY = 'AK/+$x"y';
+const S3_ARGV_TEST_SECRET = "s3c/r+e\$t 'q\"uo\\te`d`";
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    config(['broadcasting.default' => 'null']);
+    defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+});
+
+afterEach(function () {
+    Server::flushIdentityMap();
+});
+
+/**
+ * @return array{0: Team, 1: Server, 2: S3Storage}
+ */
+function s3ArgvTestSetup(string $sshUser): array
+{
+    $team = Team::factory()->create();
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'user' => $sshUser,
+        'ip' => '203.0.113.10',
+        'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+    ]);
+    $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+    $storage = S3Storage::create([
+        'name' => 'Destination',
+        'region' => 'ru-central-1',
+        'key' => S3_ARGV_TEST_KEY,
+        'secret' => S3_ARGV_TEST_SECRET,
+        'bucket' => 'bucket',
+        'endpoint' => 'https://8.8.8.8',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $disk = Mockery::mock(FilesystemAdapter::class);
+    $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
+    $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+    $sshDisk = Storage::fake('ssh-keys');
+    Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+    Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
+
+    return [$team, $server, $storage];
+}
+
+/**
+ * Records the SSH processes that a backup job runs, with their stdin.
+ *
+ * @return Collection<int, array{command: string, input: ?string}>
+ */
+function s3ArgvTestFakeSsh(): Collection
+{
+    $processes = collect();
+    Process::fake(function ($process) use ($processes) {
+        $processes->push(['command' => $process->command, 'input' => $process->input]);
+        $output = match (true) {
+            str_contains(remoteProcessScript($process), 'du -b') => '128',
+            str_contains(remoteProcessScript($process), 'mc pipe') => '128',
+            default => '',
+        };
+
+        return Process::result(output: $output);
+    });
+
+    return $processes;
+}
+
+/**
+ * Runs the recorded `mc alias set` SSH process in a real shell with its stdin. Stubs for `timeout`, `ssh`
+ * (runs the remote command in `sh -c`, like a login shell), `sudo`, `docker` and `mc` log their arguments.
+ *
+ * @param  array{command: string, input: ?string}  $aliasProcess
+ * @return array{argv: string, mcStdin: string}
+ */
+function s3ArgvTestRunAliasProcess(array $aliasProcess): array
+{
+    $bin = sys_get_temp_dir().'/s3-argv-'.bin2hex(random_bytes(6));
+    mkdir($bin);
+    $log = "{$bin}/argv.log";
+    $stubs = [
+        'timeout' => "shift\nexec \"\$@\"",
+        'ssh' => "for last; do :; done\nexec sh -c \"\$last\"",
+        'sudo' => 'exec "$@"',
+        'docker' => "[ \"\$1\" = exec ] || exit 0\nshift\nwhile [ \"\${1#-}\" != \"\$1\" ]; do shift; done\nshift\nexec \"\$@\"",
+        'mc' => "cat > {$bin}/mc.stdin",
+    ];
+    foreach ($stubs as $name => $body) {
+        file_put_contents("{$bin}/{$name}", "#!/bin/sh\nprintf '%s\\n' \"{$name} \$*\" >> {$log}\n{$body}\n");
+        chmod("{$bin}/{$name}", 0755);
+    }
+
+    try {
+        (new SymfonyProcess(['bash', '-c', $aliasProcess['command']], env: ['PATH' => "{$bin}:/usr/bin:/bin"], input: $aliasProcess['input']))->mustRun();
+
+        return ['argv' => (string) file_get_contents($log), 'mcStdin' => (string) file_get_contents("{$bin}/mc.stdin")];
+    } finally {
+        array_map('unlink', glob("{$bin}/*"));
+        rmdir($bin);
+    }
+}
+
+function s3ArgvTestAssertSecretOnlyOnStdin(Collection $processes, bool $nonRoot): void
+{
+    foreach ($processes as $process) {
+        expect($process['command'])->not->toContain(S3_ARGV_TEST_SECRET)
+            ->and($process['command'])->not->toContain(escapeshellarg(S3_ARGV_TEST_SECRET))
+            ->and($process['command'])->not->toContain(escapeshellarg(S3_ARGV_TEST_KEY))
+            ->and($process['command'])->not->toContain(base64_encode(S3_ARGV_TEST_SECRET));
+    }
+
+    $aliasProcess = $processes->sole(fn (array $process) => str_contains($process['command'], 'mc alias set'));
+    $run = s3ArgvTestRunAliasProcess($aliasProcess);
+
+    expect($run['mcStdin'])->toBe(S3_ARGV_TEST_KEY."\n".S3_ARGV_TEST_SECRET."\n")
+        ->and($run['argv'])->toContain('mc alias set')
+        ->and($run['argv'])->toContain('temporary https://8.8.8.8')
+        ->and($run['argv'])->not->toContain(S3_ARGV_TEST_SECRET)
+        ->and($run['argv'])->not->toContain(S3_ARGV_TEST_KEY);
+    if ($nonRoot) {
+        expect($run['argv'])->toContain('sudo docker exec -i');
+    }
+    expect($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], "-e 'MC_REGION=ru-central-1'")))->toBeTrue();
+}
+
+it('passes the S3 keys of a database backup upload to mc on stdin, not in a command line', function (string $sshUser) {
+    [$team, $server, $storage] = s3ArgvTestSetup($sshUser);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+    $database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']);
+    $backup = new ScheduledDatabaseBackup;
+    $backup->forceFill([
+        'enabled' => true,
+        'save_s3' => true,
+        'disable_local_backup' => true,
+        's3_storage_id' => $storage->id,
+        'frequency' => 'daily',
+        'databases_to_backup' => 'app',
+        'database_id' => $database->id,
+        'database_type' => $database->getMorphClass(),
+        'team_id' => $team->id,
+    ])->save();
+    $backup->syncS3Storages([$storage->id]);
+    $processes = s3ArgvTestFakeSsh();
+
+    (new DatabaseBackupJob($backup->fresh()))->handle();
+
+    expect($backup->executions()->sole()->s3_uploaded)->toBeTrue();
+    s3ArgvTestAssertSecretOnlyOnStdin($processes, $sshUser !== 'root');
+})->with(['root', 'deploy']);
+
+it('passes the S3 keys of a volume backup to mc on stdin, not in a command line', function (string $sshUser, bool $streamToS3) {
+    [$team, $server, $storage] = s3ArgvTestSetup($sshUser);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+    $application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+    $volume = LocalPersistentVolume::create([
+        'name' => 'app-data',
+        'mount_path' => '/data',
+        'resource_id' => $application->id,
+        'resource_type' => $application->getMorphClass(),
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        'disable_local_backup' => $streamToS3,
+        's3_storage_id' => $storage->id,
+    ]);
+    $processes = s3ArgvTestFakeSsh();
+
+    (new VolumeBackupJob($backup))->handle();
+
+    $execution = ScheduledVolumeBackupExecution::query()->sole();
+    expect($execution->status)->toBe('success')
+        ->and($execution->s3_uploaded)->toBeTrue()
+        ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBe($streamToS3);
+    s3ArgvTestAssertSecretOnlyOnStdin($processes, $sshUser !== 'root');
+})->with(['root', 'deploy'])->with(['local archive upload' => false, 'S3-only stream' => true]);

tests/Feature/BackupS3DestinationRedactionTest.php

@@ -91,7 +91,7 @@ it('redacts the credentials of every S3 destination from a database backup uploa
     ]);
     $backup->syncS3Storages($this->storages->pluck('id')->all());
     Process::fake(function ($process) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
 
         return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
     });
@@ -142,7 +142,7 @@ it('redacts the credentials of every S3 destination from a volume backup upload
     $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
     Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
     Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
-    Process::fake([
+    fakeRemoteProcesses([
         '*du -b*' => '128',
         '*mc cp*' => Process::result(errorOutput: $this->uploadError, exitCode: 1),
         '*' => '',

tests/Feature/BackupS3DownloadTest.php

@@ -271,10 +271,10 @@ it('uploads the backup file from the server host with curl', function () {
     (new PrepareFileDownloadJob('dl-host', $this->user->id, $this->server->id, null, BACKUP_S3_DOWNLOAD_FILE, false, 'pg.dmp', $this->downloadStorage->id))->handle($downloads);
 
     expect(Cache::get(PrepareFileDownloadJob::cacheKey('dl-host')))->toMatchArray(['status' => 'ready', 'url' => 'https://s3.example.com/get']);
-    Process::assertRan(fn ($process) => str_contains($process->command, 'curl -fsS')
-        && str_contains($process->command, BACKUP_S3_DOWNLOAD_FILE)
-        && str_contains($process->command, 'X-Amz-Signature=abc&x=1')
-        && ! str_contains($process->command, 'docker exec'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'curl -fsS')
+        && str_contains(remoteProcessScript($process), BACKUP_S3_DOWNLOAD_FILE)
+        && str_contains(remoteProcessScript($process), 'X-Amz-Signature=abc&x=1')
+        && ! str_contains(remoteProcessScript($process), 'docker exec'));
 });
 
 it('builds host file commands that the non-root sudo parser keeps whole', function () {

tests/Feature/BackupTimeoutSecretRedactionTest.php

@@ -57,7 +57,7 @@ it('does not put the command into the error when a remote command times out', fu
     Process::fake(function ($process) use (&$attempts) {
         $attempts++;
 
-        throw backupRedactionTimeoutException($process->command, 7200);
+        throw backupRedactionTimeoutException(remoteProcessScript($process), 7200);
     });
 
     try {
@@ -94,8 +94,8 @@ it('does not put database credentials into the backup execution or the failure n
         'timeout' => 600,
     ]);
     Process::fake(function ($process) {
-        if (str_contains($process->command, 'mysqldump')) {
-            throw backupRedactionTimeoutException($process->command, 600);
+        if (str_contains(remoteProcessScript($process), 'mysqldump')) {
+            throw backupRedactionTimeoutException(remoteProcessScript($process), 600);
         }
 
         return Process::result(output: '');

tests/Feature/ClientIpSpoofingTest.php

@@ -0,0 +1,106 @@
+<?php
+
+use App\Models\InstanceSettings;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Testing\TestResponse;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    config()->set('app.maintenance.store', 'array');
+    config()->set('constants.coolify.self_hosted', true);
+
+    InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true, 'allowed_ips' => '203.0.113.50,2001:db8::50']);
+
+    $team = Team::factory()->create();
+    $user = User::factory()->create();
+    $team->members()->attach($user->id, ['role' => 'owner']);
+    $this->apiToken = $user->createToken('client-ip', ['read'], teamId: $team->id)->plainTextToken;
+});
+
+/**
+ * An API call with a valid token: 403 when the allowed IPs block the client, 200 when they let it through.
+ *
+ * @param  array<string, string>  $headers
+ */
+function apiIpCheckFromPeer(string $peer, array $headers = []): TestResponse
+{
+    return test()
+        ->withToken(test()->apiToken)
+        ->withHeaders($headers)
+        ->withServerVariables(['REMOTE_ADDR' => $peer])
+        ->getJson('/api/v1/version');
+}
+
+it('ignores forwarded headers from a public peer for the allowed IPs', function (array $headers) {
+    apiIpCheckFromPeer('198.51.100.7', $headers)->assertForbidden();
+})->with([
+    'X-Forwarded-For' => [['X-Forwarded-For' => '203.0.113.50']],
+    'fake Cloudflare hop' => [['X-Forwarded-For' => '104.16.0.1', 'CF-Connecting-IP' => '203.0.113.50']],
+]);
+
+it('ignores forwarded headers from a public IPv6 peer for the allowed IPs', function (string $peer) {
+    apiIpCheckFromPeer($peer, ['X-Forwarded-For' => '2001:db8::50'])->assertForbidden();
+})->with(['2001:db8:ffff::7', '::ffff:198.51.100.7']);
+
+it('uses the forwarded client from a local reverse proxy for the allowed IPs', function (string $peer, string $client) {
+    apiIpCheckFromPeer($peer, ['X-Forwarded-For' => $client])->assertOk();
+    apiIpCheckFromPeer($peer, ['X-Forwarded-For' => '198.51.100.7'])->assertForbidden();
+})->with([
+    'Docker network' => ['10.0.1.5', '203.0.113.50'],
+    'Docker bridge' => ['172.18.0.2', '203.0.113.50'],
+    'loopback' => ['127.0.0.1', '203.0.113.50'],
+    'IPv6 loopback' => ['::1', '2001:db8::50'],
+    'IPv6 unique local' => ['fd00::2', '2001:db8::50'],
+    'IPv6 link local' => ['fe80::2', '2001:db8::50'],
+    'IPv4-mapped private' => ['::ffff:10.0.1.5', '203.0.113.50'],
+]);
+
+it('uses the Cloudflare client IP behind a local reverse proxy only when the proxy received the request from Cloudflare', function () {
+    apiIpCheckFromPeer('10.0.1.5', [
+        'X-Forwarded-For' => '203.0.113.50, 162.158.79.140',
+        'CF-Connecting-IP' => '203.0.113.50',
+    ])->assertOk();
+
+    apiIpCheckFromPeer('10.0.1.5', [
+        'X-Forwarded-For' => '104.16.0.1, 198.51.100.7',
+        'CF-Connecting-IP' => '203.0.113.50',
+    ])->assertForbidden();
+});
+
+it('uses the Cloudflare client IP when Cloudflare connects directly', function () {
+    apiIpCheckFromPeer('162.158.79.140', ['CF-Connecting-IP' => '203.0.113.50'])->assertOk();
+});
+
+it('keys the API rate limit by the public peer, not by forwarded headers', function () {
+    config()->set('api.rate_limit', 2);
+    InstanceSettings::query()->update(['allowed_ips' => null]);
+
+    $push = fn (string $peer, array $headers) => test()
+        ->withHeaders($headers)
+        ->withServerVariables(['REMOTE_ADDR' => $peer])
+        ->postJson('/api/v1/sentinel/push');
+
+    $push('198.51.100.7', ['X-Forwarded-For' => '203.0.113.1'])->assertUnauthorized();
+    $push('198.51.100.7', ['X-Forwarded-For' => '203.0.113.2', 'CF-Connecting-IP' => '203.0.113.2'])->assertUnauthorized();
+    $push('198.51.100.7', ['X-Forwarded-For' => '104.16.0.1', 'CF-Connecting-IP' => '203.0.113.3'])->assertStatus(429);
+
+    $push('10.0.1.5', ['X-Forwarded-For' => '203.0.113.10'])->assertUnauthorized();
+    $push('10.0.1.5', ['X-Forwarded-For' => '203.0.113.20'])->assertUnauthorized();
+});
+
+it('ignores X-Forwarded-For from a public peer on cloud', function () {
+    config()->set('constants.coolify.self_hosted', false);
+    config()->set('api.rate_limit', 2);
+
+    $push = fn (string $forwardedFor) => test()
+        ->withHeader('X-Forwarded-For', $forwardedFor)
+        ->withServerVariables(['REMOTE_ADDR' => '198.51.100.7'])
+        ->postJson('/api/v1/sentinel/push');
+
+    $push('203.0.113.1')->assertUnauthorized();
+    $push('203.0.113.2')->assertUnauthorized();
+    $push('203.0.113.3')->assertStatus(429);
+});

tests/Feature/CloneArchiveDirectoryTest.php

@@ -47,17 +47,17 @@ beforeEach(function () {
 function fakeCloneArchiveServers(object $test, bool $failScp = false): void
 {
     Process::fake(function ($process) use ($test, $failScp) {
-        if (str_contains($process->command, 'mktemp -d')) {
+        if (str_contains(remoteProcessScript($process), 'mktemp -d')) {
             $directory = '/var/tmp/coolify-clone.'.Str::random(10);
             $test->mktempDirectories[] = $directory;
 
             return Process::result(output: $directory);
         }
-        if ($failScp && str_contains($process->command, ' scp ')) {
+        if ($failScp && str_contains(remoteProcessScript($process), ' scp ')) {
             return Process::result(output: '', errorOutput: 'scp failed', exitCode: 1);
         }
 
-        return Process::result(output: str_contains($process->command, 'docker volume inspect') ? '[]' : '');
+        return Process::result(output: str_contains(remoteProcessScript($process), 'docker volume inspect') ? '[]' : '');
     });
 }
 
@@ -68,7 +68,7 @@ function cloneArchiveCommands(): array
 {
     $commands = [];
     Process::assertRan(function ($process) use (&$commands) {
-        $commands[] = $process->command;
+        $commands[] = remoteProcessScript($process);
 
         return true;
     });

tests/Feature/CloneServiceVolumeDataTest.php

@@ -76,11 +76,10 @@ beforeEach(function () {
  */
 function chainedServiceVolumeClones(): array
 {
-    $first = Bus::dispatched(CallQueuedClosure::class)->first();
+    $first = Bus::dispatched(CallQueuedClosure::class)->first() ?? Bus::dispatched(VolumeCloneJob::class)->first();
     expect($first)->not->toBeNull('No volume clone chain was dispatched.');
 
-    return collect($first->chained)
-        ->map(fn (string $job) => unserialize($job))
+    return collect([$first, ...array_map(fn (string $job) => unserialize($job), $first->chained)])
         ->filter(fn ($job) => $job instanceof VolumeCloneJob)
         ->map(fn (VolumeCloneJob $job) => (fn () => [$this->sourceVolume, $this->targetVolume])->call($job))
         ->values()

tests/Feature/CloneVolumeDataSourceSafetyTest.php

@@ -0,0 +1,612 @@
+<?php
+
+use App\Actions\Application\StopApplication;
+use App\Actions\Database\StartDatabase;
+use App\Actions\Database\StopDatabase;
+use App\Actions\Service\StartService;
+use App\Actions\Service\StopService;
+use App\Jobs\VolumeCloneJob;
+use App\Livewire\Project\CloneMe;
+use App\Livewire\Project\Shared\ResourceOperations;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\Service;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Models\User;
+use App\Notifications\Internal\GeneralNotification;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\PendingProcess;
+use Illuminate\Queue\CallQueuedClosure;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Gate;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Str;
+use Laravel\Horizon\Tags;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+const CLONE_SAFETY_SERVICE_COMPOSE = <<<'YAML'
+services:
+  web:
+    image: nginx:alpine
+    volumes:
+      - 'web-data:/usr/share/nginx/html'
+YAML;
+
+beforeEach(function () {
+    $this->withoutVite();
+    Server::flushIdentityMap();
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+    config(['constants.ssh.mux_enabled' => false]);
+    Notification::fake();
+    Process::fake(['*' => Process::result(output: '')]);
+
+    $this->user = User::factory()->create();
+    $this->team = Team::factory()->create();
+    $this->user->teams()->attach($this->team, ['role' => 'owner']);
+
+    $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
+    $server = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $privateKey->id]);
+    $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+    $this->destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $this->project = Project::factory()->create(['team_id' => $this->team->id]);
+    $this->environment = $this->project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $this->project->id]);
+
+    $this->service = Service::factory()->create([
+        'docker_compose_raw' => CLONE_SAFETY_SERVICE_COMPOSE,
+        'environment_id' => $this->environment->id,
+        'server_id' => $server->id,
+        'destination_id' => $this->destination->id,
+        'destination_type' => $this->destination->getMorphClass(),
+    ]);
+    $this->service->parse();
+
+    $this->actingAs($this->user);
+    session(['currentTeam' => $this->team]);
+});
+
+/**
+ * Runs the queued jobs with the database driver, so every chain step and the catch callback
+ * are serialized and restored like on a Horizon worker.
+ */
+function cloneSafetyUseDatabaseQueue(): void
+{
+    Schema::create('jobs', function (Blueprint $table) {
+        $table->id();
+        $table->string('queue')->index();
+        $table->longText('payload');
+        $table->unsignedTinyInteger('attempts');
+        $table->unsignedInteger('reserved_at')->nullable();
+        $table->unsignedInteger('available_at');
+        $table->unsignedInteger('created_at');
+    });
+    config(['queue.default' => 'database']);
+}
+
+/**
+ * The worker runs in the test process, so a high `--memory` keeps the default 128 MB limit from
+ * stopping it after the first chain step when earlier tests have grown the process.
+ */
+function cloneSafetyWorkQueue(): void
+{
+    Artisan::call('queue:work', ['connection' => 'database', '--stop-when-empty' => true, '--queue' => 'high', '--sleep' => 0, '--memory' => 4096]);
+}
+
+/** The volume copy (`cp -a` in a helper container) fails like on a full disk; every other remote command works. */
+function cloneSafetyFailVolumeCopy(): void
+{
+    Process::fake(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'cp -a')
+        ? Process::result(errorOutput: 'No space left on device', exitCode: 1)
+        : Process::result(output: ''));
+}
+
+/** `docker ps` (running containers only) lists the given container names; every other remote command works. */
+function cloneSafetyListRunningContainers(string ...$names): void
+{
+    Process::fake(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'docker ps --filter')
+        ? Process::result(output: implode("\n", $names))
+        : Process::result(output: ''));
+}
+
+function cloneSafetyMarkServiceRunning(Service $service): void
+{
+    $service->applications()->update(['status' => 'running:healthy']);
+    $service->databases()->update(['status' => 'running:healthy']);
+}
+
+test('a failed volume copy starts the running source service again and notifies the team', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyFailVolumeCopy();
+    cloneSafetyMarkServiceRunning($this->service);
+    StopService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service));
+    StartService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service));
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    expect(DB::table('failed_jobs')->count())->toBe(1);
+    Notification::assertSentTo($this->team, GeneralNotification::class, function (GeneralNotification $notification) {
+        return $notification->success === false
+            && str_contains($notification->message, $this->service->name)
+            && str_contains($notification->message, 'No space left on device');
+    });
+});
+
+test('a volume copy of a stopped source service neither stops nor starts it', function () {
+    cloneSafetyUseDatabaseQueue();
+    StopService::shouldRun()->never();
+    StartService::shouldRun()->never();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    expect(DB::table('failed_jobs')->count())->toBe(0);
+    Notification::assertNothingSent();
+});
+
+test('a successful volume copy stops the running source service and starts it again', function () {
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyMarkServiceRunning($this->service);
+    StopService::shouldRun()->once();
+    StartService::shouldRun()->once();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    expect(DB::table('failed_jobs')->count())->toBe(0);
+    Notification::assertNothingSent();
+});
+
+test('a failed stop that leaves some containers of the source service running starts only the stopped containers again', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyMarkServiceRunning($this->service);
+    cloneSafetyListRunningContainers('web-'.$this->service->uuid);
+    StopService::shouldRun()->once()->andReturn('docker stop timed out');
+    StartService::shouldRun()->never();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    $workdir = $this->service->workdir();
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker compose --project-directory {$workdir} -f {$workdir}/docker-compose.yml --project-name {$this->service->uuid} up -d --no-recreate < /dev/null"));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), '--force-recreate'));
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false
+        && str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'started its stopped containers again')
+        && str_contains($notification->message, 'docker stop timed out'));
+});
+
+test('a failed start of the stopped containers of a partly stopped source service is in the notification', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyMarkServiceRunning($this->service);
+    Process::fake(fn (PendingProcess $process) => match (true) {
+        str_contains(remoteProcessScript($process), 'docker ps --filter') => Process::result(output: 'web-'.$this->service->uuid),
+        str_contains(remoteProcessScript($process), 'up -d --no-recreate') => Process::result(errorOutput: 'Cannot connect to the Docker daemon', exitCode: 1),
+        default => Process::result(output: ''),
+    });
+    StopService::shouldRun()->once()->andReturn(null);
+    StartService::shouldRun()->never();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'could not start its stopped containers again'));
+});
+
+test('a stop that leaves containers of the source service running copies no data', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyMarkServiceRunning($this->service);
+    cloneSafetyListRunningContainers('web-'.$this->service->uuid);
+    StopService::shouldRun()->once()->andReturn(null);
+    StartService::shouldRun()->never();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'web-'.$this->service->uuid));
+});
+
+test('a failed stop that left the source service down starts it again', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyMarkServiceRunning($this->service);
+    StopService::shouldRun()->once()->andReturn('docker stop timed out');
+    StartService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service));
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'started')
+        && str_contains($notification->message, 'docker stop timed out'));
+});
+
+test('cloning volume data needs the permission to stop the source resource', function () {
+    Bus::fake();
+    Gate::before(fn ($user, string $ability) => $ability === 'stop' ? false : null);
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid);
+
+    expect(Service::query()->count())->toBe(1);
+    Bus::assertNotDispatched(CallQueuedClosure::class);
+
+    Livewire::test(CloneMe::class, [
+        'project_uuid' => $this->project->uuid,
+        'environment_uuid' => $this->environment->uuid,
+    ])
+        ->set('selectedDestination', $this->destination->uuid)
+        ->set('newName', 'cloned-env')
+        ->set('cloneVolumeData', true)
+        ->call('clone', 'environment');
+
+    expect(Service::query()->count())->toBe(1)
+        ->and($this->project->environments()->where('name', 'cloned-env')->exists())->toBeFalse();
+    Bus::assertNotDispatched(CallQueuedClosure::class);
+});
+
+test('cloning without volume data needs no stop permission and stops nothing', function () {
+    Bus::fake();
+    Gate::before(fn ($user, string $ability) => $ability === 'stop' ? false : null);
+
+    Livewire::test(ResourceOperations::class, ['resource' => $this->service])
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    expect(Service::query()->count())->toBe(2);
+    Bus::assertNotDispatched(CallQueuedClosure::class);
+    Bus::assertNotDispatched(VolumeCloneJob::class);
+});
+
+test('cloning a running database with volume data copies its volumes in one chain between stop and start', function (string $component) {
+    Bus::fake();
+    $database = StandalonePostgresql::create([
+        'name' => 'orders-db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db',
+        'image' => 'postgres:17', 'environment_id' => $this->environment->id,
+        'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(),
+    ]);
+    $database->forceFill(['status' => 'running:healthy'])->saveQuietly();
+    LocalPersistentVolume::query()->where('resource_type', $database->getMorphClass())->where('resource_id', $database->id)->delete();
+    foreach (['postgres-data-'.$database->uuid, $database->uuid.'-extra'] as $name) {
+        LocalPersistentVolume::create([
+            'name' => $name,
+            'mount_path' => '/data/'.$name,
+            'resource_id' => $database->id,
+            'resource_type' => $database->getMorphClass(),
+        ]);
+    }
+    $this->service->delete();
+
+    if ($component === ResourceOperations::class) {
+        Livewire::test(ResourceOperations::class, ['resource' => $database])
+            ->set('cloneVolumeData', true)
+            ->call('cloneTo', $this->destination->uuid)
+            ->assertHasNoErrors();
+    } else {
+        Livewire::test(CloneMe::class, [
+            'project_uuid' => $this->project->uuid,
+            'environment_uuid' => $this->environment->uuid,
+        ])
+            ->set('selectedDestination', $this->destination->uuid)
+            ->set('newName', 'cloned-env')
+            ->set('cloneVolumeData', true)
+            ->call('clone', 'environment')
+            ->assertHasNoErrors();
+    }
+
+    Bus::assertNotDispatched(VolumeCloneJob::class);
+    Bus::assertChained([
+        CallQueuedClosure::class,
+        VolumeCloneJob::class,
+        VolumeCloneJob::class,
+        CallQueuedClosure::class,
+    ]);
+})->with([
+    'resource operations' => [ResourceOperations::class],
+    'environment clone' => [CloneMe::class],
+]);
+
+test('a restored same-server volume clone job has explicit Horizon tags', function () {
+    $server = $this->destination->server;
+    $volume = LocalPersistentVolume::create(['name' => 'clone-target', 'mount_path' => '/data', 'resource_id' => 1, 'resource_type' => Service::class]);
+
+    // Without explicit tags, Horizon compares the job's models with `==`. Two restored copies of one
+    // server fail that comparison on PHP 8.4 ("Nesting level too deep"), so the chain stops before the copy.
+    $restoredJob = unserialize(serialize(new VolumeCloneJob('clone-source', 'clone-target', $server, $server, $volume)));
+
+    expect(Tags::extractExplicitTags($restoredJob))->toBe([Server::class.':'.$server->id, LocalPersistentVolume::class.':'.$volume->id]);
+});
+
+function cloneSafetyCreateDatabase(object $test, string $status): StandalonePostgresql
+{
+    $database = StandalonePostgresql::create([
+        'name' => 'api-db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db',
+        'image' => 'postgres:17', 'environment_id' => $test->environment->id,
+        'destination_id' => $test->destination->id, 'destination_type' => $test->destination->getMorphClass(),
+    ]);
+    $database->forceFill(['status' => $status])->saveQuietly();
+
+    return $database;
+}
+
+function cloneSafetyApiHeaders(User $user, int $teamId): array
+{
+    $plainTextToken = Str::random(40);
+    $token = $user->tokens()->create([
+        'name' => 'clone-safety',
+        'token' => hash('sha256', $plainTextToken),
+        'abilities' => ['*'],
+        'team_id' => $teamId,
+    ]);
+    auth()->forgetGuards();
+
+    return ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken];
+}
+
+function cloneSafetyCreateApplication(object $test, string $status): Application
+{
+    $application = Application::factory()->create([
+        'environment_id' => $test->environment->id,
+        'destination_id' => $test->destination->id,
+        'destination_type' => $test->destination->getMorphClass(),
+    ]);
+    $application->forceFill(['status' => $status])->saveQuietly();
+    LocalPersistentVolume::create([
+        'name' => $application->uuid.'-data',
+        'mount_path' => '/data',
+        'resource_id' => $application->id,
+        'resource_type' => $application->getMorphClass(),
+    ]);
+
+    return $application;
+}
+
+test('a failed volume copy of a running database cloned through the API starts it again and notifies the team', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    cloneSafetyFailVolumeCopy();
+    $database = cloneSafetyCreateDatabase($this, 'running:healthy');
+    StopDatabase::shouldRun()->once()->withArgs(fn ($source) => $source->is($database));
+    StartDatabase::shouldRun()->once()->withArgs(fn ($source) => $source->is($database));
+
+    $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id))
+        ->postJson("/api/v1/databases/{$database->uuid}/clone", [
+            'destination_uuid' => $this->destination->uuid,
+            'clone_volumes' => true,
+        ])
+        ->assertCreated();
+
+    cloneSafetyWorkQueue();
+
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false
+        && str_contains($notification->message, 'api-db')
+        && str_contains($notification->message, 'No space left on device'));
+});
+
+test('a failed stop of a running database cloned through the API copies no data and leaves it running', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    $database = cloneSafetyCreateDatabase($this, 'running:healthy');
+    cloneSafetyListRunningContainers($database->uuid);
+    StopDatabase::shouldRun()->once()->andReturn('Database stop failed: Connection refused');
+    StartDatabase::shouldRun()->never();
+
+    $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id))
+        ->postJson("/api/v1/databases/{$database->uuid}/clone", [
+            'destination_uuid' => $this->destination->uuid,
+            'clone_volumes' => true,
+        ])
+        ->assertCreated();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'api-db')
+        && str_contains($notification->message, 'Connection refused'));
+});
+
+test('cloning a running database through the API copies its volumes after a successful stop', function () {
+    cloneSafetyUseDatabaseQueue();
+    $database = cloneSafetyCreateDatabase($this, 'running:healthy');
+    StopDatabase::shouldRun()->once()->andReturn('Database stopped successfully');
+    StartDatabase::shouldRun()->once();
+
+    $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id))
+        ->postJson("/api/v1/databases/{$database->uuid}/clone", [
+            'destination_uuid' => $this->destination->uuid,
+            'clone_volumes' => true,
+        ])
+        ->assertCreated();
+
+    cloneSafetyWorkQueue();
+
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    expect(DB::table('failed_jobs')->count())->toBe(0);
+    Notification::assertNothingSent();
+});
+
+test('cloning a stopped database through the API copies its volumes without stopping or starting it', function () {
+    Bus::fake();
+    $database = cloneSafetyCreateDatabase($this, 'exited');
+
+    $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id))
+        ->postJson("/api/v1/databases/{$database->uuid}/clone", [
+            'destination_uuid' => $this->destination->uuid,
+            'clone_volumes' => true,
+        ])
+        ->assertCreated();
+
+    Bus::assertNotDispatched(CallQueuedClosure::class);
+    Bus::assertDispatched(VolumeCloneJob::class);
+});
+
+/**
+ * `docker ps` lists the running containers: before the stop the given ones, after it the remaining ones.
+ *
+ * @param  list<string>  $before
+ * @param  list<string>  $after
+ */
+function cloneSafetyListRunningContainersBeforeAndAfterStop(array $before, array $after): void
+{
+    $listings = 0;
+    Process::fake(function (PendingProcess $process) use (&$listings, $before, $after) {
+        if (! str_contains(remoteProcessScript($process), 'docker ps --filter')) {
+            return Process::result(output: '');
+        }
+
+        return Process::result(output: implode("\n", $listings++ === 0 ? $before : $after));
+    });
+}
+
+function cloneSafetyCloneApplicationWithVolumes(object $test, Application $application): void
+{
+    $test->service->delete();
+    Livewire::test(ResourceOperations::class, ['resource' => $application])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $test->destination->uuid)
+        ->assertHasNoErrors();
+    cloneSafetyWorkQueue();
+}
+
+test('a failed stop that took down some containers of the source application redeploys the commit that ran before', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    $application = cloneSafetyCreateApplication($this, 'running:healthy');
+    $runningCommit = str_repeat('a', 40);
+    foreach ([$runningCommit => now()->subHour(), str_repeat('b', 40) => now()->subDay()] as $commit => $updatedAt) {
+        ApplicationDeploymentQueue::create([
+            'application_id' => $application->id, 'deployment_uuid' => new_public_id(), 'pull_request_id' => 0,
+            'commit' => $commit, 'status' => 'finished', 'server_id' => $this->destination->server_id, 'destination_id' => $this->destination->id,
+        ])->forceFill(['updated_at' => $updatedAt])->saveQuietly();
+    }
+    cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid, 'worker-'.$application->uuid], ['worker-'.$application->uuid]);
+    StopApplication::shouldRun()->once()->andReturn('docker stop failed for worker');
+
+    cloneSafetyCloneApplicationWithVolumes($this, $application);
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    $deployments = ApplicationDeploymentQueue::query()->where('application_id', $application->id)->orderBy('id')->get();
+    expect($deployments)->toHaveCount(3);
+    $redeploy = $deployments->last();
+    expect($redeploy->commit)->toBe($runningCommit)
+        ->and($redeploy->pull_request_id)->toBe(0);
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false
+        && str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, "started a redeploy of '{$application->name}'")
+        && str_contains($notification->message, 'docker stop failed for worker'));
+});
+
+test('a failed stop that took down containers of a source application without a known commit redeploys its configured commit', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    $application = cloneSafetyCreateApplication($this, 'running:healthy');
+    cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid, 'worker-'.$application->uuid], ['worker-'.$application->uuid]);
+    StopApplication::shouldRun()->once()->andReturn(null);
+
+    cloneSafetyCloneApplicationWithVolumes($this, $application);
+
+    expect(ApplicationDeploymentQueue::query()->where('application_id', $application->id)->sole()->commit)->toBe('HEAD');
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, "started a redeploy of '{$application->name}'")
+        && str_contains($notification->message, 'could not find the commit'));
+});
+
+test('a failed stop that took down no container of the source application does not redeploy it', function () {
+    $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']);
+    cloneSafetyUseDatabaseQueue();
+    $application = cloneSafetyCreateApplication($this, 'running:healthy');
+    cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid], ['web-'.$application->uuid]);
+    StopApplication::shouldRun()->once()->andReturn('Server is not functional');
+
+    cloneSafetyCloneApplicationWithVolumes($this, $application);
+
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a'));
+    expect(ApplicationDeploymentQueue::query()->where('application_id', $application->id)->count())->toBe(0);
+    Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy')
+        && str_contains($notification->message, 'is still running'));
+});
+
+test('cloning a running application with volume data copies its volumes in one chain between stop and start', function () {
+    Bus::fake();
+    $application = cloneSafetyCreateApplication($this, 'running:healthy');
+    $this->service->delete();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $application])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    Bus::assertChained([CallQueuedClosure::class, VolumeCloneJob::class, CallQueuedClosure::class]);
+    expect(ApplicationDeploymentQueue::query()->count())->toBe(0);
+});
+
+test('cloning a stopped application with volume data neither stops nor redeploys it', function () {
+    Bus::fake();
+    $application = cloneSafetyCreateApplication($this, 'exited');
+    $this->service->delete();
+
+    Livewire::test(ResourceOperations::class, ['resource' => $application])
+        ->set('cloneVolumeData', true)
+        ->call('cloneTo', $this->destination->uuid)
+        ->assertHasNoErrors();
+
+    Bus::assertNotDispatched(CallQueuedClosure::class);
+    Bus::assertDispatched(VolumeCloneJob::class);
+    expect(ApplicationDeploymentQueue::query()->count())->toBe(0);
+});

tests/Feature/CloudflareTunnelApiSetupTest.php

@@ -272,7 +272,7 @@ describe('disable', function () {
             ->assertNotDispatched('warning');
 
         // Detached, so stopping cloudflared does not cut the SSH session that runs through the tunnel.
-        Process::assertRan(fn ($process) => str_contains($process->command, "nohup setsid sh -c 'sleep 2; docker rm -f coolify-cloudflared' > /dev/null 2>&1 < /dev/null &"));
+        Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "nohup setsid sh -c 'sleep 2; docker rm -f coolify-cloudflared' > /dev/null 2>&1 < /dev/null &"));
         Sleep::assertNeverSlept();
 
         Http::assertSent(fn (Request $request) => $request->method() === 'DELETE' && str_ends_with($request->url(), CF_TUNNEL_API.'/tun-1'));
@@ -335,7 +335,7 @@ describe('disable', function () {
         Livewire::test(CloudflareTunnel::class, ['server_uuid' => $this->server->uuid])
             ->call('toggleCloudflareTunnels');
 
-        Process::assertRan(fn ($process) => str_contains($process->command,
+        Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process),
             "sudo bash -c 'nohup setsid sh -c '\\''sleep 2; docker rm -f coolify-cloudflared'\\'' > /dev/null 2>&1 < /dev/null &'"));
     });
 

tests/Feature/ComposeContentVolumeConfinementTest.php

@@ -132,8 +132,8 @@ function contentHostPathUseServerUser(string $user): void
 function contentHostPathFakeServer(): void
 {
     Process::fake(fn ($process) => Process::result(output: match (true) {
-        str_contains($process->command, 'readlink -f') => 'OK',
-        str_contains($process->command, 'test -') => 'NOK',
+        str_contains(remoteProcessScript($process), 'readlink -f') => 'OK',
+        str_contains(remoteProcessScript($process), 'test -') => 'NOK',
         default => '',
     }));
 }
@@ -165,8 +165,8 @@ function contentHostPathAssertWritten(string $path, string $content, string $use
     $escapedParent = escapeshellarg(dirname($path));
     $base64 = base64_encode($content);
 
-    Process::assertRan(fn ($process) => str_contains($process->command, "{$sudo}mkdir -p {$escapedParent}"));
-    Process::assertRan(fn ($process) => str_contains($process->command, "echo '{$base64}' | {$sudo}base64 -d | {$sudo}tee {$escapedPath} > /dev/null"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "{$sudo}mkdir -p {$escapedParent}"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "echo '{$base64}' | {$sudo}base64 -d | {$sudo}tee {$escapedPath} > /dev/null"));
 }
 
 /**
@@ -230,8 +230,8 @@ test('loading files for a service writes content inside the resource directory',
     $sudo = $user === 'root' ? '' : 'sudo ';
     $escapedPath = escapeshellarg($writePath);
     $base64 = base64_encode($content);
-    Process::assertRan(fn ($process) => str_contains($process->command, "{$sudo}mkdir -p -- \"$({$sudo}dirname -- {$escapedPath})\""));
-    Process::assertRan(fn ($process) => str_contains($process->command, "echo '{$base64}' | {$sudo}base64 -d | {$sudo}tee -- {$escapedPath}"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "{$sudo}mkdir -p -- \"$({$sudo}dirname -- {$escapedPath})\""));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "echo '{$base64}' | {$sudo}base64 -d | {$sudo}tee -- {$escapedPath}"));
 })->with([
     'home directory' => ['/etc/b.conf', 'b'],
     'bare relative path' => ['/etc/c.conf', 'c'],
@@ -247,7 +247,7 @@ test('loading files for a service does not write content outside the resource di
     contentHostPathFakeServer();
     getFilesystemVolumesFromServer($fileVolume->resource, true);
 
-    Process::assertNotRan(fn ($process) => str_contains($process->command, base64_encode($content)));
+    Process::assertNotRan(fn ($process) => str_contains(remoteProcessScript($process), base64_encode($content)));
 })->with([
     'absolute host file' => ['/etc/a.conf', 'a'],
     'parent directory' => ['/etc/d.conf', 'd'],
@@ -266,8 +266,8 @@ test('a quote in a content path is shell-escaped in every write command', functi
     $fileVolume->saveStorageOnServer();
 
     $sudo = $user === 'root' ? '' : 'sudo ';
-    Process::assertRan(fn ($process) => str_contains($process->command, "{$sudo}tee '/etc/my'\\''app/app.conf' > /dev/null"));
-    Process::assertRan(fn ($process) => str_contains($process->command, "{$sudo}mkdir -p '/etc/my'\\''app'"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "{$sudo}tee '/etc/my'\\''app/app.conf' > /dev/null"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "{$sudo}mkdir -p '/etc/my'\\''app'"));
 })->with(['root', 'coolify']);
 
 test('the parsers reject shell injection in a content volume source', function () {
@@ -294,8 +294,8 @@ test('a bind mount without content keeps its administrator-selected host path',
     contentHostPathFakeServer();
     $fileVolume->saveStorageOnServer();
 
-    Process::assertRan(fn ($process) => str_contains($process->command, "mkdir -p '/srv/shared'"));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'readlink -f'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "mkdir -p '/srv/shared'"));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'readlink -f'));
 });
 
 test('the service stack form saves content volumes with host paths', function () {
@@ -357,7 +357,7 @@ test('the application General form rejects shell injection in a content volume s
 function contentHostPathFakeGitCompose(string $compose): void
 {
     Process::fake(function ($process) use ($compose) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
 
         return Process::result(output: match (true) {
             str_contains($command, 'git cat-file blob') => $compose,
@@ -436,5 +436,5 @@ test('loading files for a service does not create directories outside the resour
     contentHostPathFakeServer();
     getFilesystemVolumesFromServer($fileVolume->resource, true);
 
-    Process::assertNotRan(fn ($process) => str_contains($process->command, '/etc/cron.d/coolify-test') && str_contains($process->command, 'mkdir'));
+    Process::assertNotRan(fn ($process) => str_contains(remoteProcessScript($process), '/etc/cron.d/coolify-test') && str_contains(remoteProcessScript($process), 'mkdir'));
 });

tests/Feature/ComposeExternalVolumeTest.php

@@ -261,7 +261,7 @@ function loginExternalVolumeOwner(): void
 function fakeExternalVolumeServer(array &$commands): void
 {
     Process::fake(function ($process) use (&$commands) {
-        $commands[] = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $commands[] = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
 
         return Process::result(output: '');
     });

tests/Feature/ComposeFileLoadCommandsTest.php

@@ -1,9 +1,14 @@
 <?php
 
 use App\Models\Application;
+use App\Models\GiteaApp;
+use App\Models\GithubApp;
+use App\Models\GitlabApp;
 use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
 use App\Models\Server;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
 use Symfony\Component\Process\Process;
 
 uses(RefreshDatabase::class);
@@ -240,3 +245,165 @@ test('a symlinked Compose file inside a Git submodule is rejected', function ()
         ->and($process->getErrorOutput())->toContain('symbolic link')
         ->and($process->getErrorOutput())->not->toContain('linked-target-value');
 });
+
+test('the Compose file loads from a Git submodule with a deploy key', function (string $user) {
+    Storage::fake('ssh-keys');
+    $submoduleSource = $this->root.'/infra-src';
+    mkdir($submoduleSource);
+    file_put_contents("{$submoduleSource}/docker-compose.yml", $this->compose);
+    $gitEnv = ['GIT_CONFIG_COUNT' => '1', 'GIT_CONFIG_KEY_0' => 'protocol.file.allow', 'GIT_CONFIG_VALUE_0' => 'always'];
+    foreach ([
+        [$submoduleSource, ['git', 'init', '-q', '-b', 'main']],
+        [$submoduleSource, ['git', 'add', '-A']],
+        [$submoduleSource, ['git', '-c', 'user.email=test@example.com', '-c', 'user.name=Test', 'commit', '-qm', 'infra']],
+        [$submoduleSource, ['git', 'clone', '-q', '--bare', '.', $this->root.'/infra.git']],
+        [$this->root.'/src', ['git', 'submodule', '-q', 'add', $this->root.'/infra.git', 'infra']],
+        [$this->root.'/src', ['git', '-c', 'user.email=test@example.com', '-c', 'user.name=Test', 'commit', '-qm', 'add infra']],
+        [$this->root.'/src', ['git', 'push', '-q', $this->root.'/repo.git', 'main']],
+    ] as [$cwd, $command]) {
+        (new Process($command, $cwd, $gitEnv))->mustRun();
+    }
+
+    // A git stub that clones the SSH remote from the local bare repository, and fails a submodule
+    // update like an SSH authentication error when the deploy key of GIT_SSH_COMMAND is missing.
+    $realGit = trim((new Process(['sh', '-c', 'command -v git']))->mustRun()->getOutput());
+    file_put_contents($this->root.'/bin/git', <<<SH
+        #!/bin/bash
+        args=()
+        for arg in "\$@"; do
+            [[ \$arg == git@* ]] && arg='{$this->root}/repo.git'
+            args+=("\$arg")
+        done
+        if [[ " \$* " == *" submodule update "* ]]; then
+            key=\$(printf '%s' "\$GIT_SSH_COMMAND" | sed -n 's/.* -i \\([^ ]*\\).*/\\1/p')
+            if [ ! -f "\$key" ]; then echo 'Permission denied (publickey).' >&2; exit 128; fi
+        fi
+        exec {$realGit} "\${args[@]}"
+        SH);
+    chmod($this->root.'/bin/git', 0755);
+
+    $application = Application::factory()->create([
+        'build_pack' => 'dockercompose',
+        'git_repository' => 'git@git.example.com:org/repo.git',
+        'git_branch' => 'main',
+        'git_commit_sha' => 'HEAD',
+        'base_directory' => '/',
+        'docker_compose_location' => '/infra/docker-compose.yml',
+        'private_key_id' => PrivateKey::factory()->create()->id,
+    ]);
+    $application->settings->update(['is_git_submodules_enabled' => true]);
+    $uuid = 'compose-load-'.bin2hex(random_bytes(4));
+
+    $commands = (new ReflectionMethod(Application::class, 'composeFileReadCommands'))->invoke($application->fresh(), $uuid);
+    // Keep the deploy key out of the real /root/.ssh of the test host.
+    $commands = $commands->map(fn (string $command) => str_replace('/root/.ssh', $this->root.'/ssh', $command));
+    if ($user !== 'root') {
+        $server = Mockery::mock(Server::class)->makePartial();
+        $server->shouldReceive('getAttribute')->with('user')->andReturn($user);
+        $server->shouldReceive('setAttribute')->andReturnSelf();
+        $commands = collect(parseCommandsByLineForSudo($commands, $server));
+    }
+
+    $process = new Process(['bash', '-se'], env: ['PATH' => $this->root.'/bin:'.getenv('PATH')] + $gitEnv);
+    $process->setInput($commands->implode("\n")."\n");
+    $process->run();
+    (new Process(['rm', '-rf', "/tmp/{$uuid}"]))->run();
+
+    expect($process->getErrorOutput())->not->toContain('Permission denied')
+        ->and($process->getExitCode())->toBe(0)
+        ->and($process->getOutput())->toBe($this->compose)
+        ->and(glob($this->root.'/ssh/*'))->toBe([]);
+})->with([
+    'root' => ['root'],
+    'non-root' => ['ubuntu'],
+]);
+
+test('a pinned commit loads the Compose file of that commit for every source type', function (string $sourceType, bool $shallow, string $user) {
+    Storage::fake('ssh-keys');
+    $pinnedCommit = $this->commit;
+    file_put_contents($this->root.'/src/apps/web/docker-compose.yml', "services:\n  api:\n    image: branch-head\n");
+    foreach ([
+        ['git', '-c', 'user.email=test@example.com', '-c', 'user.name=Test', 'commit', '-qam', 'branch head'],
+        ['git', 'push', '-q', $this->root.'/repo.git', 'main'],
+    ] as $command) {
+        (new Process($command, $this->root.'/src'))->mustRun();
+    }
+
+    // A git stub that sends every remote URL (HTTPS with a token, or SSH) to the local bare repository.
+    $realGit = trim((new Process(['sh', '-c', 'command -v git']))->mustRun()->getOutput());
+    file_put_contents($this->root.'/bin/git', <<<SH
+        #!/bin/bash
+        args=()
+        for arg in "\$@"; do
+            [[ \$arg == git@* || \$arg == https://* ]] && arg='{$this->root}/repo.git'
+            args+=("\$arg")
+        done
+        exec {$realGit} "\${args[@]}"
+        SH);
+    chmod($this->root.'/bin/git', 0755);
+
+    $privateKey = PrivateKey::factory()->create();
+    $application = Application::factory()->create([
+        'build_pack' => 'dockercompose',
+        'git_repository' => $sourceType === 'other' ? $this->root.'/repo.git' : 'git@git.example.com:org/repo.git',
+        'git_branch' => 'main',
+        'git_commit_sha' => $pinnedCommit,
+        'base_directory' => '/apps/web',
+        'docker_compose_location' => '/docker-compose.yml',
+        'private_key_id' => $sourceType === 'deploy key' ? $privateKey->id : null,
+    ]);
+    $application->settings->update(['is_git_shallow_clone_enabled' => $shallow]);
+    $application = $application->fresh();
+
+    if ($sourceType === 'GitHub App') {
+        $application->git_repository = 'org/repo';
+        $source = Mockery::mock(GithubApp::class)->makePartial();
+        $source->forceFill(['html_url' => 'https://github.example.test', 'is_public' => false]);
+        $source->shouldReceive('getMorphClass')->andReturn(GithubApp::class);
+        $source->shouldReceive('installationToken')->andReturn('ghs_token');
+        $application->setRelation('source', $source);
+    } elseif ($sourceType === 'Gitea token') {
+        $application->git_repository = 'org/repo';
+        $source = (new GiteaApp)->forceFill(['html_url' => 'https://gitea.example.test', 'api_url' => 'https://gitea.example.test/api/v1', 'is_public' => false]);
+        $source->access_token = 'gitea-token';
+        $source->refresh_token = 'gitea-refresh-token';
+        $source->expires_at = time() + 3600;
+        $application->setRelation('source', $source);
+    } elseif ($sourceType === 'GitLab SSH key') {
+        $source = (new GitlabApp)->forceFill(['html_url' => 'https://gitlab.example.test', 'team_id' => $privateKey->team_id, 'custom_port' => 22]);
+        $source->setRelation('privateKey', $privateKey);
+        $application->setRelation('source', $source);
+    }
+
+    $uuid = 'compose-load-'.bin2hex(random_bytes(4));
+    $commands = (new ReflectionMethod(Application::class, 'composeFileReadCommands'))->invoke($application, $uuid);
+    // Keep the deploy key out of the real /root/.ssh of the test host.
+    $commands = $commands->map(fn (string $command) => str_replace('/root/.ssh', $this->root.'/ssh', $command));
+    if ($user !== 'root') {
+        $server = Mockery::mock(Server::class)->makePartial();
+        $server->shouldReceive('getAttribute')->with('user')->andReturn($user);
+        $server->shouldReceive('setAttribute')->andReturnSelf();
+        $commands = collect(parseCommandsByLineForSudo($commands, $server));
+    }
+
+    $process = new Process(['bash', '-se'], env: ['PATH' => $this->root.'/bin:'.getenv('PATH')]);
+    $process->setInput($commands->implode("\n")."\n");
+    $process->run();
+    (new Process(['rm', '-rf', "/tmp/{$uuid}"]))->run();
+
+    expect($process->getErrorOutput())->not->toContain('syntax error')
+        ->and($process->getExitCode())->toBe(0)
+        ->and($process->getOutput())->toBe($this->compose);
+})->with([
+    'public URL' => ['other'],
+    'deploy key' => ['deploy key'],
+    'GitHub App' => ['GitHub App'],
+    'Gitea token' => ['Gitea token'],
+    'GitLab SSH key' => ['GitLab SSH key'],
+])->with([
+    'full clone' => [false],
+    'shallow clone' => [true],
+])->with([
+    'root' => ['root'],
+    'non-root' => ['ubuntu'],
+]);

tests/Feature/ComposeFileLoadErrorTest.php

@@ -35,7 +35,7 @@ beforeEach(function () {
 function fakeComposeLoadServer(?string $failingStep, string $errorOutput): void
 {
     Process::fake(function ($process) use ($failingStep, $errorOutput) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         if ($failingStep !== null && str_contains($command, $failingStep)) {
             return Process::result(errorOutput: $errorOutput, exitCode: 128);
         }

tests/Feature/ComposePreviewEnvironmentValuesTest.php

@@ -0,0 +1,109 @@
+<?php
+
+use App\Models\Application;
+use App\Models\ApplicationPreview;
+use App\Models\Environment;
+use App\Models\EnvironmentVariable;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\SharedEnvironmentVariable;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
+    $destination = StandaloneDocker::query()->where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    SharedEnvironmentVariable::create([
+        'key' => 'TEAM_SECRET', 'value' => 'team-secret-value', 'type' => 'team', 'team_id' => $team->id,
+    ]);
+
+    $this->application = Application::factory()->create([
+        'build_pack' => 'dockercompose',
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+        'docker_compose_raw' => <<<'YAML'
+services:
+  web:
+    image: nginx:alpine
+    environment:
+      CHANGED_FOR_PREVIEW: ''
+      SAME_AS_PRODUCTION: ''
+      FROM_TEAM: '{{team.TEAM_SECRET}}'
+YAML,
+    ]);
+
+    // Creating a production variable also creates a preview copy with the same value.
+    foreach (['CHANGED_FOR_PREVIEW', 'SAME_AS_PRODUCTION'] as $key) {
+        EnvironmentVariable::create([
+            'key' => $key,
+            'value' => 'production-value',
+            'resourceable_type' => Application::class,
+            'resourceable_id' => $this->application->id,
+            'is_preview' => false,
+        ]);
+    }
+    $this->application->environment_variables_preview()->where('key', 'CHANGED_FOR_PREVIEW')->firstOrFail()
+        ->update(['value' => 'preview-value']);
+});
+
+function composePreviewEnvironmentValues(Application $application, int $pullRequestId = 0): array
+{
+    $previewId = null;
+    if ($pullRequestId !== 0) {
+        $previewId = ApplicationPreview::create([
+            'application_id' => $application->id,
+            'pull_request_id' => $pullRequestId,
+            'pull_request_html_url' => "https://github.com/example/repository/pull/{$pullRequestId}",
+        ])->id;
+    }
+
+    $serviceName = $pullRequestId === 0 ? 'web' : "web-pr-{$pullRequestId}";
+    $environment = collect(data_get($application->fresh()->parse(pull_request_id: $pullRequestId, preview_id: $previewId)->toArray(), "services.{$serviceName}.environment"));
+
+    return $environment->only(['CHANGED_FOR_PREVIEW', 'SAME_AS_PRODUCTION', 'FROM_TEAM'])->all();
+}
+
+it('fills empty compose values of a production deployment with the production variables', function () {
+    expect(composePreviewEnvironmentValues($this->application))->toBe([
+        'CHANGED_FOR_PREVIEW' => 'production-value',
+        'SAME_AS_PRODUCTION' => 'production-value',
+        'FROM_TEAM' => 'team-secret-value',
+    ]);
+});
+
+it('fills empty compose values of a pull request deployment with the preview variables', function () {
+    expect(composePreviewEnvironmentValues($this->application, 7))->toBe([
+        'CHANGED_FOR_PREVIEW' => 'preview-value',
+        'SAME_AS_PRODUCTION' => 'production-value',
+        'FROM_TEAM' => 'team-secret-value',
+    ]);
+});
+
+it('gives a public pull request deployment no production values or shared variables from the compose file', function () {
+    $this->application->settings->update(['is_pr_deployments_public_enabled' => true]);
+
+    expect(composePreviewEnvironmentValues($this->application, 7))->toBe([
+        'CHANGED_FOR_PREVIEW' => 'preview-value',
+        'SAME_AS_PRODUCTION' => '',
+        'FROM_TEAM' => '{{team.TEAM_SECRET}}',
+    ]);
+});
+
+it('resolves shared variables in a preview value of a public pull request deployment', function () {
+    $this->application->settings->update(['is_pr_deployments_public_enabled' => true]);
+    $this->application->environment_variables_preview()->where('key', 'CHANGED_FOR_PREVIEW')->firstOrFail()
+        ->update(['value' => '{{team.TEAM_SECRET}}']);
+
+    expect(composePreviewEnvironmentValues($this->application, 7)['CHANGED_FOR_PREVIEW'])->toBe('team-secret-value');
+});

tests/Feature/ComposePreviewMagicVariablesTest.php

@@ -0,0 +1,217 @@
+<?php
+
+use App\Models\Application;
+use App\Models\ApplicationPreview;
+use App\Models\Environment;
+use App\Models\EnvironmentVariable;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
+    $destination = StandaloneDocker::query()->where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    $this->application = Application::factory()->create([
+        'build_pack' => 'dockercompose',
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+        'docker_compose_raw' => <<<'YAML'
+services:
+  db:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: ${SERVICE_USER_DB}
+      POSTGRES_PASSWORD: ${SERVICE_PASSWORD_DB}
+YAML,
+    ]);
+});
+
+function composePreviewMagicParse(Application $application, int $pullRequestId = 0): void
+{
+    $previewId = null;
+    if ($pullRequestId !== 0) {
+        $previewId = ApplicationPreview::query()->firstOrCreate([
+            'application_id' => $application->id,
+            'pull_request_id' => $pullRequestId,
+        ], [
+            'pull_request_html_url' => "https://github.com/example/repository/pull/{$pullRequestId}",
+        ])->id;
+    }
+
+    $application->fresh()->parse(pull_request_id: $pullRequestId, preview_id: $previewId);
+}
+
+/**
+ * @return array{production: array<string, ?string>, preview: array<string, ?string>}
+ */
+function composePreviewMagicValues(Application $application): array
+{
+    $keys = ['SERVICE_PASSWORD_DB', 'SERVICE_USER_DB'];
+
+    return [
+        'production' => $application->environment_variables()->whereIn('key', $keys)->pluck('value', 'key')->sortKeys()->all(),
+        'preview' => $application->environment_variables_preview()->whereIn('key', $keys)->pluck('value', 'key')->sortKeys()->all(),
+    ];
+}
+
+it('generates the magic variables of a first pull request parse only in the preview variables', function () {
+    composePreviewMagicParse($this->application, 7);
+
+    $values = composePreviewMagicValues($this->application);
+    expect($values['production'])->toBe([])
+        ->and(array_keys($values['preview']))->toBe(['SERVICE_PASSWORD_DB', 'SERVICE_USER_DB'])
+        ->and($values['preview']['SERVICE_PASSWORD_DB'])->not->toBeEmpty();
+
+    // The production deployment generates its own values and keeps the preview values.
+    composePreviewMagicParse($this->application);
+
+    $afterProduction = composePreviewMagicValues($this->application);
+    expect($afterProduction['preview'])->toBe($values['preview'])
+        ->and($afterProduction['production']['SERVICE_PASSWORD_DB'])->not->toBeEmpty()
+        ->and($afterProduction['production']['SERVICE_PASSWORD_DB'])->not->toBe($values['preview']['SERVICE_PASSWORD_DB']);
+});
+
+it('does not copy a generated production magic secret to the preview variables', function (string $key) {
+    EnvironmentVariable::create([
+        'key' => $key,
+        'value' => 'production-secret',
+        'is_preview' => false,
+        'resourceable_type' => Application::class,
+        'resourceable_id' => $this->application->id,
+    ]);
+
+    expect($this->application->environment_variables_preview()->where('key', $key)->exists())->toBeFalse();
+})->with([
+    'SERVICE_PASSWORD_DB',
+    'SERVICE_PASSWORD_64_DB',
+    'SERVICE_PASSWORDWITHSYMBOLS_DB',
+    'SERVICE_USER_DB',
+    'SERVICE_LOWERCASEUSER_DB',
+    'SERVICE_BASE64_DB',
+    'SERVICE_BASE64_64_DB',
+    'SERVICE_BASE64_128_DB',
+    'SERVICE_REALBASE64_DB',
+    'SERVICE_HEX_32_DB',
+]);
+
+it('still copies a normal or service url production variable to the preview variables', function (string $key) {
+    EnvironmentVariable::create([
+        'key' => $key,
+        'value' => 'production-value',
+        'is_preview' => false,
+        'resourceable_type' => Application::class,
+        'resourceable_id' => $this->application->id,
+    ]);
+
+    expect($this->application->environment_variables_preview()->where('key', $key)->value('value'))->toBe('production-value');
+})->with(['DATABASE_HOST', 'SERVICE_URL_DB', 'SERVICE_FQDN_DB', 'SERVICE_PASSWORD_DB_MAIN']);
+
+it('gives a private pull request its own magic values instead of the production values', function () {
+    composePreviewMagicParse($this->application);
+    $production = composePreviewMagicValues($this->application)['production'];
+
+    composePreviewMagicParse($this->application, 7);
+
+    $values = composePreviewMagicValues($this->application);
+    expect($values['production'])->toBe($production)
+        ->and(array_keys($values['preview']))->toBe(['SERVICE_PASSWORD_DB', 'SERVICE_USER_DB'])
+        ->and($values['preview']['SERVICE_PASSWORD_DB'])->not->toBeEmpty()
+        ->and($values['preview']['SERVICE_PASSWORD_DB'])->not->toBe($production['SERVICE_PASSWORD_DB'])
+        ->and($values['preview']['SERVICE_USER_DB'])->not->toBe($production['SERVICE_USER_DB']);
+
+    // The next pull request parse keeps the preview values, so the preview database keeps its password.
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application))->toBe($values);
+});
+
+it('keeps an existing preview copy of a production magic value for a private pull request', function () {
+    composePreviewMagicParse($this->application);
+    $production = composePreviewMagicValues($this->application)['production'];
+    // A preview copy that an earlier version created; existing preview databases use this password.
+    $this->application->environment_variables_preview()->create([
+        'key' => 'SERVICE_PASSWORD_DB',
+        'value' => $production['SERVICE_PASSWORD_DB'],
+        'is_preview' => true,
+    ]);
+
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application)['preview']['SERVICE_PASSWORD_DB'])->toBe($production['SERVICE_PASSWORD_DB']);
+});
+
+it('keeps a preview magic variable that the user created', function () {
+    EnvironmentVariable::create([
+        'key' => 'SERVICE_PASSWORD_DB',
+        'value' => 'user-preview-password',
+        'is_preview' => true,
+        'resourceable_type' => Application::class,
+        'resourceable_id' => $this->application->id,
+    ]);
+
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application)['preview']['SERVICE_PASSWORD_DB'])->toBe('user-preview-password');
+});
+
+it('gives a public pull request its own magic values instead of the production values', function () {
+    composePreviewMagicParse($this->application);
+    $production = composePreviewMagicValues($this->application)['production'];
+    $this->application->settings->update(['is_pr_deployments_public_enabled' => true]);
+
+    composePreviewMagicParse($this->application, 7);
+
+    $values = composePreviewMagicValues($this->application);
+    expect($values['production'])->toBe($production)
+        ->and($values['preview']['SERVICE_PASSWORD_DB'])->not->toBeEmpty()
+        ->and($values['preview']['SERVICE_PASSWORD_DB'])->not->toBe($production['SERVICE_PASSWORD_DB'])
+        ->and($values['preview']['SERVICE_USER_DB'])->not->toBe($production['SERVICE_USER_DB']);
+
+    // The next pull request parse keeps the preview values, so the preview database keeps its password.
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application))->toBe($values);
+});
+
+it('keeps a changed preview magic value of a public pull request', function () {
+    composePreviewMagicParse($this->application);
+    $this->application->settings->update(['is_pr_deployments_public_enabled' => true]);
+    $this->application->environment_variables_preview()->create([
+        'key' => 'SERVICE_PASSWORD_DB',
+        'value' => 'preview-password',
+        'is_preview' => true,
+    ]);
+
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application)['preview']['SERVICE_PASSWORD_DB'])->toBe('preview-password');
+});
+
+it('replaces an existing preview copy of a production magic value for a public pull request', function () {
+    composePreviewMagicParse($this->application);
+    $production = composePreviewMagicValues($this->application)['production'];
+    $this->application->environment_variables_preview()->create([
+        'key' => 'SERVICE_PASSWORD_DB',
+        'value' => $production['SERVICE_PASSWORD_DB'],
+        'is_preview' => true,
+    ]);
+    $this->application->settings->update(['is_pr_deployments_public_enabled' => true]);
+
+    composePreviewMagicParse($this->application, 7);
+
+    expect(composePreviewMagicValues($this->application)['preview']['SERVICE_PASSWORD_DB'])
+        ->not->toBeEmpty()
+        ->not->toBe($production['SERVICE_PASSWORD_DB']);
+});

tests/Feature/ComposeStaleVolumeTest.php

@@ -95,7 +95,7 @@ it('deletes the entry of a volume that the Compose file no longer declares', fun
     $name = staleVolume('third')->name;
     $commands = [];
     Process::fake(function ($process) use (&$commands) {
-        $commands[] = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $commands[] = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
 
         return Process::result(output: '');
     });

tests/Feature/ComposeVolumeDriverOptionsTest.php

@@ -527,7 +527,7 @@ describe('storage page', function () {
         $volume = LocalPersistentVolume::where('name', $name)->sole();
         $commands = [];
         Process::fake(function ($process) use (&$commands) {
-            $commands[] = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+            $commands[] = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
 
             return Process::result(output: '');
         });

tests/Feature/ContainerFilesystemServiceTest.php

@@ -157,9 +157,9 @@ it('lists through a locked-down helper container that runs as the container proc
     $server->update(['user' => 'root']);
     $commands = [];
     Process::fake(function ($process) use (&$commands) {
-        $commands[] = $process->command;
+        $commands[] = remoteProcessScript($process);
 
-        return str_contains($process->command, '/status')
+        return str_contains(remoteProcessScript($process), '/status')
             ? Process::result(output: fsProcessStatus('65532', '0', '0 1000', '0000000000000000'))
             : Process::result(output: "regular file\t9\t1\t644\troot\troot\t./users.yml");
     });
@@ -189,7 +189,7 @@ it('wraps the helper command in one sudo shell for non-root servers', function (
     $server->update(['user' => 'deploy']);
     $commands = [];
     Process::fake(function ($process) use (&$commands) {
-        $commands[] = $process->command;
+        $commands[] = remoteProcessScript($process);
 
         return count($commands) === 1
             ? Process::result(output: fsProcessStatus('0', '0', '0', '00000000a80425fb'))
@@ -264,7 +264,7 @@ it('writes content near or above the shell argument limit via docker cp', functi
 
     (new ContainerFilesystemService($server, 'app-123'))->write('/app/big.txt', str_repeat('a', $size));
 
-    Process::assertRan(fn ($process) => str_contains($process->command, 'docker cp'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'docker cp'));
 })->with([96 * 1024, 200_000]);
 
 it('base64-encodes content in the write command', function () {
@@ -363,8 +363,8 @@ it('uploads by scp-ing to the server then docker cp into the container', functio
 
     (new ContainerFilesystemService($server, 'app-123'))->upload($local, '/app/plugins/x.jar');
 
-    Process::assertRan(fn ($process) => str_contains($process->command, 'docker cp')
-        && str_contains($process->command, 'app-123'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'docker cp')
+        && str_contains(remoteProcessScript($process), 'app-123'));
     @unlink($local);
 });
 
@@ -379,7 +379,7 @@ it('executes read guards for real file bytes through the non-root sudo parser',
     $path = tempnam(sys_get_temp_dir(), 'fs-read-');
     file_put_contents($path, $content);
     Process::fake(function ($process) {
-        $lines = explode("\n", $process->command);
+        $lines = explode("\n", remoteProcessScript($process));
         $body = implode("\n", array_slice($lines, 1, -1));
         $local = new Symfony\Component\Process\Process([
             'bash', '-c',
@@ -415,7 +415,7 @@ it('rejects a symlink to an oversized file before reading its content', function
     fclose($handle);
     symlink($path, $link);
     Process::fake(function ($process) {
-        $body = implode("\n", array_slice(explode("\n", $process->command), 1, -1));
+        $body = implode("\n", array_slice(explode("\n", remoteProcessScript($process)), 1, -1));
         $local = new Symfony\Component\Process\Process(['bash', '-c', 'sudo() { "$@"; }; docker() { shift 2; "$@"; }; export -f sudo docker;'."\n".$body]);
         $local->run();
 
@@ -461,13 +461,13 @@ it('streams the remote command output to the writer in chunks', function () {
     $server = fsServer();
     $svc = new class($server, 'app-123') extends ContainerFilesystemService
     {
-        public ?string $remoteCommand = null;
+        public string $stdinFile;
 
-        protected function sshCommand(string $command): string
+        protected function sshCommand(): string
         {
-            $this->remoteCommand = $command;
+            $this->stdinFile = tempnam(sys_get_temp_dir(), 'fs-stream-');
 
-            return 'head -c 200000 /dev/zero';
+            return 'cat > '.escapeshellarg($this->stdinFile).'; head -c 200000 /dev/zero';
         }
     };
     $received = '';
@@ -478,7 +478,9 @@ it('streams the remote command output to the writer in chunks', function () {
         $chunks++;
     });
 
-    expect($svc->remoteCommand)->toEndWith("docker exec 'app-123' cat -- '/app/a.bin'");
+    $remoteCommand = file_get_contents($svc->stdinFile);
+    unlink($svc->stdinFile);
+    expect($remoteCommand)->toEndWith("docker exec 'app-123' cat -- '/app/a.bin'\n");
     expect(strlen($received))->toBe(200000);
     expect($chunks)->toBeGreaterThan(1);
 });

tests/Feature/ContainerNameShellQuotingTest.php

@@ -85,12 +85,12 @@ function quotingUseNonRootUser(): void
 
 function quotingAssertCommandRan(string $command): void
 {
-    Process::assertRan(fn ($process) => str_contains($process->command, $command));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), $command));
 }
 
 function quotingAssertNoUnquotedName(): void
 {
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, ' app;touch'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), ' app;touch'));
 }
 
 test('stopping a service application quotes its container name', function (bool $nonRoot, string $sudo) {
@@ -155,7 +155,7 @@ test('a scheduled task in a service quotes the container name', function (bool $
 function quotingFakeApplicationContainer(string $name): void
 {
     Process::fake(function ($process) use ($name) {
-        if (str_contains($process->command, 'docker ps -a --filter')) {
+        if (str_contains(remoteProcessScript($process), 'docker ps -a --filter')) {
             return Process::result(output: json_encode(['Names' => $name, 'Labels' => 'coolify.applicationId=1', 'State' => 'running']));
         }
 
@@ -183,7 +183,7 @@ test('stopping an application quotes the container names from docker ps', functi
     StopApplication::run($application, dockerCleanup: false);
 
     quotingAssertCommandRan("{$sudo}docker stop --timeout=");
-    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', $process->command) === 1);
+    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', remoteProcessScript($process)) === 1);
     quotingAssertCommandRan("\n{$sudo}docker rm -f {$quoted}\n");
     quotingAssertNoUnquotedName();
 })->with(['root' => [false, ''], 'non-root' => [true, 'sudo ']]);
@@ -196,7 +196,7 @@ test('stopping an application preview quotes the container names from docker ps'
         'pull_request_html_url' => 'https://example.com/pull/1',
     ]);
     Process::fake(function ($process) {
-        if (str_contains($process->command, 'docker ps -a --filter')) {
+        if (str_contains(remoteProcessScript($process), 'docker ps -a --filter')) {
             return Process::result(output: json_encode(['Names' => QUOTING_HOSTILE_NAME, 'Labels' => 'coolify.applicationId=1,coolify.pullRequestId=1']));
         }
 
@@ -206,7 +206,7 @@ test('stopping an application preview quotes the container names from docker ps'
 
     StopApplicationPreview::run($preview);
 
-    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', $process->command) === 1);
+    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', remoteProcessScript($process)) === 1);
     quotingAssertCommandRan("\ndocker rm -f {$quoted}\n");
     quotingAssertNoUnquotedName();
 });
@@ -218,7 +218,7 @@ test('stopping an application on one server quotes the container names from dock
 
     StopApplicationOneServer::run($application, $this->server);
 
-    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', $process->command) === 1);
+    Process::assertRan(fn ($process) => preg_match('/docker stop --timeout=\d+ '.preg_quote($quoted, '/').'\n/', remoteProcessScript($process)) === 1);
     quotingAssertNoUnquotedName();
 });
 

tests/Feature/ContainerOwnerUuidLabelsTest.php

@@ -87,7 +87,7 @@ describe('container lookups for services and databases', function () {
         ]);
         $commands = collect();
         Process::fake(function ($process) use ($commands) {
-            $commands->push($process->command);
+            $commands->push(remoteProcessScript($process));
 
             return Process::result(output: '');
         });

tests/Feature/ContentFilesBeforeComposeUpTest.php

@@ -97,7 +97,7 @@ function fakeContentFilesServer(array $states): void
     ContentFilesUpRecorder::$commands = [];
 
     Process::fake(function ($process) use ($states) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         ContentFilesUpRecorder::$commands[] = ['ssh', $command];
 
         if (str_contains($command, 'empty-directory')) {
@@ -132,9 +132,9 @@ function contentFilesStateChecks(): array
 
 function contentFilesAssertNoWrite(): void
 {
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'base64 -d'));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rmdir'));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rm -fr'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'base64 -d'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rmdir'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rm -fr'));
 }
 
 beforeEach(function () {
@@ -256,9 +256,9 @@ test('a Compose deployment checks all content files with one server command', fu
         ->toContain(escapeshellarg($second->fs_path))
         ->toContain(escapeshellarg($third->fs_path))
         ->and($this->logEntries)->toContain(['Writing 1 missing configuration file.', 'stdout']);
-    Process::assertRan(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($second->fs_path)));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($first->fs_path)));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($third->fs_path)));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($second->fs_path)));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($first->fs_path)));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($third->fs_path)));
 });
 
 test('a Compose deployment does not change a content file that exists on the server', function () {
@@ -286,7 +286,7 @@ test('a Compose deployment replaces an empty directory with the content file', f
         ->and($write)->toBeLessThan(ContentFilesUpRecorder::indexOf('deployment', ' up -d'))
         ->and(strpos($writeCommand, "rmdir {$escapedPath}"))->toBeLessThan(strpos($writeCommand, "tee {$escapedPath}"))
         ->and($this->logEntries)->toContain(['Writing 1 missing configuration file.', 'stdout']);
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rm -fr'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rm -fr'));
 });
 
 test('a Compose deployment keeps a directory that is not empty and shows a warning', function () {
@@ -420,8 +420,8 @@ test('the content file check and write are safe for non-root servers', function
     $escapedPath = escapeshellarg($volume->fs_path);
     expect(contentFilesStateChecks())->not->toBeEmpty()
         ->and(contentFilesStateChecks()[0])->toContain("sudo bash -c 'sh -c ");
-    Process::assertRan(fn ($process) => str_contains($process->command, "sudo rmdir {$escapedPath}")
-        && str_contains($process->command, "| sudo tee {$escapedPath}"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "sudo rmdir {$escapedPath}")
+        && str_contains(remoteProcessScript($process), "| sudo tee {$escapedPath}"));
 });
 
 test('starting a service writes missing content files before docker compose up', function () {
@@ -446,10 +446,10 @@ test('starting a service writes missing content files before docker compose up',
         ->and($command)->toContain("echo 'Warning: A directory that is not empty is at {$workerConf->fs_path} on the server.")
         ->and(strpos($command, 'Writing 1 missing configuration file.'))->toBeLessThan(strpos($command, ' up -d'))
         ->and($command)->not->toContain(base64_encode('listen 8080;'));
-    Process::assertRan(fn ($process) => str_contains($process->command, 'base64 -d | tee '.escapeshellarg($appConf->fs_path))
-        && str_contains($process->command, base64_encode('listen 8080;')));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($workerConf->fs_path)));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rm -fr'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'base64 -d | tee '.escapeshellarg($appConf->fs_path))
+        && str_contains(remoteProcessScript($process), base64_encode('listen 8080;')));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($workerConf->fs_path)));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rm -fr'));
 });
 
 test('deploying one service application writes only its missing content files', function () {
@@ -472,8 +472,8 @@ test('deploying one service application writes only its missing content files',
     expect(contentFilesStateChecks())->toHaveCount(1)
         ->and(contentFilesStateChecks()[0])->not->toContain(escapeshellarg($workerConf->fs_path))
         ->and(strpos($command, 'Writing 1 missing configuration file.'))->toBeLessThan(strpos($command, ' up -d'));
-    Process::assertRan(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($appConf->fs_path)));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'tee '.escapeshellarg($workerConf->fs_path)));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($appConf->fs_path)));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'tee '.escapeshellarg($workerConf->fs_path)));
 });
 
 test('starting a service without content files does not run the file check', function () {
@@ -523,8 +523,8 @@ test('the storage save job writes the content again when the same file exists',
 
     $writes = array_filter(ContentFilesUpRecorder::ssh(), fn (string $command) => str_contains($command, 'base64 -d | tee '.escapeshellarg($volume->fs_path)));
     expect($writes)->toHaveCount(2);
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rm -fr'));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rmdir'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rm -fr'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rmdir'));
 });
 
 test('the storage save job replaces an empty directory with the content file', function () {
@@ -534,11 +534,11 @@ test('the storage save job replaces an empty directory with the content file', f
     (new ServerStorageSaveJob($volume))->handle();
 
     $escapedPath = escapeshellarg($volume->fs_path);
-    Process::assertRan(fn ($process) => str_contains($process->command, "rmdir {$escapedPath}")
-        && str_contains($process->command, base64_encode(CONTENT_FILES_UP_CONTENT))
-        && strpos($process->command, "rmdir {$escapedPath}") < strpos($process->command, "tee {$escapedPath}"));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, 'rm -fr'));
-    Process::assertDidntRun(fn ($process) => str_contains($process->command, "touch {$escapedPath}"));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "rmdir {$escapedPath}")
+        && str_contains(remoteProcessScript($process), base64_encode(CONTENT_FILES_UP_CONTENT))
+        && strpos(remoteProcessScript($process), "rmdir {$escapedPath}") < strpos(remoteProcessScript($process), "tee {$escapedPath}"));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'rm -fr'));
+    Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), "touch {$escapedPath}"));
 });
 
 test('the storage save job does not delete a directory that is not empty', function () {
@@ -573,8 +573,8 @@ test('the server file sync replaces an empty directory with the content file', f
     expect($volume->fresh())
         ->is_directory->toBeFalsy()
         ->content->toBe(CONTENT_FILES_UP_CONTENT);
-    Process::assertRan(fn ($process) => str_contains($process->command, 'rmdir'));
-    Process::assertRan(fn ($process) => str_contains($process->command, 'base64 -d'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'rmdir'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'base64 -d'));
 });
 
 test('the server file sync still marks a storage without content as a directory', function () {

tests/Feature/CreateScheduledBackupValidationTest.php

@@ -226,7 +226,7 @@ it('backs up sqlite files added to the database after the backup was scheduled',
 
     $commands = collect();
     Process::fake(function ($process) use ($commands) {
-        $commands->push(is_array($process->command) ? implode(' ', $process->command) : $process->command);
+        $commands->push(is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process));
 
         return Process::result(output: '');
     });

tests/Feature/DashboardServerMetricsChartTest.php

@@ -102,5 +102,5 @@ it('loads dashboard metrics over SSH for a reachable server', function () {
             ],
         );
 
-    Process::assertRan(fn ($process) => str_contains($process->command, 'coolify-sentinel'));
+    Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'coolify-sentinel'));
 });

tests/Feature/DatabaseBackupInstanceDatabaseTest.php

@@ -53,7 +53,7 @@ it('backs up a user database named coolify-db as an ordinary database', function
     ])->save();
     $commands = collect();
     Process::fake(function ($process) use ($commands) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands->push($command);
 
         return Process::result(output: str_contains($command, 'du -b') ? '128' : '');

tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php

@@ -87,7 +87,7 @@ function runMultiS3BackupJob(array $failingStorages): array
     $backup = createMultiS3Backup();
     $commands = collect();
     Process::fake(function ($process) use ($commands) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands->push($command);
 
         return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
@@ -321,15 +321,27 @@ describe('backup job', function () {
         $sshDisk = Storage::disk('ssh-keys');
         Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
         Storage::shouldReceive('build')->andReturn($disk);
-        Process::fake(fn ($process) => Process::result(output: str_contains($process->command, 'du -b') ? '128' : ''));
+        $commands = collect();
+        Process::fake(function ($process) use ($commands) {
+            $commands->push(remoteProcessScript($process));
+
+            return Process::result(output: str_contains(remoteProcessScript($process), 'du -b') ? '128' : '');
+        });
 
         (new DatabaseBackupJob($backup))->handle();
 
         expect(ScheduledDatabaseBackupExecution::query()->sole()->s3_uploaded)->toBeTrue();
-        Process::assertRan(fn ($process) => str_contains($process->command, "-e 'MC_REGION=ru-central-1'")
-            && str_contains($process->command, 'temporary/first-bucket/'));
-        Process::assertRan(fn ($process) => str_contains($process->command, "-e 'MC_REGION=us-east-1'")
-            && str_contains($process->command, 'temporary/second-bucket/'));
+        $regionOfBucket = [];
+        $region = null;
+        foreach ($commands as $command) {
+            if (preg_match("/-e 'MC_REGION=([^']+)'/", $command, $match)) {
+                $region = $match[1];
+            }
+            if (preg_match("#mc cp .*'temporary/([^/']+)/#", $command, $match)) {
+                $regionOfBucket[$match[1]] = $region;
+            }
+        }
+        expect($regionOfBucket)->toBe(['first-bucket' => 'ru-central-1', 'second-bucket' => 'us-east-1']);
     });
 
     it('keeps the local file and names the failed destination when one upload fails', function () {
@@ -442,7 +454,7 @@ it('requires a live S3 copy for local retention only when local backups are disa
     }
     $commands = collect();
     Process::fake(function ($process) use ($commands) {
-        $commands->push(is_array($process->command) ? implode(' ', $process->command) : $process->command);
+        $commands->push(is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process));
 
         return Process::result();
     });

tests/Feature/DatabaseBackupPerDatabaseStateTest.php

@@ -54,7 +54,7 @@ it('keeps the successful execution of an earlier database when creating the next
     ])->save();
     $commands = collect();
     Process::fake(function ($process) use ($commands) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         $commands->push($command);
 
         return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
@@ -95,7 +95,7 @@ it('finishes the execution of every database in a run with several databases', f
         'team_id' => $team->id,
     ])->save();
     Process::fake(function ($process) {
-        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $command = is_array(remoteProcessScript($process)) ? implode(' ', remoteProcessScript($process)) : remoteProcessScript($process);
         if (str_contains($command, 'pg_dump') && str_contains($command, 'second')) {
             return Process::result(errorOutput: 'pg_dump: database "second" does not exist', exitCode: 1);
         }