mirrors/coolify
0
0 Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-10-28 17:44:36 +08:00
Code Activity

feat(scheduler): dispatch scheduled jobs by their stored next run time

Browse Source 
The scheduled job manager checked every enabled schedule each minute and
evaluated its cron expression in PHP. On Coolify Cloud one run took more than
60 seconds, so runs overlapped and minutes were skipped.

Each schedule now stores its next due time:
- next_run_at on scheduled database backups, scheduled tasks, and volume
  backups, and docker_cleanup_next_run_at on server settings (UTC, indexed)
- model hooks calculate it when a schedule is created or its frequency or
  enabled flag changes; a server timezone change or a move to another server
  calculates it again for the affected schedules

The `scheduled:dispatch` command runs the manager every minute from the
scheduler, not from a queue. It selects only due rows and claims each
occurrence with an atomic update of next_run_at, together with its delivery
record, so parallel runs on several nodes never dispatch it twice.
- database and volume backups run once when they are late; tasks and Docker
  cleanups more than 10 minutes late are logged as missed
- a local time that occurs two times when the clocks go back runs once
- the migration continues each schedule after its last recorded occurrence and
  drops scheduled_job_states

Show the next run, in the server timezone, on the database backup, scheduled
task, volume backup, and Docker cleanup pages, or "Invalid frequency" when the
frequency cannot be calculated. scheduled:diagnostics now lists overdue
schedules and open deliveries.

Remove workarounds that are no longer needed: the queue overlap lock and its
stale-lock repair, the catch-up window, and debug-only logs.
This commit is contained in:
Andras Bacsai committed 2026-09-29 10:07:23 +02:00
1 parent 2928a27966
commit 452259aacc
35 files changed
+1423 -1521
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.ai/lessons.md
+2
View File
@@ -56,6 +56,8 @@
## Make distributed schedules durable
- Use the database as the correctness source for dynamic cron occurrences shared by multiple scheduler and Horizon nodes; Redis locks are load controls, not a durable execution ledger.
- Give each schedule occurrence a unique database identity and make queue consumers claim it atomically before external work.
- Store the next due time on each schedule and select only due rows; claim a run with `UPDATE ... WHERE next_run_at = :old`. Do not evaluate every cron expression each minute: on a remote database the per-row queries alone exceed the one-minute budget.
- Run a per-minute dispatcher from the scheduler process, not as a queued job, so queue backlog cannot delay it.
- Keep pending occurrences recoverable across publisher interruptions, and define an explicit bounded policy for late or offline schedules.
- Horizon workers are long-lived: flush every static or `once()` cache (for example `Server::flushIdentityMap()`) in `Queue::before`, or later jobs decide with stale state.
app/Actions/Shared/MigrateResourceToDestination.php
+2
View File
@@ -23,6 +23,7 @@ use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\StandaloneSqlite;
use App\Models\SwarmDocker;
use App\Services\ScheduleNextRunRecalculator;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Illuminate\Validation\ValidationException;
@@ -151,6 +152,7 @@ class MigrateResourceToDestination
}
$resource->fill($payload)->save();
app(ScheduleNextRunRecalculator::class)->forResource($resource);
if ($resource instanceof Application) {
$resource->additional_networks()->detach();
app/Console/Commands/DispatchScheduledJobs.php
+20
View File
@@ -0,0 +1,20 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ScheduledJobManager;
use Illuminate\Console\Command;
class DispatchScheduledJobs extends Command
{
protected $signature = 'scheduled:dispatch';
protected $description = 'Dispatch the due scheduled backups, tasks, volume backups, and Docker cleanups';
public function handle(ScheduledJobManager $manager): int
{
$manager->handle();
return self::SUCCESS;
}
}
app/Console/Commands/ScheduledJobDiagnostics.php
+44 -231
View File
@@ -2,254 +2,67 @@
namespace App\Console\Commands;
use App\Models\DockerCleanupExecution;
use App\Jobs\ScheduledJobManager;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledTask;
use App\Models\Server;
use App\Models\Team;
use App\Models\ScheduledVolumeBackup;
use App\Models\ServerSetting;
use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
class ScheduledJobDiagnostics extends Command
{
protected $signature = 'scheduled:diagnostics
{--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}
{--server= : Filter by server ID}';
protected $signature = 'scheduled:diagnostics {--all : List every schedule, not only the overdue ones and the ones without a next run}';
protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';
protected $description = 'Show the scheduler heartbeat, overdue schedules, schedules without a next run, and open deliveries';
public function handle(): int
{
$type = $this->option('type');
$serverFilter = $this->option('server');
$heartbeat = Cache::get('scheduled-job-manager:heartbeat');
$heartbeat
? $this->info("Scheduler heartbeat: {$heartbeat} (".Carbon::parse($heartbeat)->diffForHumans().')')
: $this->error('Scheduler heartbeat: missing. The scheduled:dispatch command does not run.');
$this->newLine();
$this->outputHeartbeat();
$overdueBefore = now()->subMinutes(ScheduledJobManager::LATE_RUN_WINDOW_MINUTES);
$schedules = [
'Database backups' => [ScheduledDatabaseBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],
'Scheduled tasks' => [ScheduledTask::query()->where('enabled', true), 'next_run_at', 'frequency'],
'Volume backups' => [ScheduledVolumeBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],
'Docker cleanups' => [ServerSetting::query(), 'docker_cleanup_next_run_at', 'docker_cleanup_frequency'],
];
if (in_array($type, ['all', 'docker-cleanup'])) {
$this->inspectDockerCleanups($serverFilter);
foreach ($schedules as $label => [$query, $column, $frequencyColumn]) {
/** @var Builder $query */
$problems = (clone $query)->where(fn (Builder $query) => $query->whereNull($column)->orWhere($column, '<', $overdueBefore));
$this->info(sprintf('=== %s: %d enabled, %d overdue or without a next run ===', $label, (clone $query)->count(), (clone $problems)->count()));
$rows = ($this->option('all') ? $query : $problems)
->orderBy($column)
->limit(100)
->get()
->map(fn ($schedule) => [$schedule->getKey(), $schedule->getRawOriginal($frequencyColumn), $schedule->{$column}?->toIso8601String() ?? (next_cron_run_at((string) $schedule->getRawOriginal($frequencyColumn), null, now()) ? 'not calculated' : 'invalid frequency')]);
if ($rows->isNotEmpty()) {
$this->table(['ID', 'Frequency', 'Next run (UTC)'], $rows);
}
$this->newLine();
}
if (in_array($type, ['all', 'backups'])) {
$this->inspectBackups();
}
if (in_array($type, ['all', 'tasks'])) {
$this->inspectTasks();
}
if (in_array($type, ['all', 'server-jobs'])) {
$this->inspectServerJobs($serverFilter);
}
$this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ===');
$this->table(
['Job type', 'Status', 'Count', 'Oldest due (UTC)'],
ScheduledJobDelivery::query()
->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed'])
->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest')
->groupBy('job_type', 'status')
->get()
->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest]),
);
return self::SUCCESS;
}
private function outputHeartbeat(): void
{
$heartbeat = Cache::get('scheduled-job-manager:heartbeat');
if ($heartbeat) {
$age = Carbon::parse($heartbeat)->diffForHumans();
$this->info("Scheduler heartbeat: {$heartbeat} ({$age})");
} else {
$this->error('Scheduler heartbeat: MISSING — ScheduledJobManager may not be running');
}
$this->newLine();
}
private function inspectDockerCleanups(?string $serverFilter): void
{
$this->info('=== Docker Cleanup Jobs ===');
$servers = $this->getServers($serverFilter);
$rows = [];
foreach ($servers as $server) {
$frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
if (isset(VALID_CRON_STRINGS[$frequency])) {
$frequency = VALID_CRON_STRINGS[$frequency];
}
$dedupKey = "docker-cleanup:{$server->id}";
$cacheValue = Cache::get($dedupKey);
$timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$lastExecution = DockerCleanupExecution::where('server_id', $server->id)
->latest()
->first();
$rows[] = [
$server->id,
$server->name,
$timezone,
$frequency,
$dedupKey,
$cacheValue ?? '<missing>',
$wouldFire ? 'YES' : 'no',
$lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',
];
}
$this->table(
['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],
$rows
);
$this->newLine();
}
private function inspectBackups(): void
{
$this->info('=== Scheduled Backups ===');
$backups = ScheduledDatabaseBackup::with(['database'])
->where('enabled', true)
->get();
$rows = [];
foreach ($backups as $backup) {
$server = $backup->server();
$frequency = $backup->frequency;
if (isset(VALID_CRON_STRINGS[$frequency])) {
$frequency = VALID_CRON_STRINGS[$frequency];
}
$dedupKey = "scheduled-backup:{$backup->id}";
$cacheValue = Cache::get($dedupKey);
$timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$rows[] = [
$backup->id,
$backup->database_type ?? 'unknown',
$server?->name ?? 'N/A',
$frequency,
$cacheValue ?? '<missing>',
$wouldFire ? 'YES' : 'no',
];
}
$this->table(
['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
$rows
);
$this->newLine();
}
private function inspectTasks(): void
{
$this->info('=== Scheduled Tasks ===');
$tasks = ScheduledTask::with(['service', 'application'])
->where('enabled', true)
->get();
$rows = [];
foreach ($tasks as $task) {
$server = $task->server();
$frequency = $task->frequency;
if (isset(VALID_CRON_STRINGS[$frequency])) {
$frequency = VALID_CRON_STRINGS[$frequency];
}
$dedupKey = "scheduled-task:{$task->id}";
$cacheValue = Cache::get($dedupKey);
$timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$rows[] = [
$task->id,
$task->name,
$server?->name ?? 'N/A',
$frequency,
$cacheValue ?? '<missing>',
$wouldFire ? 'YES' : 'no',
];
}
$this->table(
['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
$rows
);
$this->newLine();
}
private function inspectServerJobs(?string $serverFilter): void
{
$this->info('=== Server Manager Jobs ===');
$servers = $this->getServers($serverFilter);
$rows = [];
foreach ($servers as $server) {
$timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$dedupKeys = [
"server-patch-check:{$server->id}" => '0 0 * * 0',
"server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',
"server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),
];
foreach ($dedupKeys as $dedupKey => $frequency) {
if (isset(VALID_CRON_STRINGS[$frequency])) {
$frequency = VALID_CRON_STRINGS[$frequency];
}
$cacheValue = Cache::get($dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$rows[] = [
$server->id,
$server->name,
$dedupKey,
$frequency,
$cacheValue ?? '<missing>',
$wouldFire ? 'YES' : 'no',
];
}
}
$this->table(
['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],
$rows
);
$this->newLine();
}
private function getServers(?string $serverFilter): Collection
{
$query = Server::with('settings')->where('ip', '!=', '1.2.3.4');
if ($serverFilter) {
$query->where('id', $serverFilter);
}
if (isCloud()) {
$servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
$own = Team::find(0)?->servers()->with('settings')->get() ?? collect();
return $servers->merge($own);
}
return $query->get();
}
}
app/Console/Kernel.php
+14 -3
View File
@@ -12,7 +12,6 @@ use App\Jobs\CleanupStaleMultiplexedConnections;
use App\Jobs\PullChangelog;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\RegenerateSslCertJob;
use App\Jobs\ScheduledJobManager;
use App\Jobs\ServerManagerJob;
use App\Jobs\UpdateCoolifyJob;
use App\Models\InstanceSettings;
@@ -75,7 +74,7 @@ class Kernel extends ConsoleKernel
$this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();
// Scheduled Jobs (Backups & Tasks)
$this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
$this->scheduleScheduledJobs();
$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();
@@ -96,7 +95,7 @@ class Kernel extends ConsoleKernel
$this->pullImages();
// Scheduled Jobs (Backups & Tasks)
$this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
$this->scheduleScheduledJobs();
$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily()->onOneServer();
@@ -116,6 +115,18 @@ class Kernel extends ConsoleKernel
->onOneServer();
}
/**
* Run the dispatcher in the scheduler process, so a busy queue cannot delay it. Parallel runs
* are safe because each occurrence is claimed with an atomic update of next_run_at.
*/
private function scheduleScheduledJobs(): void
{
$this->scheduleInstance->command('scheduled:dispatch')
->everyMinute()
->onOneServer()
->runInBackground();
}
private function scheduleUpdates(): void
{
$this->scheduleInstance->job(new CheckForUpdatesJob)
app/Jobs/ScheduledJobManager.php
+320 -691
View File
File diff suppressed because it is too large. Load diff
app/Livewire/Project/Shared/Destination.php
+2
View File
@@ -8,6 +8,7 @@ use App\Events\ApplicationStatusChanged;
use App\Models\Application;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Services\ScheduleNextRunRecalculator;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Livewire\Component;
@@ -136,6 +137,7 @@ class Destination extends Component
->wherePivot('server_id', $server->id)
->detach($network->id);
$this->resource->additional_networks()->attach($mainDestination->id, ['server_id' => $mainDestination->server->id]);
app(ScheduleNextRunRecalculator::class)->forResource($this->resource);
});
$this->resource->refresh();
$this->refreshServers();
app/Livewire/Project/Shared/ScheduledTask/Show.php
+10
View File
@@ -7,6 +7,7 @@ use App\Models\Application;
use App\Models\ScheduledTask;
use App\Models\Service;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Attributes\Computed;
use Livewire\Attributes\Locked;
use Livewire\Attributes\Validate;
use Livewire\Component;
@@ -87,6 +88,15 @@ class Show extends Component
}
}
/**
* Timezone of the server the task runs on, used to display its next run.
*/
#[Computed]
public function serverTimezone(): ?string
{
return $this->task->scheduleTimezone();
}
private function syncData(bool $toModel = false): void
{
if ($toModel) {
app/Livewire/Server/DockerCleanup.php
+10 -26
View File
@@ -5,9 +5,7 @@ namespace App\Livewire\Server;
use App\Jobs\DockerCleanupJob;
use App\Models\DockerCleanupExecution;
use App\Models\Server;
use Cron\CronExpression;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Cache;
use Livewire\Attributes\Computed;
use Livewire\Attributes\Validate;
@@ -39,35 +37,21 @@ class DockerCleanup extends Component
#[Validate('boolean')]
public bool $disableApplicationImageRetention = false;
/**
* The scheduled cleanup is stale when its next run is more than 10 minutes overdue,
* meaning the scheduled job dispatcher did not run it.
*/
#[Computed]
public function isCleanupStale(): bool
{
try {
$lastExecution = DockerCleanupExecution::where('server_id', $this->server->id)
->orderBy('created_at', 'desc')
->first();
if (! $lastExecution) {
return false;
}
$frequency = $this->server->settings->docker_cleanup_frequency ?? '0 0 * * *';
if (isset(VALID_CRON_STRINGS[$frequency])) {
$frequency = VALID_CRON_STRINGS[$frequency];
}
$cron = new CronExpression($frequency);
$now = Carbon::now();
$nextRun = Carbon::parse($cron->getNextRunDate($now));
$afterThat = Carbon::parse($cron->getNextRunDate($nextRun));
$intervalMinutes = $nextRun->diffInMinutes($afterThat);
$threshold = max($intervalMinutes * 2, 10);
return Carbon::parse($lastExecution->created_at)->diffInMinutes($now) > $threshold;
} catch (\Throwable) {
// The dispatcher does not run cleanups on servers without a real IP address.
if (blank($this->server->ip) || $this->server->hasPlaceholderIp()) {
return false;
}
$nextRunAt = $this->server->settings->docker_cleanup_next_run_at;
return $nextRunAt !== null && $nextRunAt->lt(now()->subMinutes(10));
}
#[Computed]
app/Models/ScheduledDatabaseBackup.php
+4
View File
@@ -2,12 +2,15 @@
namespace App\Models;
use App\Traits\HasNextRunAt;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledDatabaseBackup extends BaseModel
{
use HasNextRunAt;
protected function casts(): array
{
return [
@@ -17,6 +20,7 @@ class ScheduledDatabaseBackup extends BaseModel
'missing_backup_notification_days' => 'integer',
'missing_backup_notification_sent_at' => 'datetime',
'last_execution_at' => 'datetime',
'next_run_at' => 'datetime',
];
}
app/Models/ScheduledJobState.php
-18
View File
@@ -1,18 +0,0 @@
<?php
namespace App\Models;
class ScheduledJobState extends BaseModel
{
protected $fillable = [
'schedule_key',
'last_scheduled_for',
];
protected function casts(): array
{
return [
'last_scheduled_for' => 'immutable_datetime',
];
}
}
app/Models/ScheduledTask.php
+4
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Traits\HasNextRunAt;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
@@ -20,6 +21,7 @@ use OpenApi\Attributes as OA;
'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'],
'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'],
'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'],
'next_run_at' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'],
'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'],
'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'],
],
@@ -27,6 +29,7 @@ use OpenApi\Attributes as OA;
class ScheduledTask extends BaseModel
{
use HasFactory;
use HasNextRunAt;
use HasSafeStringAttribute;
protected $fillable = [
@@ -52,6 +55,7 @@ class ScheduledTask extends BaseModel
return [
'enabled' => 'boolean',
'timeout' => 'integer',
'next_run_at' => 'datetime',
];
}
app/Models/ScheduledVolumeBackup.php
+4
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Traits\HasNextRunAt;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
@@ -11,6 +12,8 @@ use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledVolumeBackup extends BaseModel
{
use HasNextRunAt;
public const int DEFAULT_TIMEOUT = 36000;
protected $fillable = [
@@ -47,6 +50,7 @@ class ScheduledVolumeBackup extends BaseModel
'retention_days_s3' => 'integer',
'retention_max_storage_s3' => 'float',
'timeout' => 'integer',
'next_run_at' => 'datetime',
];
}
app/Models/ServerSetting.php
+16
View File
@@ -3,6 +3,7 @@
namespace App\Models;
use App\Enums\ServerRole;
use App\Services\ScheduleNextRunRecalculator;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model;
@@ -168,6 +169,7 @@ class ServerSetting extends Model
'docker_version_checked_at' => 'datetime',
'compose_version_checked_at' => 'datetime',
'backup_compression_cpu_percentage' => 'integer',
'docker_cleanup_next_run_at' => 'datetime',
];
/**
@@ -188,6 +190,15 @@ class ServerSetting extends Model
protected static function booted()
{
static::saving(function ($setting) {
if (! $setting->exists || $setting->isDirty(['docker_cleanup_frequency', 'server_timezone'])) {
// Without a frequency (a new row that uses the column default), the dispatcher calculates it.
$frequency = $setting->getAttributes()['docker_cleanup_frequency'] ?? null;
$setting->docker_cleanup_next_run_at = $frequency === null
? null
: next_cron_run_at($frequency, $setting->server_timezone, now());
}
});
static::creating(function ($setting) {
try {
if (str($setting->sentinel_token)->isEmpty()) {
@@ -201,6 +212,11 @@ class ServerSetting extends Model
}
});
static::updated(function ($settings) {
if ($settings->wasChanged('server_timezone') && $settings->server) {
// A failure must not fail the settings save: the schedules correct themselves after their next run.
rescue(fn () => app(ScheduleNextRunRecalculator::class)->forServer($settings->server));
}
if (
$settings->wasChanged('sentinel_token') ||
$settings->wasChanged('sentinel_custom_url') ||
app/Services/ScheduleNextRunRecalculator.php
+85
View File
@@ -0,0 +1,85 @@
<?php
namespace App\Services;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandalonePostgresql;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\Relation;
/**
* Recalculates next_run_at of schedules after their server timezone can have changed:
* a server timezone change, or a resource that moved to another server.
*/
class ScheduleNextRunRecalculator
{
/**
* @param string|null $timezone The timezone of the resource's server. Read from the database when null.
*/
public function forResource(Model $resource, ?string $timezone = null): void
{
$timezone ??= $this->serverTimezone($resource);
foreach (['scheduled_tasks', 'scheduledBackups'] as $relation) {
if (method_exists($resource, $relation)) {
$this->recalculate($resource->{$relation}(), $timezone);
}
}
foreach (['persistentStorages', 'fileStorages'] as $relation) {
if (method_exists($resource, $relation)) {
$storages = $resource->{$relation}();
$this->recalculate(ScheduledVolumeBackup::query()
->where('backupable_type', $storages->getRelated()->getMorphClass())
->whereIn('backupable_id', $storages->select('id')), $timezone);
}
}
if ($resource instanceof Service) {
$resource->applications()->get()->each(fn (Model $application) => $this->forResource($application, $timezone));
$resource->databases()->get()->each(fn (Model $database) => $this->forResource($database, $timezone));
}
}
/**
* Each resource uses the timezone of its own main server: Server::applications() also returns
* applications for which this server is only an additional server.
*/
public function forServer(Server $server): void
{
$server->applications()->each(fn (Model $application) => $this->forResource($application));
$server->databases()->each(fn (Model $database) => $this->forResource($database));
$server->services()->get()->each(fn (Service $service) => $this->forResource($service));
// Server::databases() leaves out the instance database, which has the instance backup.
if ($server->id === 0 && ($instanceDatabase = StandalonePostgresql::find(0))) {
$this->forResource($instanceDatabase);
}
}
/**
* Move enabled schedules to their next due time after now. The current minute is excluded,
* so an occurrence that already ran in this minute does not run again.
*/
private function recalculate(Relation|Builder $schedules, ?string $timezone): void
{
$schedules->where('enabled', true)->get()->each(fn (Model $schedule) => $schedule->forceFill([
'next_run_at' => next_cron_run_at((string) $schedule->frequency, $timezone, now()),
])->saveQuietly());
}
private function serverTimezone(Model $resource): ?string
{
$owner = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase
? $resource->service()->first()
: $resource;
$server = $owner?->destination()->first()?->server()->first();
return $server?->settings()->value('server_timezone');
}
}
app/Services/ScheduledJobDeliveryService.php
+16 -132
View File
@@ -8,107 +8,28 @@ use App\Jobs\ScheduledTaskJob;
use App\Jobs\VolumeBackupJob;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use Cron\CronExpression;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Carbon\CarbonInterface;
use Illuminate\Support\Facades\Log;
class ScheduledJobDeliveryService
{
public const CATCH_UP_WINDOW_MINUTES = 10;
/**
* Record a pending delivery for one occurrence of a schedule. Returns null when the
* occurrence already has a delivery.
*
* @param array<string, mixed> $payload
*/
public function create(string $scheduleKey, CarbonInterface $scheduledFor, string $jobType, int $resourceId, array $payload = []): ?ScheduledJobDelivery
{
$delivery = ScheduledJobDelivery::createOrFirst(
['schedule_key' => $scheduleKey, 'scheduled_for' => $scheduledFor->utc()],
['job_type' => $jobType, 'resource_id' => $resourceId, 'payload' => $payload, 'status' => 'pending'],
);
public function recordAndPublish(
string $scheduleKey,
string $frequency,
string $timezone,
string $jobType,
int $resourceId,
array $payload = [],
?Carbon $executionTime = null,
): bool {
$executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);
$cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);
$scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {
return false;
}
$delivery = DB::transaction(function () use ($scheduleKey, $scheduledFor, $jobType, $resourceId, $payload): ?ScheduledJobDelivery {
ScheduledJobState::query()->insertOrIgnore([
'uuid' => new_public_id(),
'schedule_key' => $scheduleKey,
'created_at' => now(),
'updated_at' => now(),
]);
$state = ScheduledJobState::query()
->where('schedule_key', $scheduleKey)
->lockForUpdate()
->firstOrFail();
if ($state->last_scheduled_for?->gte($scheduledFor)) {
return null;
}
$state->update(['last_scheduled_for' => $scheduledFor->utc()]);
return ScheduledJobDelivery::create([
'schedule_key' => $scheduleKey,
'scheduled_for' => $scheduledFor->utc(),
'job_type' => $jobType,
'resource_id' => $resourceId,
'payload' => $payload,
'status' => 'pending',
]);
});
if ($delivery === null) {
return false;
}
return $this->publish($delivery);
}
public function recordSkipped(
string $scheduleKey,
string $frequency,
string $timezone,
?Carbon $executionTime = null,
): bool {
$executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);
$cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);
$scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {
return false;
}
return DB::transaction(function () use ($scheduleKey, $scheduledFor): bool {
ScheduledJobState::query()->insertOrIgnore([
'uuid' => new_public_id(),
'schedule_key' => $scheduleKey,
'created_at' => now(),
'updated_at' => now(),
]);
$state = ScheduledJobState::query()
->where('schedule_key', $scheduleKey)
->lockForUpdate()
->firstOrFail();
if ($state->last_scheduled_for?->gte($scheduledFor)) {
return false;
}
$state->update(['last_scheduled_for' => $scheduledFor->utc()]);
return true;
});
return $delivery->wasRecentlyCreated ? $delivery : null;
}
public function publishPending(): void
@@ -132,24 +53,6 @@ class ScheduledJobDeliveryService
});
}
/**
* Count occurrences that are still open 15 minutes after they were created. Pending or
* enqueued rows here never started; claimed rows are still running or their worker died.
*
* @return array<string, int>
*/
public function staleOpenOccurrenceCounts(): array
{
return ScheduledJobDelivery::query()
->whereIn('status', ['pending', 'enqueued', 'claimed'])
->where('created_at', '<', now()->subMinutes(15))
->selectRaw('status, count(*) as aggregate')
->groupBy('status')
->pluck('aggregate', 'status')
->map(fn ($count) => (int) $count)
->all();
}
public function deleteOldOccurrences(): void
{
ScheduledJobDelivery::query()
@@ -244,7 +147,7 @@ class ScheduledJobDeliveryService
}
}
private function publish(ScheduledJobDelivery $occurrence): bool
public function publish(ScheduledJobDelivery $occurrence): bool
{
if ($occurrence->status !== 'pending') {
return false;
@@ -283,15 +186,6 @@ class ScheduledJobDeliveryService
return false;
}
// An open previous occurrence usually means the previous run is still running. Jobs with
// WithoutOverlapping(...)->dontRelease() then drop this one without running it.
$previousOpen = ScheduledJobDelivery::query()
->where('schedule_key', $occurrence->schedule_key)
->where('scheduled_for', '<', $occurrence->scheduled_for)
->whereIn('status', ['pending', 'enqueued', 'claimed'])
->orderByDesc('scheduled_for')
->first(['uuid', 'status', 'scheduled_for']);
dispatch($job);
ScheduledJobDelivery::query()
@@ -304,17 +198,7 @@ class ScheduledJobDeliveryService
]);
// Log from the loaded row: a fast worker can complete and delete the occurrence already.
$enqueued = ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()];
if ($previousOpen) {
$this->logOccurrence($occurrence, 'Scheduled occurrence enqueued while the previous occurrence is still open', 'warning', [
...$enqueued,
'previous_occurrence_uuid' => $previousOpen->uuid,
'previous_status' => $previousOpen->status,
'previous_scheduled_for' => $previousOpen->scheduled_for->toIso8601String(),
]);
} else {
$this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: $enqueued);
}
$this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()]);
return true;
}
app/Traits/HasNextRunAt.php
+38
View File
@@ -0,0 +1,38 @@
<?php
namespace App\Traits;
use Carbon\CarbonImmutable;
use DateTimeInterface;
/**
* Keeps next_run_at of a schedule model in sync with its frequency and enabled flag.
* The model must have `frequency`, `enabled`, `next_run_at`, and a `server()` method.
*/
trait HasNextRunAt
{
protected static function bootHasNextRunAt(): void
{
static::saving(function ($schedule) {
if ($schedule->exists && ! $schedule->isDirty(['frequency', 'enabled'])) {
return;
}
$schedule->next_run_at = $schedule->calculateNextRunAt(now());
});
}
public function calculateNextRunAt(DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable
{
if ($this->getAttribute('enabled') !== null && ! $this->enabled) {
return null;
}
return next_cron_run_at((string) $this->frequency, $this->scheduleTimezone(), $after, $includeCurrentMinute);
}
public function scheduleTimezone(): ?string
{
return data_get($this->server(), 'settings.server_timezone');
}
}
bootstrap/helpers/shared.php
+22
View File
@@ -1111,6 +1111,28 @@ function validate_timezone(string $timezone): bool
return in_array($timezone, timezone_identifiers_list());
}
/**
* The next due time of a cron frequency after $after, in UTC. An invalid frequency returns null.
* With $includeCurrentMinute, a frequency that is due in the minute of $after returns that minute.
*/
function next_cron_run_at(string $frequency, ?string $timezone, DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable
{
$frequency = trim($frequency);
$timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');
try {
$cron = new Cron\CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);
$time = CarbonImmutable::instance($after)->setTimezone($timezone);
if ($includeCurrentMinute) {
$time = $time->startOfMinute();
}
return CarbonImmutable::instance($cron->getNextRunDate($time, 0, $includeCurrentMinute))->utc();
} catch (Throwable) {
return null;
}
}
function parseEnvFormatToArray($env_file_contents)
{
$env_array = [];
database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php
+94
View File
@@ -0,0 +1,94 @@
<?php
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledTask;
use App\Models\ScheduledVolumeBackup;
use App\Models\ServerSetting;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Schedules without a recorded occurrence keep next_run_at = null. The dispatcher calculates it on its first run.
*/
public function up(): void
{
foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {
Schema::table($table, function (Blueprint $table) {
$table->timestampTz('next_run_at')->nullable();
$table->index(['enabled', 'next_run_at']);
});
}
Schema::table('server_settings', function (Blueprint $table) {
$table->timestampTz('docker_cleanup_next_run_at')->nullable()->index();
});
$this->continueAfterLastScheduledOccurrences();
Schema::dropIfExists('scheduled_job_states');
}
public function down(): void
{
Schema::create('scheduled_job_states', function (Blueprint $table) {
$table->id();
$table->string('uuid')->unique();
$table->string('schedule_key')->unique();
$table->timestampTz('last_scheduled_for')->nullable();
$table->timestamps();
});
Schema::table('server_settings', function (Blueprint $table) {
$table->dropIndex(['docker_cleanup_next_run_at']);
$table->dropColumn('docker_cleanup_next_run_at');
});
foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {
Schema::table($table, function (Blueprint $table) {
$table->dropIndex(['enabled', 'next_run_at']);
$table->dropColumn('next_run_at');
});
}
}
/**
* Continue each schedule after the last occurrence that the previous dispatcher recorded, so
* an occurrence that was due during the upgrade still runs late instead of being skipped.
*/
public function continueAfterLastScheduledOccurrences(): void
{
if (! Schema::hasTable('scheduled_job_states')) {
return;
}
DB::table('scheduled_job_states')->whereNotNull('last_scheduled_for')->orderBy('id')->chunkById(500, function ($states) {
foreach ($states as $state) {
try {
[$type, $id] = explode(':', $state->schedule_key, 2) + [null, null];
$schedule = match ($type) {
'scheduled-backup' => ScheduledDatabaseBackup::find($id),
'scheduled-task' => ScheduledTask::find($id),
'scheduled-volume-backup' => ScheduledVolumeBackup::find($id),
'docker-cleanup' => ServerSetting::query()->where('server_id', $id)->first(),
default => null,
};
if ($schedule instanceof ServerSetting) {
$schedule->forceFill(['docker_cleanup_next_run_at' => next_cron_run_at(
(string) $schedule->docker_cleanup_frequency, $schedule->server_timezone, Carbon::parse($state->last_scheduled_for),
)])->saveQuietly();
} elseif ($schedule?->enabled) {
$schedule->forceFill(['next_run_at' => $schedule->calculateNextRunAt(Carbon::parse($state->last_scheduled_for))])->saveQuietly();
}
} catch (Throwable) {
// The dispatcher calculates the next run of this schedule on its first run.
}
}
});
}
};
openapi.json
+6
View File
@@ -23557,6 +23557,12 @@
"type": "integer",
"description": "The timeout of the scheduled task in seconds."
},
"next_run_at": {
"type": "string",
"format": "date-time",
"nullable": true,
"description": "The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet."
},
"created_at": {
"type": "string",
"format": "date-time",
openapi.yaml
+5
View File
@@ -15091,6 +15091,11 @@ components:
timeout:
type: integer
description: 'The timeout of the scheduled task in seconds.'
next_run_at:
type: string
format: date-time
nullable: true
description: 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'
created_at:
type: string
format: date-time
resources/views/components/next-run.blade.php
+30
View File
@@ -0,0 +1,30 @@
@props([
'at' => null,
'timezone' => null,
'enabled' => true,
'frequency' => null,
])
@php
// Same fallback as next_cron_run_at(): schedules without a valid server timezone run in the instance timezone.
$displayTimezone = filled($timezone) && validate_timezone((string) $timezone) ? $timezone : config('app.timezone');
$nextRunAt = $at ? \Illuminate\Support\Carbon::parse($at) : null;
// Without a next run, the frequency is either not calculated yet or cannot be calculated at all.
$isFrequencyInvalid = ! $nextRunAt && filled($frequency) && next_cron_run_at((string) $frequency, $displayTimezone, now()) === null;
@endphp
<p {{ $attributes->merge(['class' => 'text-xs leading-5 text-neutral-500 dark:text-fg-dim']) }}>
<span class="font-medium text-neutral-700 dark:text-fg">Next run</span>
@if (! $enabled)
<span>Disabled</span>
@elseif ($isFrequencyInvalid)
<span class="text-red-500">Invalid frequency. Change it to a valid cron expression, or this schedule never runs.</span>
@elseif (! $nextRunAt)
<span>Calculating…</span>
@else
<time datetime="{{ $nextRunAt->toIso8601String() }}"
title="{{ $nextRunAt->copy()->utc()->format('Y-m-d H:i') }} UTC">{{ $nextRunAt->copy()->setTimezone($displayTimezone)->format('Y-m-d H:i') }} ({{ $displayTimezone }})</time>
<span>·
{{ $nextRunAt->isFuture() ? 'in '.$nextRunAt->diffForHumans(syntax: \Carbon\CarbonInterface::DIFF_ABSOLUTE) : $nextRunAt->diffForHumans() }}</span>
@endif
</p>
resources/views/livewire/project/database/backup-edit/general.blade.php
+1
View File
@@ -96,6 +96,7 @@
min="0" max="365" suffix="days" canGate="manageBackups" :canResource="$backup->database"
helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required />
</div>
<x-next-run :at="$backup->next_run_at" :timezone="$timezone" :enabled="$backup->enabled" :frequency="$backup->frequency" />
</div>
</x-application.settings-section>
</form>
resources/views/livewire/project/shared/scheduled-task/show.blade.php
+1
View File
@@ -44,6 +44,7 @@
<x-forms.input canGate="update" :canResource="$resource" :disabled="!auth()->user()->can('update', $resource)"
placeholder="php artisan schedule:run" id="command" label="Command" required />
</div>
<x-next-run class="mt-4" :at="$task->next_run_at" :timezone="$this->serverTimezone" :enabled="$task->enabled" :frequency="$task->frequency" />
</x-application.settings-section>
</form>
resources/views/livewire/project/shared/storages/volume-backups/general.blade.php
+1
View File
@@ -45,5 +45,6 @@
<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"
helper="Maximum backup runtime in seconds." required />
</div>
<x-next-run class="mt-4" :at="$backup?->next_run_at" :timezone="$timezone" :enabled="(bool) $backup?->enabled" :frequency="$backup?->frequency" />
</x-application.settings-section>
</form>
resources/views/livewire/server/docker-cleanup.blade.php
+4
View File
@@ -33,6 +33,8 @@
@if (!isCloud() && $this->isCleanupStale)
<x-callout type="warning" title="Docker cleanup may be stalled">
A scheduled cleanup was due
{{ $server->settings->docker_cleanup_next_run_at?->diffForHumans() }} but has not started.
The last cleanup ran {{ $this->lastExecutionTime ?? 'at an unknown time' }}.
@if (!$this->isSchedulerHealthy)
The scheduled job manager also appears inactive.
@@ -52,6 +54,8 @@
<p class="mt-1 text-xs leading-5 text-neutral-500 dark:text-fg-dim">
Cleanup runs automatically using the schedule and threshold configured below.
</p>
<x-next-run class="mt-1" :at="$server->settings->docker_cleanup_next_run_at"
:timezone="$server->settings->server_timezone" :frequency="$server->settings->docker_cleanup_frequency" />
</div>
</div>
@endif
tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.php
-50
View File
@@ -1,50 +0,0 @@
<?php
use App\Jobs\ScheduledJobManager;
use Illuminate\Support\Facades\Redis;
function scheduledJobManagerLockKey(): string
{
return config('cache.prefix').'laravel-queue-overlap:'.ScheduledJobManager::class.':scheduled-job-manager';
}
it('clears stale lock when TTL is -1', function () {
$lockKey = scheduledJobManagerLockKey();
$redis = Mockery::mock();
$redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-1);
$redis->shouldReceive('del')->once()->with($lockKey)->andReturn(1);
Redis::shouldReceive('connection')->with('default')->andReturn($redis);
$job = new ScheduledJobManager;
$job->middleware();
});
it('preserves valid lock with positive TTL', function () {
$lockKey = scheduledJobManagerLockKey();
$redis = Mockery::mock();
$redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(60);
$redis->shouldNotReceive('del');
Redis::shouldReceive('connection')->with('default')->andReturn($redis);
$job = new ScheduledJobManager;
$job->middleware();
});
it('does not fail when no lock exists', function () {
$lockKey = scheduledJobManagerLockKey();
$redis = Mockery::mock();
$redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-2);
$redis->shouldNotReceive('del');
Redis::shouldReceive('connection')->with('default')->andReturn($redis);
$job = new ScheduledJobManager;
$middleware = $job->middleware();
expect($middleware)->toBeArray()->toHaveCount(1);
});
tests/Feature/QueueRoutingTest.php
-3
View File
@@ -4,7 +4,6 @@ use App\Actions\Database\StartDatabase;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Service\StartService;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\ScheduledJobManager;
use App\Models\ScheduledDatabaseBackup;
describe('deployment_queue helper', function () {
@@ -61,14 +60,12 @@ describe('scheduled job routing', function () {
test('scheduled jobs use the crons queue on cloud', function () {
config(['constants.coolify.self_hosted' => false]);
expect((new ScheduledJobManager)->queue)->toBe('crons');
expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('crons');
});
test('scheduled jobs use the high queue on self-hosted', function () {
config(['constants.coolify.self_hosted' => true]);
expect((new ScheduledJobManager)->queue)->toBe('high');
expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('high');
});
});
tests/Feature/ScheduleNextRunAtTest.php
+179
View File
@@ -0,0 +1,179 @@
<?php
use App\Models\Application;
use App\Models\Environment;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledTask;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Services\ScheduleNextRunRecalculator;
use Carbon\CarbonImmutable;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));
});
it('calculates the next due time of a frequency in the server timezone', function (string $frequency, string $timezone, string $after, bool $includeCurrentMinute, ?string $expected) {
$next = next_cron_run_at($frequency, $timezone, CarbonImmutable::parse($after, 'UTC'), $includeCurrentMinute);
expect($next?->toDateTimeString())->toBe($expected);
})->with([
'alias' => ['daily', 'UTC', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],
'server timezone' => ['0 2 * * *', 'Europe/Budapest', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],
'strictly after' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:00', false, '2026-09-18 10:20:00'],
'current minute included' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:30', true, '2026-09-17 10:20:00'],
'a time that does not exist when daylight saving time starts' => ['30 2 * * *', 'Europe/Budapest', '2026-03-28 12:00:00', false, '2026-03-29 01:30:00'],
'invalid timezone uses the app timezone' => ['0 2 * * *', 'Mars/Olympus', '2026-09-17 10:20:30', false, '2026-09-18 02:00:00'],
'invalid frequency' => ['not a cron', 'UTC', '2026-09-17 10:20:30', false, null],
]);
it('calculates the next run of a schedule when it is created or its frequency or enabled flag changes', function () {
$task = nextRunAtTestTask(nextRunAtTestApplication('Europe/Budapest'), '0 2 * * *');
expect($task->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');
$task->update(['frequency' => 'hourly']);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');
$task->update(['enabled' => false]);
expect($task->fresh()->next_run_at)->toBeNull();
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));
$task->update(['enabled' => true]);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');
$task->update(['name' => 'renamed']);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');
});
it('calculates the next Docker cleanup when its frequency or the server timezone changes', function () {
$settings = nextRunAtTestApplication()->destination->server->settings;
$settings->update(['docker_cleanup_frequency' => '0 2 * * *']);
expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');
$settings->update(['server_timezone' => 'Europe/Budapest']);
expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');
});
it('recalculates the next run of the server schedules when the server timezone changes', function () {
$application = nextRunAtTestApplication();
$task = nextRunAtTestTask($application, '0 2 * * *');
$otherTask = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');
$application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')
->and($otherTask->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');
});
it('keeps the timezone of the main server for applications that also run on the changed server', function () {
$application = nextRunAtTestApplication();
$task = nextRunAtTestTask($application, '0 2 * * *');
$additionalServer = nextRunAtTestApplication()->destination->server;
$application->additional_networks()->attach(
StandaloneDocker::where('server_id', $additionalServer->id)->value('id'),
['server_id' => $additionalServer->id],
);
$additionalServer->settings->update(['server_timezone' => 'Europe/Budapest']);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');
});
it('does not run an occurrence again when the timezone changes in the minute it ran', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 11, 59, 0, 'UTC'));
$application = nextRunAtTestApplication();
$task = nextRunAtTestTask($application, '0 * * * *');
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 30, 'UTC'));
$application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');
});
it('recalculates the next run of the resource schedules with the timezone of its current server', function () {
$application = nextRunAtTestApplication();
$task = nextRunAtTestTask($application, '0 2 * * *');
$application->destination->server->settings()->update(['server_timezone' => 'Europe/Budapest']);
app(ScheduleNextRunRecalculator::class)->forResource($application);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');
});
it('lists overdue schedules and schedules without a next run in the diagnostics', function () {
$overdue = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');
ScheduledTask::query()->whereKey($overdue->id)->update(['next_run_at' => now()->subHour()]);
$invalid = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');
ScheduledTask::query()->whereKey($invalid->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);
nextRunAtTestTask(nextRunAtTestApplication(), '0 3 * * *');
$this->artisan('scheduled:diagnostics')
->expectsOutputToContain('Scheduled tasks: 3 enabled, 2 overdue or without a next run')
->expectsOutputToContain('invalid frequency')
->assertSuccessful();
});
it('continues each schedule after the occurrence that the previous dispatcher recorded last', function () {
$task = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');
Schema::create('scheduled_job_states', function (Blueprint $table) {
$table->id();
$table->string('uuid')->unique();
$table->string('schedule_key')->unique();
$table->timestampTz('last_scheduled_for')->nullable();
$table->timestamps();
});
DB::table('scheduled_job_states')->insert([
'uuid' => 'state-1',
'schedule_key' => "scheduled-task:{$task->id}",
'last_scheduled_for' => '2026-09-16 02:00:00',
]);
$migration = require database_path('migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php');
$migration->continueAfterLastScheduledOccurrences();
// The occurrence of 2026-09-17 02:00 was not recorded, so the dispatcher runs it on its next run.
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');
});
function nextRunAtTestApplication(string $timezone = 'UTC'): Application
{
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
]);
$server->settings()->update(['server_timezone' => $timezone]);
$destination = StandaloneDocker::where('server_id', $server->id)->first()
?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
$environment = Environment::factory()->create([
'project_id' => Project::factory()->create(['team_id' => $team->id])->id,
]);
return Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => StandaloneDocker::class,
]);
}
function nextRunAtTestTask(Application $application, string $frequency): ScheduledTask
{
return ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => $frequency,
'enabled' => true,
])->fresh();
}
tests/Feature/ScheduleNextRunDisplayTest.php
+222
View File
@@ -0,0 +1,222 @@
<?php
use App\Livewire\Project\Database\BackupEdit;
use App\Livewire\Project\Shared\ScheduledTask\Show;
use App\Livewire\Project\Shared\Storages\VolumeBackups;
use App\Livewire\Server\DockerCleanup;
use App\Models\Application;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\LocalPersistentVolume;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledTask;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Routing\Events\RouteMatched;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Event;
use Livewire\Livewire;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
$this->user->teams()->attach($this->team, ['role' => 'owner']);
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
$this->server->settings->update(['server_timezone' => 'Europe/Budapest']);
$this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
$project = Project::factory()->create(['team_id' => $this->team->id]);
$this->environment = Environment::factory()->create(['project_id' => $project->id]);
$this->application = Application::factory()->create([
'environment_id' => $this->environment->id,
'destination_id' => $this->destination->id,
'destination_type' => $this->destination->getMorphClass(),
]);
});
afterEach(function () {
Event::forget(RouteMatched::class);
Carbon::setTestNow();
});
function nextRunDisplayBindTaskRoute(Application $application, ScheduledTask $task): void
{
$application->loadMissing('environment.project');
Event::listen(RouteMatched::class, function (RouteMatched $event) use ($application, $task): void {
$event->route->setParameter('task_uuid', $task->uuid);
$event->route->setParameter('project_uuid', $application->environment->project->uuid);
$event->route->setParameter('environment_uuid', $application->environment->uuid);
$event->route->setParameter('application_uuid', $application->uuid);
});
}
function nextRunDisplaySetCleanupNextRun(Server $server, ?Carbon $nextRunAt): void
{
ServerSetting::query()->where('server_id', $server->id)->update(['docker_cleanup_next_run_at' => $nextRunAt]);
}
it('shows the next Docker cleanup in the server timezone and refreshes it after saving a new frequency', function () {
$this->server->settings->update(['docker_cleanup_frequency' => '0 2 * * *']);
Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])
->assertSee('Next run')
->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)
->set('dockerCleanupFrequency', 'hourly')
->call('submit')
->assertSee('2026-09-17 13:00 (Europe/Budapest)', false)
->assertSee('in 39 minutes');
});
it('shows Calculating when the Docker cleanup next run is not calculated yet', function () {
nextRunDisplaySetCleanupNextRun($this->server, null);
Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])
->assertSee('Next run')
->assertSee('Calculating…');
});
it('marks the Docker cleanup as stale only when its next run is more than 10 minutes overdue', function (?int $minutesFromNow, bool $stale) {
nextRunDisplaySetCleanupNextRun($this->server, $minutesFromNow === null ? null : now()->addMinutes($minutesFromNow));
$component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);
expect($component->instance()->isCleanupStale())->toBe($stale);
if ($stale) {
$component->assertSee('Docker cleanup may be stalled');
} else {
$component->assertDontSee('Docker cleanup may be stalled');
}
})->with([
'30 minutes overdue' => [-30, true],
'5 minutes overdue' => [-5, false],
'in the future' => [30, false],
'not calculated yet' => [null, false],
]);
it('does not mark the Docker cleanup as stale on a server without a real IP address', function () {
$this->server->update(['ip' => Server::PLACEHOLDER_IP]);
nextRunDisplaySetCleanupNextRun($this->server, now()->subMinutes(30));
$component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);
expect($component->instance()->isCleanupStale())->toBeFalse();
});
it('shows the next scheduled task run in the server timezone and refreshes it on changes', function () {
$task = ScheduledTask::factory()->create([
'team_id' => $this->team->id,
'application_id' => $this->application->id,
'frequency' => '0 2 * * *',
'enabled' => true,
]);
nextRunDisplayBindTaskRoute($this->application, $task);
$component = Livewire::test(Show::class)
->assertSee('Next run')
->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)
->assertSee('in 13 hours');
$component->call('toggleEnabled')
->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)
->assertSeeInOrder(['Next run', 'Disabled']);
$component->call('toggleEnabled')
->set('frequency', 'hourly')
->call('submit')
->assertSee('2026-09-17 13:00 (Europe/Budapest)', false);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');
});
it('shows Calculating for an enabled scheduled task without a next run', function () {
$task = ScheduledTask::factory()->create([
'team_id' => $this->team->id,
'application_id' => $this->application->id,
'frequency' => '0 2 * * *',
'enabled' => true,
]);
ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);
nextRunDisplayBindTaskRoute($this->application, $task);
Livewire::test(Show::class)->assertSee('Calculating…');
});
it('shows an invalid frequency instead of Calculating when the next run cannot be calculated', function () {
$task = ScheduledTask::factory()->create([
'team_id' => $this->team->id,
'application_id' => $this->application->id,
'frequency' => '0 2 * * *',
'enabled' => true,
]);
// The forms reject such a value, so only old data or a direct database change can have it.
ScheduledTask::query()->whereKey($task->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);
nextRunDisplayBindTaskRoute($this->application, $task);
Livewire::test(Show::class)
->assertSee('Invalid frequency')
->assertDontSee('Calculating…');
});
it('shows an invalid Docker cleanup frequency', function () {
ServerSetting::query()->where('server_id', $this->server->id)->update(['docker_cleanup_frequency' => '0 0 31 2 *', 'docker_cleanup_next_run_at' => null]);
Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])
->assertSee('Invalid frequency')
->assertDontSee('Calculating…');
});
it('shows the next database backup run in the server timezone', function () {
$database = StandalonePostgresql::create([
'name' => 'pg-next-run',
'image' => 'postgres:16-alpine',
'postgres_user' => 'postgres',
'postgres_password' => 'password',
'postgres_db' => 'postgres',
'environment_id' => $this->environment->id,
'destination_id' => $this->destination->id,
'destination_type' => $this->destination->getMorphClass(),
]);
$backup = ScheduledDatabaseBackup::create([
'frequency' => '0 2 * * *',
'enabled' => true,
'database_type' => $database->getMorphClass(),
'database_id' => $database->id,
'team_id' => $this->team->id,
]);
Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect()])
->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)
->call('toggleEnabled')
->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)
->assertSeeInOrder(['Next run', 'Disabled']);
});
it('shows the next volume backup run in the server timezone', function () {
$volume = LocalPersistentVolume::create([
'name' => 'app-data',
'mount_path' => '/data',
'resource_id' => $this->application->id,
'resource_type' => $this->application->getMorphClass(),
]);
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $this->application])
->assertSeeInOrder(['Next run', 'Disabled'])
->set('frequency', '0 2 * * *')
->call('toggleEnabled')
->assertSee('2026-09-18 02:00 (Europe/Budapest)', false);
});
tests/Feature/ScheduleOnOneServerTest.php
+12
View File
@@ -72,3 +72,15 @@ it('schedules stuck resource cleanup in the background once per day', function (
->and($event->withoutOverlapping)->toBeTrue()
->and($event->runInBackground)->toBeTrue();
});
it('runs the scheduled job dispatcher from the scheduler instead of a queue', function () {
$events = collect(app(Schedule::class)->events());
$event = $events->first(fn ($event) => str_contains((string) $event->command, 'scheduled:dispatch'));
expect($event)->not->toBeNull()
->and($event->expression)->toBe('* * * * *')
->and($event->onOneServer)->toBeTrue()
->and($event->runInBackground)->toBeTrue()
->and($events->contains(fn ($event) => str_contains((string) $event->description, 'ScheduledJobManager')))->toBeFalse();
});
tests/Feature/ScheduledJobManagerDispatchTest.php
+243 -299
View File
@@ -2,6 +2,7 @@
use App\Events\BackupCreated;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\ScheduledJobManager;
use App\Jobs\ScheduledTaskJob;
use App\Models\Application;
@@ -10,10 +11,11 @@ use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\Service;
use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
@@ -21,7 +23,6 @@ use App\Services\ScheduledJobDeliveryService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
@@ -31,173 +32,219 @@ uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
config(['constants.coolify.self_hosted' => true]);
Queue::fake();
});
it('dispatches scheduled tasks across chunks', function () {
config(['constants.coolify.self_hosted' => true]);
it('dispatches due scheduled tasks across chunks and moves their next run forward', function () {
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));
$application = createScheduledTaskApplication();
foreach (range(1, 101) as $ignored) {
createScheduledApplicationTask($application, ['frequency' => '* * * * *']);
}
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
Queue::fake();
$team = Team::factory()->create();
$privateKey = PrivateKey::create([
'name' => 'Test Key',
'private_key' => '-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk
hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA
AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
-----END OPENSSH PRIVATE KEY-----',
'team_id' => $team->id,
]);
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
]);
$server->settings()->update([
'is_reachable' => true,
'is_usable' => true,
'force_disabled' => false,
'docker_cleanup_frequency' => '0 * * * *',
]);
$destination = StandaloneDocker::where('server_id', $server->id)->first()
?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
$project = Project::factory()->create(['team_id' => $team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$application = Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => StandaloneDocker::class,
'status' => 'running',
]);
ScheduledTask::factory()
->count(101)
->create([
'team_id' => $team->id,
'application_id' => $application->id,
'frequency' => '* * * * *',
'enabled' => true,
]);
(new ScheduledJobManager)->handle();
Queue::assertPushed(ScheduledTaskJob::class, 101);
expect(ScheduledTask::query()->pluck('next_run_at')->map->toDateTimeString()->unique()->all())
->toBe(['2026-05-27 00:02:00']);
});
it('skips expensive dispatch for schedules outside the catch-up window', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
Queue::fake();
$application = createScheduledTaskApplication();
$task = ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => '0 2 * * *',
'enabled' => true,
]);
it('runs a new schedule first at its next due time', function () {
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 20, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => '* * * * *']);
(new ScheduledJobManager)->handle();
Queue::assertNotPushed(ScheduledTaskJob::class);
expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-05-27 00:02:00');
});
it('dispatches a recently missed daily task when deduplication cache is empty', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 10, 0, 'UTC'));
Queue::fake();
it('does not query schedules that are not due', function () {
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 0, 'UTC'));
$application = createScheduledTaskApplication();
$database = createScheduledBackupDatabase();
$countQueries = function (): int {
DB::flushQueryLog();
DB::enableQueryLog();
(new ScheduledJobManager)->handle();
$count = count(DB::getQueryLog());
DB::disableQueryLog();
ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => 'daily',
'enabled' => true,
]);
return $count;
};
// Initialize the Docker cleanup schedules of both servers, so only the schedules below change.
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
$countQueries();
$withoutSchedules = $countQueries();
(new ScheduledJobManager)->handle();
foreach (range(1, 5) as $ignored) {
createScheduledApplicationTask($application, ['frequency' => '0 2 * * *']);
createScheduledDatabaseBackup($database, ['frequency' => '0 2 * * *']);
}
Queue::assertPushed(ScheduledTaskJob::class, 1);
expect($countQueries())->toBe($withoutSchedules);
Queue::assertNothingPushed();
});
it('dispatches one job when multiple managers evaluate the same occurrence', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Queue::fake();
$application = createScheduledTaskApplication();
$task = ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => 'daily',
'enabled' => true,
]);
it('dispatches one job when several managers run for the same occurrence', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
(new ScheduledJobManager)->handle();
(new ScheduledJobManager)->handle();
Queue::assertPushed(ScheduledTaskJob::class, 1);
expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1)
->and(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1);
->and($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');
});
it('does not retry an occurrence skipped while its server is not functional', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Queue::fake();
it('does not dispatch an occurrence that another manager claims after this one read it', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
$application = createScheduledTaskApplication();
$task = createScheduledApplicationTask($application, ['frequency' => 'daily']);
// Another node claims the occurrence between this manager's select and its update.
$claimedByOtherNode = false;
ScheduledTask::retrieved(function (ScheduledTask $retrieved) use (&$claimedByOtherNode) {
if (! $claimedByOtherNode) {
$claimedByOtherNode = true;
ScheduledTask::query()->whereKey($retrieved->id)->toBase()->update(['next_run_at' => '2026-09-18 00:00:00']);
}
});
(new ScheduledJobManager)->handle();
Queue::assertNotPushed(ScheduledTaskJob::class);
expect($claimedByOtherNode)->toBeTrue()
->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();
});
it('calculates the next run of a schedule without one and runs it when it is due now', function (string $frequency, bool $runsNow, string $nextRunAt) {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 20, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => $frequency]);
ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);
(new ScheduledJobManager)->handle();
Queue::assertPushed(ScheduledTaskJob::class, $runsNow ? 1 : 0);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe($nextRunAt);
})->with([
'due in the current minute' => ['0 0 * * *', true, '2026-09-18 00:00:00'],
'due later' => ['30 * * * *', false, '2026-09-17 00:30:00'],
]);
it('moves the next run forward when an occurrence is skipped, and does not retry it', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
$server = $task->server();
$server->settings()->update(['is_reachable' => false]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
(new ScheduledJobManager)->handle();
$server->settings()->update(['is_reachable' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 6, 0, 'UTC'));
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 1, 0, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertNotPushed(ScheduledTaskJob::class);
expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->value('last_scheduled_for'))
->not->toBeNull()
->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())
->toBeFalse();
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')
->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();
});
it('does not publish a task occurrence when its application is not running', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Queue::fake();
it('does not dispatch a task when its application is not running', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$application = createScheduledTaskApplication();
$application->update(['status' => 'stopped']);
$task = createScheduledApplicationTask($application, ['frequency' => 'daily']);
$logPath = captureScheduledJobManagerTestLog('scheduled');
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertNotPushed(ScheduledTaskJob::class);
expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeTrue()
->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())
->toBeFalse();
expect(file_get_contents($logPath))->toContain('Task skipped')->toContain('"skip_reason":"application_not_running"');
@unlink($logPath);
});
it('runs late backups once, but logs late tasks outside the late-run window as missed', function (int $minutesLate, bool $taskRuns) {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
$logPath = captureScheduledJobManagerTestLog('scheduled');
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')->addMinutes($minutesLate));
(new ScheduledJobManager)->handle();
Queue::assertPushed(ScheduledTaskJob::class, $taskRuns ? 1 : 0);
Queue::assertPushed(DatabaseBackupJob::class, 1);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')
->and($backup->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')
->and(str_contains(file_get_contents($logPath), 'Task missed'))->toBe(! $taskRuns);
@unlink($logPath);
})->with([
'five minutes late' => [5, true],
'three hours late' => [180, false],
]);
it('runs a daily schedule once on the night the clocks go back', function () {
Carbon::setTestNow(Carbon::create(2026, 10, 24, 12, 0, 0, 'UTC'));
$application = createScheduledTaskApplication();
$application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);
$task = createScheduledApplicationTask($application, ['frequency' => '30 2 * * *']);
// 02:30 CEST. At 03:00 CEST the clocks go back to 02:00, so 02:30 CET follows one hour later.
Carbon::setTestNow(Carbon::create(2026, 10, 25, 0, 30, 5, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertPushed(ScheduledTaskJob::class, 1);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-10-26 01:30:00');
});
it('dispatches due Docker cleanups with their settings', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));
$server = createScheduledTaskApplication()->destination->server;
$server->settings->update(['docker_cleanup_frequency' => '0 * * * *', 'delete_unused_volumes' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 1, 0, 10, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertPushed(DockerCleanupJob::class, fn (DockerCleanupJob $job) => $job->server->is($server) && $job->deleteUnusedVolumes);
expect($server->settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');
});
it('does not dispatch an occurrence that already has a delivery', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
ScheduledJobDelivery::create([
'schedule_key' => "scheduled-task:{$task->id}",
'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),
'job_type' => 'scheduled-task',
'resource_id' => $task->id,
'status' => 'claimed',
]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertNotPushed(ScheduledTaskJob::class);
expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');
});
it('dispatches a due backup of a service database', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 4, 30, 'UTC'));
$backup = createScheduledServiceDatabaseBackup(['frequency' => '* * * * *']);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->is($backup));
});
it('publishes a pending occurrence after a previous publisher interruption', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
$application = createScheduledTaskApplication();
$task = ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => 'daily',
'enabled' => true,
]);
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
$occurrence = ScheduledJobDelivery::create([
'schedule_key' => "scheduled-task:{$task->id}",
'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),
@@ -211,6 +258,37 @@ it('publishes a pending occurrence after a previous publisher interruption', fun
expect($occurrence->fresh()->status)->toBe('enqueued');
});
it('keeps publishing pending occurrences after one of them fails', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
$logPath = captureScheduledJobManagerTestLog('scheduled-errors');
$database = createScheduledBackupDatabase();
$occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])
->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([
'schedule_key' => "scheduled-backup:{$backup->id}",
'scheduled_for' => now()->startOfMinute(),
'job_type' => 'database-backup',
'resource_id' => $backup->id,
'status' => 'pending',
]));
$dispatchAttempts = 0;
Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {
if (++$dispatchAttempts === 1) {
throw new RuntimeException('queue connection lost');
}
});
app(ScheduledJobDeliveryService::class)->publishPending();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($occurrences->first()->fresh()->status)->toBe('pending')
->and($occurrences->last()->fresh()->status)->toBe('enqueued')
->and($log)->toContain('Failed to publish pending scheduled occurrence')
->toContain('queue connection lost');
});
it('allows only one worker to claim an occurrence', function () {
$occurrence = ScheduledJobDelivery::create([
'schedule_key' => 'scheduled-task:claim-test',
@@ -274,30 +352,8 @@ it('marks stale claimed deliveries as failed', function () {
expect($delivery->fresh()->status)->toBe('failed');
});
it('dispatches the instance coolify-db backup even when its id is zero', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
Queue::fake();
$database = createScheduledBackupDatabase();
$backup = createScheduledDatabaseBackup($database, [
'id' => 0,
'frequency' => '* * * * *',
]);
expect($backup->id)->toBe(0);
(new ScheduledJobManager)->handle();
Queue::assertPushed(DatabaseBackupJob::class, 1);
Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->id === 0);
});
it('dispatches zero-id schedules and continues with positive ids', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
Queue::fake();
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));
$application = createScheduledTaskApplication();
$database = StandalonePostgresql::create([
'name' => 'coolify-db',
@@ -310,17 +366,14 @@ it('dispatches zero-id schedules and continues with positive ids', function () {
'destination_id' => $application->destination_id,
'destination_type' => $application->destination_type,
]);
$zeroIdBackup = createScheduledDatabaseBackup($database, ['id' => 0]);
$positiveIdBackup = createScheduledDatabaseBackup($database);
$zeroIdTask = createScheduledApplicationTask($application, ['id' => 0]);
$positiveIdTask = createScheduledApplicationTask($application);
expect($zeroIdBackup->id)->toBe(0)
->and($positiveIdBackup->id)->toBeGreaterThan(0)
->and($zeroIdTask->id)->toBe(0)
->and($positiveIdTask->id)->toBeGreaterThan(0);
expect($zeroIdBackup->id)->toBe(0)->and($zeroIdTask->id)->toBe(0);
Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));
(new ScheduledJobManager)->handle();
Queue::assertPushed(DatabaseBackupJob::class, 2);
@@ -332,14 +385,7 @@ it('dispatches zero-id schedules and continues with positive ids', function () {
});
it('does not query relationships when constructing scheduled task jobs', function () {
$application = createScheduledTaskApplication();
$task = ScheduledTask::factory()->create([
'team_id' => $application->environment->project->team_id,
'application_id' => $application->id,
'frequency' => '* * * * *',
'enabled' => true,
])->fresh();
$task = createScheduledApplicationTask(createScheduledTaskApplication());
DB::flushQueryLog();
DB::enableQueryLog();
@@ -351,56 +397,32 @@ it('does not query relationships when constructing scheduled task jobs', functio
->and($job->timeout)->toBe(300);
});
it('reads current server settings in each queued manager run of the same worker', function () {
config(['constants.coolify.self_hosted' => true]);
Queue::fakeExcept(ScheduledJobManager::class);
it('reads current server settings in each manager run of the same process', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 15, 12, 0, 0, 'UTC'));
$database = createScheduledBackupDatabase();
createScheduledDatabaseBackup($database, ['frequency' => 'daily']);
$serverId = $database->destination->server_id;
ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => false]);
Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));
dispatch(new ScheduledJobManager);
Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 0, 5, 'UTC'));
$this->artisan('scheduled:dispatch')->assertSuccessful();
Queue::assertNotPushed(DatabaseBackupJob::class);
// Another worker marks the server reachable again before the next daily run.
ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
dispatch(new ScheduledJobManager);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));
$this->artisan('scheduled:dispatch')->assertSuccessful();
Queue::assertPushed(DatabaseBackupJob::class, 1);
});
it('logs the current database server state when a backup is skipped', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Queue::fake();
$logPath = captureScheduledJobManagerTestLog('scheduled');
$database = createScheduledBackupDatabase();
$backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);
ServerSetting::query()->where('server_id', $database->destination->server_id)->update(['is_reachable' => false]);
(new ScheduledJobManager)->handle();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($log)->toContain('Backup skipped')
->toContain('"skip_reason":"server_not_functional"')
->toContain("\"backup_id\":{$backup->id}")
->toContain('"db_is_reachable":false');
});
it('logs each step of a scheduled occurrence', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Queue::fake();
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
$logPath = captureScheduledJobManagerTestLog('scheduled');
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
(new ScheduledJobManager)->handle();
$occurrenceUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');
$service = app(ScheduledJobDeliveryService::class);
@@ -415,7 +437,6 @@ it('logs each step of a scheduled occurrence', function () {
expect($log)->toContain('Backup dispatched')
->toContain('Scheduled occurrence enqueued')
->toContain('"occurrence_uuid":"'.$occurrenceUuid.'"')
->toContain('"host":"'.gethostname().'"')
->toContain('Scheduled occurrence claimed')
->toContain('Scheduled occurrence claim rejected')
->toContain('Scheduled occurrence completed')
@@ -423,128 +444,23 @@ it('logs each step of a scheduled occurrence', function () {
->toContain('"seconds_since_due":300');
});
it('warns when the manager starts after the catch-up window', function (int $minutesSincePreviousStart, bool $expectWarning) {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));
Queue::fake();
$logPath = captureScheduledJobManagerTestLog('scheduled-errors');
Cache::put('scheduled-job-manager:last-started-at', now()->subMinutes($minutesSincePreviousStart)->toIso8601String());
(new ScheduledJobManager)->handle();
$log = file_get_contents($logPath);
@unlink($logPath);
expect(str_contains($log, 'ScheduledJobManager started late'))->toBe($expectWarning);
})->with([
'normal one-minute interval' => [1, false],
'twenty-minute gap' => [20, true],
]);
it('warns when an occurrence is enqueued while the previous one is still running', function () {
config(['constants.coolify.self_hosted' => true]);
Queue::fake();
$logPath = captureScheduledJobManagerTestLog('scheduled');
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));
(new ScheduledJobManager)->handle();
$previousUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');
app(ScheduledJobDeliveryService::class)->claim($previousUuid, 'worker-a');
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
(new ScheduledJobManager)->handle();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($log)->toContain('Scheduled occurrence enqueued while the previous occurrence is still open')
->toContain('"previous_occurrence_uuid":"'.$previousUuid.'"')
->toContain('"previous_status":"claimed"');
});
it('reports occurrences that stay open after fifteen minutes', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));
Queue::fake();
$logPath = captureScheduledJobManagerTestLog('scheduled');
foreach (['enqueued' => 20, 'claimed' => 20, 'failed' => 20] as $status => $minutesAgo) {
ScheduledJobDelivery::create([
'schedule_key' => "scheduled-backup:{$status}",
'scheduled_for' => now()->subMinutes($minutesAgo),
'job_type' => 'database-backup',
'resource_id' => 1,
'status' => $status,
])->forceFill(['created_at' => now()->subMinutes($minutesAgo)])->save();
}
ScheduledJobDelivery::create([
'schedule_key' => 'scheduled-backup:recent',
'scheduled_for' => now()->subMinutes(2),
'job_type' => 'database-backup',
'resource_id' => 1,
'status' => 'enqueued',
]);
(new ScheduledJobManager)->handle();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($log)->toContain('"open_occurrences_over_15_minutes":{"claimed":1,"enqueued":1}');
});
it('keeps publishing pending occurrences after one of them fails', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
$logPath = captureScheduledJobManagerTestLog('scheduled-errors');
$database = createScheduledBackupDatabase();
$occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])
->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([
'schedule_key' => "scheduled-backup:{$backup->id}",
'scheduled_for' => now()->startOfMinute(),
'job_type' => 'database-backup',
'resource_id' => $backup->id,
'status' => 'pending',
]));
$dispatchAttempts = 0;
Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {
if (++$dispatchAttempts === 1) {
throw new RuntimeException('queue connection lost');
}
});
app(ScheduledJobDeliveryService::class)->publishPending();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($occurrences->first()->fresh()->status)->toBe('pending')
->and($occurrences->last()->fresh()->status)->toBe('enqueued')
->and($log)->toContain('Failed to publish pending scheduled occurrence')
->toContain('queue connection lost');
});
it('logs a backup run that ends early, even when it completes before the enqueue log', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));
Event::fake([BackupCreated::class]);
Queue::fakeExcept([ScheduledJobManager::class, DatabaseBackupJob::class]);
$logPath = captureScheduledJobManagerTestLog('scheduled');
Queue::fakeExcept([DatabaseBackupJob::class]);
$database = createScheduledBackupDatabase();
$database->update(['status' => 'exited']);
$backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);
$logPath = captureScheduledJobManagerTestLog('scheduled');
dispatch(new ScheduledJobManager);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));
$this->artisan('scheduled:dispatch')->assertSuccessful();
$log = file_get_contents($logPath);
@unlink($logPath);
expect($log)->toContain('Database backup job ended without a backup')
->toContain('"skip_reason":"database_not_running"')
->toContain('"database_status":"exited:unhealthy"')
->toContain('Scheduled occurrence completed')
->toMatch('/Scheduled occurrence enqueued \{[^\n]*"schedule_key":"scheduled-backup:'.$backup->id.'"[^\n]*"status":"enqueued"/')
->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->exists())->toBeFalse();
@@ -630,6 +546,34 @@ function createScheduledDatabaseBackup(StandalonePostgresql $database, array $ov
return $backup->fresh();
}
function createScheduledServiceDatabaseBackup(array $overrides = []): ScheduledDatabaseBackup
{
$application = createScheduledTaskApplication();
$service = Service::factory()->create([
'environment_id' => $application->environment_id,
'destination_id' => $application->destination_id,
'destination_type' => $application->destination_type,
]);
$database = ServiceDatabase::create([
'name' => 'postgres',
'image' => 'postgres:16-alpine',
'service_id' => $service->id,
]);
$backup = new ScheduledDatabaseBackup;
$backup->forceFill(array_merge([
'enabled' => true,
'save_s3' => false,
'frequency' => '* * * * *',
'database_id' => $database->id,
'database_type' => $database->getMorphClass(),
'team_id' => $application->environment->project->team_id,
], $overrides));
$backup->save();
return $backup->fresh();
}
function createScheduledApplicationTask(Application $application, array $overrides = []): ScheduledTask
{
$task = new ScheduledTask;
tests/Feature/VolumeBackupTest.php
+4 -2
View File
@@ -2519,9 +2519,9 @@ it('dispatches due scheduled volume backups', function () {
'frequency' => '* * * * *',
'enabled' => true,
]);
Cache::forget("scheduled-volume-backup:{$backup->id}");
expect($backup->server()?->isFunctional())->toBeTrue();
$this->travel(1)->minutes();
(new ScheduledJobManager)->handle();
Queue::assertPushed(
@@ -2547,8 +2547,8 @@ it('dispatches due scheduled directory backups', function () {
'frequency' => '* * * * *',
'enabled' => true,
]);
Cache::forget("scheduled-volume-backup:{$backup->id}");
$this->travel(1)->minutes();
(new ScheduledJobManager)->handle();
Queue::assertPushed(
@@ -2577,6 +2577,7 @@ it('retains scheduled volume backups and archive metadata when the server is mis
'destination_type' => null,
]);
$this->travel(1)->minutes();
(new ScheduledJobManager)->handle();
expect($backup->fresh())->not->toBeNull()
@@ -2606,6 +2607,7 @@ it('dispatches pending recovery without starting another volume backup', functio
'stop_recovery_pending' => true,
]);
$this->travel(1)->minutes();
(new ScheduledJobManager)->handle();
Queue::assertPushed(
tests/Unit/ScheduledJobManagerLockTest.php
-63
View File
@@ -1,63 +0,0 @@
<?php
use App\Jobs\ScheduledJobManager;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Tests\TestCase;
uses(TestCase::class);
it('uses WithoutOverlapping middleware with expireAfter to prevent stale locks', function () {
$job = new ScheduledJobManager;
$middleware = $job->middleware();
// Assert middleware exists
expect($middleware)->toBeArray()
->and($middleware)->toHaveCount(1);
$overlappingMiddleware = $middleware[0];
// Assert it's a WithoutOverlapping instance
expect($overlappingMiddleware)->toBeInstanceOf(WithoutOverlapping::class);
// Use reflection to check private properties
$reflection = new ReflectionClass($overlappingMiddleware);
// Check expireAfter is set (should be 60 seconds - matches job frequency)
$expiresAfterProperty = $reflection->getProperty('expiresAfter');
$expiresAfterProperty->setAccessible(true);
$expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);
expect($expiresAfter)->toBe(90)
->and($expiresAfter)->toBeGreaterThan(0, 'expireAfter must be set to prevent stale locks');
// Check releaseAfter is NOT set (we use dontRelease)
$releaseAfterProperty = $reflection->getProperty('releaseAfter');
$releaseAfterProperty->setAccessible(true);
$releaseAfter = $releaseAfterProperty->getValue($overlappingMiddleware);
expect($releaseAfter)->toBeNull('releaseAfter should be null when using dontRelease()');
// Check the lock key
$keyProperty = $reflection->getProperty('key');
$keyProperty->setAccessible(true);
$key = $keyProperty->getValue($overlappingMiddleware);
expect($key)->toBe('scheduled-job-manager');
});
it('prevents stale locks by ensuring expireAfter is always set', function () {
$job = new ScheduledJobManager;
$middleware = $job->middleware();
$overlappingMiddleware = $middleware[0];
$reflection = new ReflectionClass($overlappingMiddleware);
$expiresAfterProperty = $reflection->getProperty('expiresAfter');
$expiresAfterProperty->setAccessible(true);
$expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);
// Critical check: expireAfter MUST be set to prevent GitHub issue #4539
expect($expiresAfter)->not->toBeNull(
'expireAfter() is required to prevent stale locks (see GitHub #4539)'
);
});
tests/Unit/ServerPlaceholderIpTest.php
+8 -3
View File
@@ -4,8 +4,10 @@ use App\Jobs\CleanupOrphanedPreviewContainersJob;
use App\Jobs\ScheduledJobManager;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
uses(TestCase::class, RefreshDatabase::class);
@@ -99,10 +101,13 @@ it('excludes every placeholder address from scheduled Docker cleanup', function
]);
}
$method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');
$servers = $method->invoke(new ScheduledJobManager)->get();
ServerSetting::query()->update(['docker_cleanup_next_run_at' => null]);
Queue::fake();
expect($servers->modelKeys())->toBe([$realServer->id]);
(new ScheduledJobManager)->handle();
// The dispatcher calculates the next cleanup only for servers that it selects.
expect(ServerSetting::query()->whereNotNull('docker_cleanup_next_run_at')->pluck('server_id')->all())->toBe([$realServer->id]);
});
it('excludes every placeholder address from orphaned preview cleanup', function () {