Merge pull request #70 from stackkit/feature/job-releasing

Job releasing

Author: marickvantuil

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 3.?.0 - ????-??-??
+
+**Added**
+
+- Jobs can now be released back onto the queue.
+
 ## 3.2.1 - 2022-09-02
 
 **Fixed**

README.md

@@ -56,6 +56,7 @@ Please check the table below for supported Laravel and PHP versions:
       // does not respond by this deadline then the request is cancelled and the attempt
       // is marked as a DEADLINE_EXCEEDED failure.
       'dispatch_deadline' => null,
+      'backoff' => 0,
   ],
   ```
 
@@ -79,13 +80,19 @@ Please check the table below on what the values mean and what their value should
 </details>
 <details>
 <summary>
-  How it works
+  How it works & Differences
 </summary>
   <br>
   Using Cloud Tasks as a Laravel queue driver is fundamentally different than other Laravel queue drivers, like Redis.
 
 Typically a Laravel queue has a worker that listens to incoming jobs using the `queue:work` / `queue:listen` command.
 With Cloud Tasks, this is not the case. Instead, Cloud Tasks will schedule the job for you and make an HTTP request to your application with the job payload. There is no need to run a `queue:work/listen` command.
+
+#### Good to know
+
+- The "Min backoff" and "Max backoff" options in Cloud Tasks are ignored. This is intentional: Laravel has its own backoff feature (which is more powerful than what Cloud Tasks offers) and therefore I have chosen that over the Cloud Tasks one.
+- Similarly to the backoff feature, I have also chosen to let the package do job retries the 'Laravel way'. In Cloud Tasks, when a task throws an exception, Cloud Tasks will decide for itself when to retry the task (based on the backoff values). It will also manage its own state and knows how many times a task has been retried. This is different from Laravel. In typical Laravel queues, when a job throws an exception, the job is deleted and released back onto the queue. In order to support Laravel's backoff feature, this package must behave the same way about job retries.
+
 </details>
 <details>
   <summary>Dashboard (beta)</summary>

composer.json

@@ -36,5 +36,23 @@
                 "Stackkit\\LaravelGoogleCloudTasksQueue\\CloudTasksServiceProvider"
             ]
         }
+    },
+    "scripts": {
+        "l9": [
+            "composer require laravel/framework:9.* orchestra/testbench:7.* --no-interaction --no-update",
+            "composer update --prefer-stable --prefer-dist --no-interaction --no-suggest"
+        ],
+        "l8": [
+            "composer require laravel/framework:8.* orchestra/testbench:6.* --no-interaction --no-update",
+            "composer update --prefer-stable --prefer-dist --no-interaction --no-suggest"
+        ],
+        "l7": [
+            "composer require laravel/framework:7.* orchestra/testbench:5.* --no-interaction --no-update",
+            "composer update --prefer-stable --prefer-dist --no-interaction --no-suggest"
+        ],
+        "l6": [
+            "composer require laravel/framework:6.* orchestra/testbench:4.* --no-interaction --no-update",
+            "composer update --prefer-stable --prefer-dist --no-interaction --no-suggest"
+        ]
     }
 }

dashboard/dist/assets/index.1002db9a.css renamed to dashboard/dist/assets/index.d8eef428.css

@@ -5,9 +5,9 @@
     <link rel="icon" href="/favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vite App</title>
-    <script type="module" crossorigin src="/assets/index.5a46c6a0.js"></script>
+    <script type="module" crossorigin src="/assets/index.ea68d73f.js"></script>
     <link rel="modulepreload" href="/assets/vendor.433de25e.js">
-    <link rel="stylesheet" href="/assets/index.1002db9a.css">
+    <link rel="stylesheet" href="/assets/index.d8eef428.css">
   </head>
   <body class="bg-gray-100">
     <div id="app"></div>

dashboard/dist/assets/index.5a46c6a0.js renamed to dashboard/dist/assets/index.ea68d73f.js

@@ -1,13 +1,13 @@
 {
   "index.html": {
-    "file": "assets/index.5a46c6a0.js",
+    "file": "assets/index.ea68d73f.js",
     "src": "index.html",
     "isEntry": true,
     "imports": [
       "_vendor.433de25e.js"
     ],
     "css": [
-      "assets/index.1002db9a.css"
+      "assets/index.d8eef428.css"
     ]
   },
   "_vendor.433de25e.js": {

dashboard/dist/crossword.png

@@ -31,7 +31,7 @@ function ucfirst(input) {
 .task-queued, .task-scheduled {
   @apply bg-gray-100 text-gray-500
 }
-.task-running {
+.task-running, .task-released {
   @apply bg-blue-100 text-blue-800
 }
 </style>

dashboard/dist/dot-grid.png

@@ -27,6 +27,7 @@ const titles = {
   successful: 'Successful',
   error: 'An error occurred',
   failed: 'Failed permanently',
+  released: 'Released',
 }
 </script>
 
@@ -60,6 +61,13 @@ const titles = {
                   Scheduled: {{ event['scheduled_at'] }} (UTC)
                 </span>
               </div>
+              <div v-if="event['delay']">
+                <span
+                  class="bg-gray-200 text-gray-800 text-xs font-medium mr-2 inline-block mb-1 px-1.5 py-0.5 rounded dark:bg-blue-200 dark:text-blue-800"
+                >
+                  Delay: {{ event['delay'] }} seconds
+                </span>
+              </div>
             </h3>
             <Popper
               :content="event.datetime"

dashboard/dist/index.html

@@ -10,10 +10,7 @@
          stopOnFailure="false">
     <testsuites>
         <testsuite name="Testsuite">
-            <file>./tests/ConfigTest.php</file>
-            <file>./tests/TaskHandlerTest.php</file>
-            <file>./tests/CloudTasksApiTest.php</file>
-            <file>./tests/CloudTasksDashboardTest.php</file>
+            <directory suffix="Test.php">./tests</directory>
         </testsuite>
     </testsuites>
     <php>

dashboard/dist/manifest.json

@@ -91,4 +91,9 @@ public function assertTaskCreated(Closure $closure): void
 
         Assert::assertTrue($count > 0, 'Task was not created.');
     }
+
+    public function assertCreatedTaskCount(int $count): void
+    {
+        Assert::assertCount($count, $this->createdTasks);
+    }
 }

dashboard/dist/pw_maze_white.png

@@ -3,14 +3,21 @@
 namespace Stackkit\LaravelGoogleCloudTasksQueue;
 
 use Illuminate\Container\Container;
-use Illuminate\Queue\Jobs\Job as LaravelJob;
 use Illuminate\Contracts\Queue\Job as JobContract;
+use Illuminate\Queue\Jobs\Job as LaravelJob;
+use Stackkit\LaravelGoogleCloudTasksQueue\Events\JobReleased;
+use Stackkit\LaravelGoogleCloudTasksQueue\Events\JobReleasedAfterException;
 use function Safe\json_encode;
 
 class CloudTasksJob extends LaravelJob implements JobContract
 {
-    private array $job;
-    private ?int $attempts;
+    /**
+     * The Cloud Tasks raw job payload (request payload).
+     *
+     * @var array
+     */
+    public array $job;
+
     private ?int $maxTries;
     public ?int $retryUntil = null;
 
@@ -29,6 +36,11 @@ public function __construct(array $job, CloudTasksQueue $cloudTasksQueue)
         $this->queue = $command['queue'] ?? config('queue.connections.' .config('queue.default') . '.queue');
     }
 
+    public function job()
+    {
+        return $this->job;
+    }
+
     public function getJobId(): string
     {
         return $this->job['uuid'];
@@ -46,12 +58,12 @@ public function getRawBody(): string
 
     public function attempts(): ?int
     {
-        return $this->attempts;
+        return $this->job['internal']['attempts'];
     }
 
     public function setAttempts(int $attempts): void
     {
-        $this->attempts = $attempts;
+        $this->job['internal']['attempts'] = $attempts;
     }
 
     public function setMaxTries(int $maxTries): void
@@ -95,4 +107,28 @@ public function delete(): void
 
         $this->cloudTasksQueue->delete($this);
     }
+
+    public function release($delay = 0)
+    {
+        parent::release();
+
+        $this->cloudTasksQueue->release($this, $delay);
+
+        $properties = TaskHandler::getCommandProperties($this->job['data']['command']);
+        $connection = $properties['connection'] ?? config('queue.default');
+
+        // The package uses the JobReleasedAfterException provided by Laravel to grab
+        // the payload of the released job in tests to easily run and test a released
+        // job. Because the event is only accessible in Laravel 9.x, we create an
+        // identical event to hook into for Laravel versions older than 9.x
+        if (version_compare(app()->version(), '9.0.0', '<')) {
+            if (data_get($this->job, 'internal.errored')) {
+                app('events')->dispatch(new JobReleasedAfterException($connection, $this));
+            }
+        }
+
+        if (! data_get($this->job, 'internal.errored')) {
+            app('events')->dispatch(new JobReleased($connection, $this, $delay));
+        }
+    }
 }

dashboard/src/components/Status.vue

@@ -12,8 +12,9 @@
 use Illuminate\Contracts\Queue\Queue as QueueContract;
 use Illuminate\Queue\Queue as LaravelQueue;
 use Illuminate\Support\Str;
-use function Safe\json_encode;
+use Stackkit\LaravelGoogleCloudTasksQueue\Events\TaskCreated;
 use function Safe\json_decode;
+use function Safe\json_encode;
 
 class CloudTasksQueue extends LaravelQueue implements QueueContract
 {
@@ -93,7 +94,9 @@ function ($payload, $queue) {
      */
     public function pushRaw($payload, $queue = null, array $options = [])
     {
-        return $this->pushToCloudTasks($queue, $payload);
+        $delay = ! empty($options['delay']) ? $options['delay'] : 0;
+
+        $this->pushToCloudTasks($queue, $payload, $delay);
     }
 
     /**
@@ -141,11 +144,19 @@ protected function pushToCloudTasks($queue, $payload, $delay = 0)
         // we will add it manually here if it's not present yet.
         [$payload, $uuid] = $this->withUuid($payload);
 
+        // Since 3.x tasks are released back onto the queue after an exception has
+        // been thrown. This means we lose the native [X-CloudTasks-TaskRetryCount] header
+        // value and need to manually set and update the number of times a task has been attempted.
+        $payload = $this->withAttempts($payload);
+
         $httpRequest->setBody($payload);
 
         $task = $this->createTask();
         $task->setHttpRequest($httpRequest);
 
+        // The deadline for requests sent to the app. If the app does not respond by
+        // this deadline then the request is cancelled and the attempt is marked as
+        // a failure. Cloud Tasks will retry the task according to the RetryConfig.
         if (!empty($this->config['dispatch_deadline'])) {
             $task->setDispatchDeadline(new Duration(['seconds' => $this->config['dispatch_deadline']]));
         }
@@ -180,6 +191,18 @@ private function withUuid(string $payload): array
         ];
     }
 
+    private function withAttempts(string $payload): string
+    {
+        /** @var array $decoded */
+        $decoded = json_decode($payload, true);
+
+        if (!isset($decoded['internal']['attempts'])) {
+            $decoded['internal']['attempts'] = 0;
+        }
+
+        return json_encode($decoded);
+    }
+
     /**
      * Pop the next job off of the queue.
      *
@@ -217,6 +240,17 @@ public function delete(CloudTasksJob $job): void
         CloudTasksApi::deleteTask($taskName);
     }
 
+    public function release(CloudTasksJob $job, int $delay = 0): void
+    {
+        $job->delete();
+
+        $payload = $job->getRawBody();
+
+        $options = ['delay' => $delay];
+
+        $this->pushRaw($payload, $job->getQueue(), $options);
+    }
+
     private function createTask(): Task
     {
         return app(Task::class);

dashboard/src/components/Task.vue

@@ -8,6 +8,8 @@
 use Illuminate\Queue\Events\JobProcessed;
 use Illuminate\Queue\Events\JobProcessing;
 use Illuminate\Support\ServiceProvider as LaravelServiceProvider;
+use Stackkit\LaravelGoogleCloudTasksQueue\Events\JobReleased;
+use Stackkit\LaravelGoogleCloudTasksQueue\Events\TaskCreated;
 use function Safe\file_get_contents;
 use function Safe\json_decode;
 
@@ -158,6 +160,8 @@ private function registerDashboard(): void
         });
 
         app('events')->listen(JobProcessed::class, function (JobProcessed $event) {
+            data_set($event->job->job, 'internal.processed', true);
+
             if (!CloudTasks::dashboardEnabled()) {
                 return;
             }
@@ -168,6 +172,8 @@ private function registerDashboard(): void
         });
 
         app('events')->listen(JobExceptionOccurred::class, function (JobExceptionOccurred $event) {
+            data_set($event->job->job, 'internal.errored', true);
+
             if (!CloudTasks::dashboardEnabled()) {
                 return;
             }
@@ -182,5 +188,13 @@ private function registerDashboard(): void
 
             DashboardService::make()->markAsFailed($event);
         });
+
+        app('events')->listen(JobReleased::class, function (JobReleased $event) {
+            if (!CloudTasks::dashboardEnabled()) {
+                return;
+            }
+
+            DashboardService::make()->markAsReleased($event);
+        });
     }
 }