magento/magento2#23054

app/code/Magento/Cron/Observer/ProcessCronQueueObserver.php

@@ -309,7 +309,12 @@ protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule,
             );
         }
 
-        $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save();
+        $schedule
+            ->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))
+            ->setProcessId(getmypid())
+            ->setProcessHostname(gethostname())
+            ->setProcessStartedAt(strftime('%Y-%m-%d %H:%M:%S', $currentTime))
+            ->save();
 
         $this->startProfiling();
         try {
@@ -404,6 +409,21 @@ private function getPendingSchedules($groupId)
         return $pendingJobs;
     }
 
+    /**
+     * Return job collection from data base with status 'running'.
+     *
+     * @param string $groupId
+     * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection
+     */
+    public function getRunningJobs($groupId)
+    {
+        $jobs = $this->_config->getJobs();
+        $runningJobs = $this->_scheduleFactory->create()->getCollection();
+        $runningJobs->addFieldToFilter('status', Schedule::STATUS_RUNNING);
+        $runningJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]);
+        return $runningJobs;
+    }
+
     /**
      * Return job collection from database with status 'pending', 'running' or 'success'
      *
@@ -505,6 +525,18 @@ protected function _generateJobs($jobs, $exists, $groupId)
      */
     private function cleanupJobs($groupId, $currentTime)
     {
+        //remove orphan jobs marked as running
+        try {
+            $this->cleanupOrphanJobs($groupId);
+        } catch (\Exception $e) {
+            if ($this->state->getMode() === State::MODE_DEVELOPER
+            ) {
+                $this->logger->info($e->getMessage());
+            } else {
+                $this->logger->critical($e);
+            }
+        }
+
         // check if history cleanup is needed
         $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId);
         $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY);
@@ -667,6 +699,151 @@ private function cleanupDisabledJobs($groupId)
         }
     }
 
+    /**
+     * Clean up orphan (running) jobs that somehow lost their process in the server
+     *
+     * This can happen when cron process die on the server during runtime (eg. server restart, crash)
+     *
+     * @param string $groupId
+     * @return void
+     */
+    public function cleanupOrphanJobs($groupId)
+    {
+        $runningJobs = $this->getRunningJobs($groupId);
+
+        if ($runningJobs) {
+            $count = 0;
+
+            foreach ($runningJobs as $runningJob) {
+                if ($this->getCheckHostnameConfigurationValue() &&
+                    $runningJob->getProcessHostname() != gethostname()) {
+                    //job run/ran on a different host but do not need to check
+                    continue;
+                }
+
+                //to avoid shell_exec output truncate using a temporary file to catch it
+                $tempFile = tmpfile();
+                fwrite(
+                    $tempFile,
+                    shell_exec("ps -eo pid,lstart,cmd | grep --color=none " . $runningJob->getProcessId())
+                );
+                fseek($tempFile, 0);
+                $execOutput = explode(
+                    "\n",
+                    fread($tempFile, 1024)
+                );
+                fclose($tempFile);
+
+                if ($this->processRunningJob($runningJob, $execOutput) === true) {
+                    $count++;
+                }
+            }
+
+            if ($count) {
+                $this->logger->info(
+                    sprintf(
+                        '%d cron jobs seems to got stuck in group %s and were cleaned',
+                        $count,
+                        $groupId
+                    )
+                );
+            }
+        }
+    }
+
+    /**
+     * Processing a running Schedule object depending of the output of a ps command
+     *
+     * @param Schedule $runningJob
+     * @param array $execOutput
+     * @return boolean has Schedule been processed?
+     */
+    private function processRunningJob(Schedule $runningJob, array $execOutput)
+    {
+        foreach ($execOutput as $line) {
+            if (!$line) {
+                continue;
+            }
+            $line = preg_split('/\s+/', trim($line), 7);
+            if ((int) trim($line[0]) == $runningJob->getProcessId() &&
+                $isCron /* !!!assigning value!!!*/ = $this->isCronCommand(trim($line[6]))) {
+                $processStartTime = strftime(
+                    '%Y-%m-%d %H:%M:%S',
+                    strtotime(
+                        implode(
+                            ' ',
+                            array_map(
+                                function ($value, $key) {
+                                    if ($key > 1 && $key < 6) {
+                                        return $value;
+                                    }
+                                },
+                                $line,
+                                array_keys($line)
+                            )
+                        )
+                    )
+                );
+
+                if ($processStartTime == $runningJob->getProcessStartedAt()) {
+                    return false;
+                } else {
+                    $this->closeOrphanJob(
+                        $runningJob,
+                        $isCron
+                            ? "Mismatching process starting time"
+                            : "Process exists but already not aProcess already not cron job magento cron job"
+                    );
+                    return true;
+                }
+            }
+        }
+        $this->closeOrphanJob(
+            $runningJob,
+            "Process not running on server"
+        );
+        return true;
+    }
+
+    /**
+     * Check if command is one of Magento2 cron processes
+     *
+     * @param string $command
+     * @return boolean
+     */
+    public function isCronCommand(string $command)
+    {
+        if (preg_match('(magento.+cron\:run|magento2\/update\/cron.php)', $command) === 1) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Mark orphan job as ended with error
+     *
+     * This can happen when cron process die on the server during runtime (eg. server restart)
+     *
+     * @param Schedule $orphanJob
+     * @param string $message
+     * @return Schedule
+     */
+    protected function closeOrphanJob($orphanJob, string $message = "")
+    {
+        return $orphanJob
+            ->setStatus(Schedule::STATUS_ERROR)
+            ->setMessages(
+                $message
+                    ?
+                    : sprintf(
+                        "Owner process (%d) not available on host (%s) any more",
+                        $orphanJob->getProcessId(),
+                        $orphanJob->getProcessHostname()
+                    )
+            )
+            ->save();
+    }
+
     /**
      * Get cron expression of cron job.
      *
@@ -727,6 +904,19 @@ private function getCronGroupConfigurationValue($groupId, $path)
         );
     }
 
+    /**
+     * Get CheckHost Configuration Value.
+     *
+     * @return int
+     */
+    private function getCheckHostnameConfigurationValue()
+    {
+        return $this->_scopeConfig->getValue(
+            'system/cron/settings/check_hostname',
+            \Magento\Store\Model\ScopeInterface::SCOPE_STORE
+        );
+    }
+
     /**
      * Is Group In Filter.
      *

app/code/Magento/Cron/etc/adminhtml/system.xml

@@ -42,6 +42,14 @@
                         <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
                     </field>
                 </group>
+                <group id="settings" translate="label" type="text" sortOrder="9999" showInDefault="1" showInWebsite="0" showInStore="0">
+                    <label>Settings</label>
+                    <field id="check_hostname" translate="label" type="select" sortOrder="10" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
+                        <label>Check hostname</label>
+                        <comment><![CDATA[during resetting orphan jobs on database level<br /><i>If job in cron_schedule table is marked as <b>running</b>, no other job with the same job code will be start again. Sometimes could happen that the running of the cron job has been terminated (eg. server restart). The status of these orphan jobs need to be reset, otherwise the affected cron jobs never run again.</i>]]></comment>
+                        <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
+                    </field>
+                </group>
             </group>
         </section>
     </system>

app/code/Magento/Cron/etc/config.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+-->
+<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:module:Magento_Store:etc/config.xsd">
+    <default>
+        <system>
+            <cron>
+                <settings>
+                    <check_hostname>1</check_hostname>
+                </settings>
+            </cron>
+        </system>
+    </default>
+</config>

app/code/Magento/Cron/etc/db_schema.xml

@@ -17,6 +17,12 @@
                 comment="Created At"/>
         <column xsi:type="timestamp" name="scheduled_at" on_update="false" nullable="true" comment="Scheduled At"/>
         <column xsi:type="timestamp" name="executed_at" on_update="false" nullable="true" comment="Executed At"/>
+        <column xsi:type="int" name="process_id" padding="10" unsigned="true" nullable="false"
+                comment="Process Id of host process"/>
+        <column xsi:type="varchar" name="process_hostname" nullable="false" length="255" default=""
+                comment="Hostname where the process run "/>
+        <column xsi:type="timestamp" name="process_started_at" on_update="false" nullable="true"
+                comment="Process started at"/>
         <column xsi:type="timestamp" name="finished_at" on_update="false" nullable="true" comment="Finished At"/>
         <constraint xsi:type="primary" referenceId="PRIMARY">
             <column name="schedule_id"/>

app/code/Magento/Cron/etc/db_schema_whitelist.json

@@ -8,6 +8,9 @@
             "created_at": true,
             "scheduled_at": true,
             "executed_at": true,
+            "process_id": true,
+            "process_hostname": true,
+            "process_started_at" : true,
             "finished_at": true
         },
         "index": {

app/code/Magento/Cron/i18n/en_US.csv

@@ -19,3 +19,6 @@ Monthly,Monthly
 "Success History Lifetime","Success History Lifetime"
 "Failure History Lifetime","Failure History Lifetime"
 "Use Separate Process","Use Separate Process"
+"Settings","Settings"
+"Check hostname","Check hostname"
+"<![CDATA[during resetting orphan jobs on database level<br /><i>If job in cron_schedule table is marked as <b>running</b>, no other job with the same job code will be start again. Sometimes could happen that the running of the cron job has been terminated (eg. server restart). The status of these orphan jobs need to be reset, otherwise the affected cron jobs never run again.</i>]]","<![CDATA[during resetting orphan jobs on database level<br /><i>If job in cron_schedule table is marked as <b>running</b>, no other job with the same job code will be start again. Sometimes could happen that the running of the cron job has been terminated (eg. server restart). The status of these orphan jobs need to be reset, otherwise the affected cron jobs never run again.</i>]]"