<?php

namespace Symbiote\QueuedJobs\Services;

use SilverStripe\ORM\FieldType\DBDatetime;
use SilverStripe\Subsites\State\SubsiteState;
use stdClass;
use SilverStripe\Core\Config\Config;
use SilverStripe\ORM\DataObject;

/**
 * A base implementation of a queued job that provides some convenience for implementations
 *
 * This implementation assumes that when you created your job class, you initialised the
 * jobData with relevant variables needed to process() your job later on in execution. If you do not,
 * please ensure you do before you queueJob() the job, to ensure the signature that is generated is 'correct'.
 *
 * @author Marcus Nyeholt <marcus@symbiote.com.au>
 * @license BSD http://silverstripe.org/bsd-license/
 * @skipUpgrade
 */
abstract class AbstractQueuedJob implements QueuedJob
{
    /**
     * @var stdClass
     */
    protected $jobData;

    /**
     * @var array
     */
    protected $messages = array();

    /**
     * @var int
     */
    protected $totalSteps = 0;

    /**
     * @var int
     */
    protected $currentStep = 0;

    /**
     * @var boolean
     */
    protected $isComplete = false;

    /**
     * Extensions can have a construct but don't have too.
     * Without a construct, it's impossible to create a job in the CMS
     * @var array params
     */
    public function __construct($params = array())
    {
    }

    /**
     * @return string
     */
    abstract public function getTitle();

    /**
     * Sets a data object for persisting by adding its id and type to the serialised vars
     *
     * @param DataObject $object
     * @param string $name A name to give it, if you want to store more than one
     */
    protected function setObject(DataObject $object, $name = 'Object')
    {
        $this->{$name . 'ID'} = $object->ID;
        $this->{$name . 'Type'} = $object->ClassName;
    }

    /**
     * @param string $name
     * @return DataObject|null
     */
    protected function getObject($name = 'Object')
    {
        $id = $this->{$name . 'ID'};
        $type = $this->{$name . 'Type'};
        if ($id) {
            return DataObject::get_by_id($type, $id);
        }
    }

    /**
     * Return a signature for this queued job
     *
     * @return string
     */
    public function getSignature()
    {
        return md5(get_class($this) . serialize($this->jobData));
    }

    /**
     * Generate a somewhat random signature
     *
     * useful if you're want to make sure something is always added
     *
     * @return string
     */
    protected function randomSignature()
    {
        return md5(get_class($this) . DBDatetime::now()->getTimestamp() . mt_rand(0, 100000));
    }

    /**
     * By default jobs should just go into the default processing queue
     *
     * @return string
     */
    public function getJobType()
    {
        return QueuedJob::QUEUED;
    }

    /**
     * Performs setup tasks the first time this job is run.
     *
     * This is only executed once for every job. If you want to run something on every job restart, use the
     * {@link prepareForRestart} method.
     */
    public function setup()
    {
        $this->loadCustomConfig();
    }

    /**
     * Run when an already setup job is being restarted.
     */
    public function prepareForRestart()
    {
        $this->loadCustomConfig();
    }

    /**
     * Do some processing yourself!
     */
    abstract public function process();

    /**
     * Method for determining whether the job is finished - you may override it if there's
     * more to it than just this
     */
    public function jobFinished()
    {
        return $this->isComplete;
    }

    /**
     * Called when the job is determined to be 'complete'
     */
    public function afterComplete()
    {
    }

    /**
     * @return stdClass
     */
    public function getJobData()
    {
        // okay, we NEED to store the subsite ID if there's one available
        if (!$this->SubsiteID && class_exists(SubsiteState::class)) {
            $this->SubsiteID = SubsiteState::singleton()->getSubsiteId();
        }

        $data = new stdClass();
        $data->totalSteps = $this->totalSteps;
        $data->currentStep = $this->currentStep;
        $data->isComplete = $this->isComplete;
        $data->jobData = $this->jobData;
        $data->messages = $this->messages;

        return $data;
    }

    /**
     * @param int $totalSteps
     * @param int $currentStep
     * @param boolean $isComplete
     * @param stdClass $jobData
     * @param array $messages
     */
    public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
    {
        $this->totalSteps = $totalSteps;
        $this->currentStep = $currentStep;
        $this->isComplete = $isComplete;
        $this->jobData = $jobData;
        $this->messages = $messages;
    }

    /**
     * Gets custom config settings to use when running the job.
     *
     * @return array|null
     */
    public function getCustomConfig()
    {
        return $this->CustomConfig;
    }

    /**
     * Sets custom config settings to use when the job is run.
     *
     * @param array $config
     */
    public function setCustomConfig(array $config)
    {
        $this->CustomConfig = $config;
    }

    /**
     * Sets custom configuration settings from the job data.
     */
    private function loadCustomConfig()
    {
        $custom = $this->getCustomConfig();

        if (!is_array($custom)) {
            return;
        }

        foreach ($custom as $class => $settings) {
            foreach ($settings as $setting => $value) {
                Config::modify()->set($class, $setting, $value);
            }
        }
    }

    /**
     * @param string $message
     * @param string $severity
     */
    public function addMessage($message, $severity = 'INFO')
    {
        $severity = strtoupper($severity);
        $this->messages[] = '[' . DBDatetime::now()->Rfc2822() . "][$severity] $message";
    }

    /**
     * Convenience methods for setting and getting job data
     *
     * @param mixed $name
     * @param mixed $value
     */
    public function __set($name, $value)
    {
        if (!$this->jobData) {
            $this->jobData = new stdClass();
        }
        $this->jobData->$name = $value;
    }

    /**
     * Retrieve some job data
     *
     * @param mixed $name
     * @return mixed
     */
    public function __get($name)
    {
        return isset($this->jobData->$name) ? $this->jobData->$name : null;
    }
}


<?php

namespace Symbiote\QueuedJobs\Tasks;

use SilverStripe\Control\HTTPRequest;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\DataObject;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;

/**
 * An administrative task to delete all queued jobs records from the database.
 * Use with caution!
 */
class DeleteAllJobsTask extends BuildTask
{

    /**
     * @inheritdoc
     * @return string
     */
    public function getTitle()
    {
        return "Delete all queued jobs.";
    }

    /**
     * @inheritdoc
     * @return string
     */
    public function getDescription()
    {
        return "Remove all queued jobs from the database. Use with caution!";
    }

    /**
     * Run the task
     * @param HTTPRequest $request
     */
    public function run($request)
    {
        $confirm = $request->getVar('confirm');

        $jobs = DataObject::get(QueuedJobDescriptor::class);

        if (!$confirm) {
            echo "Really delete " . $jobs->count() . " jobs? Please add ?confirm=1 to the URL to confirm.";
            return;
        }

        echo "Deleting " . $jobs->count() . " jobs...<br>\n";
        $jobs->removeAll();
        echo "Done.";
    }
}


<?php

namespace Symbiote\QueuedJobs\Jobs;

use SilverStripe\Core\Config\Configurable;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\FieldType\DBDatetime;
use Symbiote\QueuedJobs\Services\AbstractQueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
 * An queued job to clean out the QueuedJobDescriptor Table
 * which often gets too full
 *
 * @author Andrew Aitken-Fincham <andrew@symbiote.com.au>
 */
class CleanupJob extends AbstractQueuedJob
{
    use Configurable;

    /**
     * How we will determine "stale"
     * Possible values: age, number
     * @config
     * @var string
     */
    private static $cleanup_method = "age";

    /**
     * Value associated with cleanupMethod
     * age => days, number => integer
     * @config
     * @var integer
     */
    private static $cleanup_value = 30;

    /**
     * Which JobStatus values are OK to be deleted
     * @config
     * @var array
     */
    private static $cleanup_statuses = array(
        "Complete",
        "Broken",
        // "Initialising",
        // "Running",
        // "New",
        // "Paused",
        // "Cancelled",
        // "Waiting",
    );

    /**
     * Database query limit
     *
     * @config
     * @var integer
     */
    private static $query_limit = 100000;

    /**
     * Check whether is enabled or not for BC
     * @config
     * @var boolean
     */
    private static $is_enabled = false;

    /**
     * Defines the title of the job
     * @return string
     */
    public function getTitle()
    {
        return _t(
            __CLASS__ . '.Title',
            "Clean up old jobs from the database"
        );
    }

    /**
     * Set immediacy of job
     * @return int
     */
    public function getJobType()
    {
        $this->totalSteps = '1';
        return QueuedJob::IMMEDIATE;
    }

    /**
     * Clear out stale jobs based on the cleanup values
     */
    public function process()
    {
        // construct limit statement if query_limit is valid int value
        $limit = '';
        $query_limit = $this->config()->get('query_limit');
        if (is_numeric($query_limit) && $query_limit >= 0) {
            $limit = ' LIMIT ' . ((int)$query_limit);
        }

        $statusList = implode('\', \'', $this->config()->get('cleanup_statuses'));
        switch ($this->config()->get('cleanup_method')) {
            // If Age, we need to get jobs that are at least n days old
            case "age":
                $cutOff = date(
                    "Y-m-d H:i:s",
                    strtotime(DBDatetime::now() .
                        " - " .
                        $this->config()->cleanup_value .
                        " days")
                );
                $stale = DB::query(
                    'SELECT "ID"
					FROM "QueuedJobDescriptor"
					WHERE "JobStatus"
					IN (\'' . $statusList . '\')
					AND "LastEdited" < \'' . $cutOff . '\'' . $limit
                );
                $staleJobs = $stale->column("ID");
                break;
            // If Number, we need to save n records, then delete from the rest
            case "number":
                $fresh = DB::query(
                    'SELECT "ID"
					FROM "QueuedJobDescriptor"
					ORDER BY "LastEdited"
					ASC LIMIT ' . $this->config()->cleanup_value
                );
                $freshJobIDs = implode('\', \'', $fresh->column("ID"));

                $stale = DB::query(
                    'SELECT "ID"
					FROM "QueuedJobDescriptor"
					WHERE "ID"
					NOT IN (\'' . $freshJobIDs . '\')
					AND "JobStatus"
					IN (\'' . $statusList . '\')' . $limit
                );
                $staleJobs = $stale->column("ID");
                break;
            default:
                $this->addMessage("Incorrect configuration values set. Cleanup ignored");
                $this->isComplete = true;
                return;
        }
        if (empty($staleJobs)) {
            $this->addMessage("No jobs to clean up.");
            $this->isComplete = true;
            $this->reenqueue();
            return;
        }
        $numJobs = count($staleJobs);
        $staleJobs = implode('\', \'', $staleJobs);
        DB::query('DELETE FROM "QueuedJobDescriptor"
			WHERE "ID"
			IN (\'' . $staleJobs . '\')');
        $this->addMessage($numJobs . " jobs cleaned up.");
        // let's make sure there is a cleanupJob in the queue
        $this->reenqueue();
        $this->isComplete = true;
    }

    private function reenqueue()
    {
        if ($this->config()->get('is_enabled')) {
            $this->addMessage("Queueing the next Cleanup Job.");
            $cleanup = Injector::inst()->create(CleanupJob::class);
            QueuedJobService::singleton()->queueJob(
                $cleanup,
                DBDatetime::create()->setValue(DBDatetime::now()->getTimestamp() + 86400)->Rfc2822()
            );
        }
    }
}


<?php

namespace Symbiote\QueuedJobs\Services;

use Exception;
use Psr\Log\LoggerInterface;
use SilverStripe\Control\Controller;
use SilverStripe\Control\Director;
use SilverStripe\Control\Email\Email;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Config\Configurable;
use SilverStripe\Core\Extensible;
use SilverStripe\Core\Injector\Injectable;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\FieldType\DBDatetime;
use SilverStripe\Security\Member;
use SilverStripe\Security\Security;
use SilverStripe\Subsites\Model\Subsite;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;
use Symbiote\QueuedJobs\QJUtils;
use Symbiote\QueuedJobs\Tasks\Engines\TaskRunnerEngine;

/**
 * A service that can be used for starting, stopping and listing queued jobs.
 *
 * When a job is first added, it is initialised, its job type determined, then persisted to the database
 *
 * When the queues are scanned, a job is reloaded and processed. Ignoring the persistence and reloading, it looks
 * something like
 *
 * job->getJobType();
 * job->getJobData();
 * data->write();
 * job->setup();
 * while !job->isComplete
 *  job->process();
 *  job->getJobData();
 *  data->write();
 *
 *
 * @author Marcus Nyeholt <marcus@symbiote.com.au>
 * @license BSD http://silverstripe.org/bsd-license/
 */
class QueuedJobService
{
    use Configurable;
    use Injectable;
    use Extensible;

    /**
     * @config
     * @var int
     */
    private static $stall_threshold = 3;

    /**
     * How much ram will we allow before pausing and releasing the memory?
     *
     * For instance, set to 268435456 (256MB) to pause this process if used memory exceeds
     * this value. This needs to be set to a value lower than the php_ini max_memory as
     * the system will otherwise crash before shutdown can be handled gracefully.
     *
     * This was increased to 256MB for SilverStripe 4.x as framework uses more memory than 3.x
     *
     * @var int
     * @config
     */
    private static $memory_limit = 268435456;

    /**
     * Optional time limit (in seconds) to run the service before restarting to release resources.
     *
     * Defaults to no limit.
     *
     * @var int
     * @config
     */
    private static $time_limit = 0;

    /**
     * Disable health checks that usually occur when a runner first picks up a queue. Note that completely disabling
     * health checks could result in many jobs that are always marked as running - that will never be restarted. If
     * this option is disabled you may alternatively use the build task
     *
     * @see \Symbiote\QueuedJobs\Tasks\CheckJobHealthTask
     *
     * @var bool
     * @config
     */
    private static $disable_health_check = false;

    /**
     * Timestamp (in seconds) when the queue was started
     *
     * @var int
     */
    protected $startedAt = 0;

    /**
     * Should "immediate" jobs be managed using the shutdown function?
     *
     * It is recommended you set up an inotify watch and use that for
     * triggering immediate jobs. See the wiki for more information
     *
     * @var boolean
     * @config
     */
    private static $use_shutdown_function = true;

    /**
     * The location for immediate jobs to be stored in
     *
     * @var string
     * @config
     */
    private static $cache_dir = 'queuedjobs';

    /**
     * @var DefaultQueueHandler
     */
    public $queueHandler;

    /**
     *
     * @var TaskRunnerEngine
     */
    public $queueRunner;

    /**
     * Config controlled list of default/required jobs
     *
     * @var array
     */
    public $defaultJobs = [];

    /**
     * Register our shutdown handler
     */
    public function __construct()
    {
        // bind a shutdown function to process all 'immediate' queued jobs if needed, but only in CLI mode
        if (static::config()->get('use_shutdown_function') && Director::is_cli()) {
            register_shutdown_function([$this, 'onShutdown']);
        }
        if (Config::inst()->get(Email::class, 'queued_job_admin_email') == '') {
            Config::modify()->set(
                Email::class,
                'queued_job_admin_email',
                Config::inst()->get(Email::class, 'admin_email')
            );
        }
    }

    /**
     * Adds a job to the queue to be started
     *
     * Relevant data about the job will be persisted using a QueuedJobDescriptor
     *
     * @param QueuedJob $job
     *          The job to start.
     * @param $startAfter
     *          The date (in Y-m-d H:i:s format) to start execution after
     * @param int $userId
     *          The ID of a user to execute the job as. Defaults to the current user
     *
     * @return int
     */
    public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null)
    {
        $signature = $job->getSignature();

        // see if we already have this job in a queue
        $filter = [
            'Signature' => $signature,
            'JobStatus' => [
                QueuedJob::STATUS_NEW,
                QueuedJob::STATUS_INIT,
            ],
        ];

        $existing = QueuedJobDescriptor::get()
            ->filter($filter)
            ->first();

        if ($existing && $existing->ID) {
            return $existing->ID;
        }

        $jobDescriptor = new QueuedJobDescriptor();
        $jobDescriptor->JobTitle = $job->getTitle();
        $jobDescriptor->JobType = $queueName ? $queueName : $job->getJobType();
        $jobDescriptor->Signature = $signature;
        $jobDescriptor->Implementation = get_class($job);
        $jobDescriptor->StartAfter = $startAfter;

        $runAsID = 0;
        if ($userId) {
            $runAsID = $userId;
        } elseif (Security::getCurrentUser() && Security::getCurrentUser()->exists()) {
            $runAsID = Security::getCurrentUser()->ID;
        }
        $jobDescriptor->RunAsID = $runAsID;

        // copy data
        $this->copyJobToDescriptor($job, $jobDescriptor);

        $jobDescriptor->write();

        $this->startJob($jobDescriptor, $startAfter);

        return $jobDescriptor->ID;
    }

    /**
     * Start a job (or however the queue handler determines it should be started)
     *
     * @param QueuedJobDescriptor $jobDescriptor
     * @param string $startAfter
     */
    public function startJob($jobDescriptor, $startAfter = null)
    {
        if ($startAfter && strtotime($startAfter) > DBDatetime::now()->getTimestamp()) {
            $this->queueHandler->scheduleJob($jobDescriptor, $startAfter);
        } else {
            // immediately start it on the queue, however that works
            $this->queueHandler->startJobOnQueue($jobDescriptor);
        }
    }

    /**
     * Copies data from a job into a descriptor for persisting
     *
     * @param QueuedJob $job
     * @param QueuedJobDescriptor $jobDescriptor
     */
    protected function copyJobToDescriptor($job, $jobDescriptor)
    {
        $data = $job->getJobData();

        $jobDescriptor->TotalSteps = $data->totalSteps;
        $jobDescriptor->StepsProcessed = $data->currentStep;
        if ($data->isComplete) {
            $jobDescriptor->JobStatus = QueuedJob::STATUS_COMPLETE;
            $jobDescriptor->JobFinished = DBDatetime::now()->Rfc2822();
        }

        $jobDescriptor->SavedJobData = serialize($data->jobData);
        $jobDescriptor->SavedJobMessages = serialize($data->messages);
    }

    /**
     * @param QueuedJobDescriptor $jobDescriptor
     * @param QueuedJob $job
     */
    protected function copyDescriptorToJob($jobDescriptor, $job)
    {
        $jobData = null;
        $messages = null;

        // switching to php's serialize methods... not sure why this wasn't done from the start!
        $jobData = @unserialize($jobDescriptor->SavedJobData);
        $messages = @unserialize($jobDescriptor->SavedJobMessages);

        // try decoding as json if null
        $jobData = $jobData ?: json_decode($jobDescriptor->SavedJobData);
        $messages = $messages ?: json_decode($jobDescriptor->SavedJobMessages);

        $job->setJobData(
            $jobDescriptor->TotalSteps,
            $jobDescriptor->StepsProcessed,
            $jobDescriptor->JobStatus == QueuedJob::STATUS_COMPLETE,
            $jobData,
            $messages
        );
    }

    /**
     * Check the current job queues and see if any of the jobs currently in there should be started. If so,
     * return the next job that should be executed
     *
     * @param string $type Job type
     *
     * @return QueuedJobDescriptor|false
     */
    public function getNextPendingJob($type = null)
    {
        // Filter jobs by type
        $type = $type ?: QueuedJob::QUEUED;
        $list = QueuedJobDescriptor::get()
            ->filter('JobType', $type)
            ->sort('ID', 'ASC');

        // see if there's any blocked jobs that need to be resumed
        /** @var QueuedJobDescriptor $waitingJob */
        $waitingJob = $list
            ->filter('JobStatus', QueuedJob::STATUS_WAIT)
            ->first();
        if ($waitingJob) {
            return $waitingJob;
        }

        // If there's an existing job either running or pending, the lets just return false to indicate
        // that we're still executing
        $runningJob = $list
            ->filter('JobStatus', [QueuedJob::STATUS_INIT, QueuedJob::STATUS_RUN])
            ->first();
        if ($runningJob) {
            return false;
        }

        // Otherwise, lets find any 'new' jobs that are waiting to execute
        /** @var QueuedJobDescriptor $newJob */
        $newJob = $list
            ->filter('JobStatus', QueuedJob::STATUS_NEW)
            ->where(sprintf(
                '"StartAfter" < \'%s\' OR "StartAfter" IS NULL',
                DBDatetime::now()->getValue()
            ))
            ->first();

        return $newJob;
    }

    /**
     * Runs an explicit check on all currently running jobs to make sure their "processed" count is incrementing
     * between each run. If it's not, then we need to flag it as paused due to an error.
     *
     * This typically happens when a PHP fatal error is thrown, which can't be picked up by the error
     * handler or exception checker; in this case, we detect these stalled jobs later and fix (try) to
     * fix them
     *
     * @param int $queue The queue to check against
     */
    public function checkJobHealth($queue = null)
    {
        $queue = $queue ?: QueuedJob::QUEUED;
        // Select all jobs currently marked as running
        $runningJobs = QueuedJobDescriptor::get()
            ->filter([
                'JobStatus' => [
                    QueuedJob::STATUS_RUN,
                    QueuedJob::STATUS_INIT,
                ],
                'JobType' => $queue,
            ]);

        // If no steps have been processed since the last run, consider it a broken job
        // Only check jobs that have been viewed before. LastProcessedCount defaults to -1 on new jobs.
        $stalledJobs = $runningJobs
            ->filter('LastProcessedCount:GreaterThanOrEqual', 0)
            ->where('"StepsProcessed" = "LastProcessedCount"');
        foreach ($stalledJobs as $stalledJob) {
            $this->restartStalledJob($stalledJob);
        }

        // now, find those that need to be marked before the next check
        // foreach job, mark it as having been incremented
        foreach ($runningJobs as $job) {
            $job->LastProcessedCount = $job->StepsProcessed;
            $job->write();
        }

        // finally, find the list of broken jobs and send an email if there's some found
        $brokenJobs = QueuedJobDescriptor::get()->filter('JobStatus', QueuedJob::STATUS_BROKEN);
        if ($brokenJobs && $brokenJobs->count()) {
            $this->getLogger()->error(
                print_r(
                    [
                        'errno' => 0,
                        'errstr' => 'Broken jobs were found in the job queue',
                        'errfile' => __FILE__,
                        'errline' => __LINE__,
                        'errcontext' => [],
                    ],
                    true
                ),
                [
                    'file' => __FILE__,
                    'line' => __LINE__,
                ]
            );
        }

        return $stalledJobs->count();
    }

    /**
     * Checks through ll the scheduled jobs that are expected to exist
     */
    public function checkDefaultJobs($queue = null)
    {
        $queue = $queue ?: QueuedJob::QUEUED;
        if (count($this->defaultJobs)) {
            $activeJobs = QueuedJobDescriptor::get()->filter(
                'JobStatus',
                [
                    QueuedJob::STATUS_NEW,
                    QueuedJob::STATUS_INIT,
                    QueuedJob::STATUS_RUN,
                    QueuedJob::STATUS_WAIT,
                    QueuedJob::STATUS_PAUSED,
                ]
            );
            foreach ($this->defaultJobs as $title => $jobConfig) {
                if (!isset($jobConfig['filter']) || !isset($jobConfig['type'])) {
                    $this->getLogger()->error(
                        "Default Job config: $title incorrectly set up. Please check the readme for examples",
                        [
                            'file' => __FILE__,
                            'line' => __LINE__,
                        ]
                    );
                    continue;
                }
                $job = $activeJobs->filter(array_merge(
                    ['Implementation' => $jobConfig['type']],
                    $jobConfig['filter']
                ));
                if (!$job->count()) {
                    $this->getLogger()->error(
                        "Default Job config: $title was missing from Queue",
                        [
                            'file' => __FILE__,
                            'line' => __LINE__,
                        ]
                    );
                    Email::create()
                        ->setTo(
                            isset($jobConfig['email'])
                                ? $jobConfig['email']
                                : Config::inst()->get(Email::class, 'queued_job_admin_email')
                        )
                        ->setFrom(Config::inst()->get(Email::class, 'admin_email'))
                        ->setSubject('Default Job "' . $title . '" missing')
                        ->setData($jobConfig)
                        ->addData('Title', $title)
                        ->addData('Site', Director::absoluteBaseURL())
                        ->setHTMLTemplate('QueuedJobsDefaultJob')
                        ->send();
                    if (isset($jobConfig['recreate']) && $jobConfig['recreate']) {
                        if (!array_key_exists('construct', $jobConfig)
                            || !isset($jobConfig['startDateFormat'])
                            || !isset($jobConfig['startTimeString'])
                        ) {
                            $this->getLogger()->error(
                                "Default Job config: $title incorrectly set up. Please check the readme for examples",
                                [
                                    'file' => __FILE__,
                                    'line' => __LINE__,
                                ]
                            );
                            continue;
                        }
                        QueuedJobService::singleton()->queueJob(
                            Injector::inst()->createWithArgs($jobConfig['type'], $jobConfig['construct']),
                            date($jobConfig['startDateFormat'], strtotime($jobConfig['startTimeString']))
                        );
                        $this->getLogger()->info(
                            "Default Job config: $title has been re-added to the Queue",
                            [
                                'file' => __FILE__,
                                'line' => __LINE__,
                            ]
                        );
                    }
                }
            }
        }
    }

    /**
     * Attempt to restart a stalled job
     *
     * @param QueuedJobDescriptor $stalledJob
     *
     * @return bool True if the job was successfully restarted
     */
    protected function restartStalledJob($stalledJob)
    {
        if ($stalledJob->ResumeCounts < static::config()->get('stall_threshold')) {
            $stalledJob->restart();
            $logLevel = 'warning';
            $message = _t(
                __CLASS__ . '.STALLED_JOB_RESTART_MSG',
                'A job named {name} (#{id}) appears to have stalled. It will be stopped and restarted, please '
                . 'login to make sure it has continued',
                ['name' => $stalledJob->JobTitle, 'id' => $stalledJob->ID]
            );
        } else {
            $stalledJob->pause();
            $logLevel = 'error';
            $message = _t(
                __CLASS__ . '.STALLED_JOB_MSG',
                'A job named {name} (#{id}) appears to have stalled. It has been paused, please login to check it',
                ['name' => $stalledJob->JobTitle, 'id' => $stalledJob->ID]
            );
        }

        $this->getLogger()->log(
            $logLevel,
            $message,
            [
                'file' => __FILE__,
                'line' => __LINE__,
            ]
        );
        $from = Config::inst()->get(Email::class, 'admin_email');
        $to = Config::inst()->get(Email::class, 'queued_job_admin_email');
        $subject = _t(__CLASS__ . '.STALLED_JOB', 'Stalled job');

        if ($to) {
            $mail = Email::create($from, $to, $subject)
                ->setData([
                    'JobID' => $stalledJob->ID,
                    'Message' => $message,
                    'Site' => Director::absoluteBaseURL(),
                ])
                ->setHTMLTemplate('QueuedJobsStalledJob');
            $mail->send();
        }
    }

    /**
     * Prepares the given jobDescriptor for execution. Returns the job that
     * will actually be run in a state ready for executing.
     *
     * Note that this is called each time a job is picked up to be executed from the cron
     * job - meaning that jobs that are paused and restarted will have 'setup()' called on them again,
     * so your job MUST detect that and act accordingly.
     *
     * @param QueuedJobDescriptor $jobDescriptor
     *          The Job descriptor of a job to prepare for execution
     *
     * @return QueuedJob|boolean
     * @throws Exception
     */
    protected function initialiseJob(QueuedJobDescriptor $jobDescriptor)
    {
        // create the job class
        $impl = $jobDescriptor->Implementation;
        /** @var QueuedJob $job */
        $job = Injector::inst()->create($impl);
        /* @var $job QueuedJob */
        if (!$job) {
            throw new Exception("Implementation $impl no longer exists");
        }

        $jobDescriptor->JobStatus = QueuedJob::STATUS_INIT;
        $jobDescriptor->write();

        // make sure the data is there
        $this->copyDescriptorToJob($jobDescriptor, $job);

        // see if it needs 'setup' or 'restart' called
        if ($jobDescriptor->StepsProcessed <= 0) {
            $job->setup();
        } else {
            $job->prepareForRestart();
        }

        // make sure the descriptor is up to date with anything changed
        $this->copyJobToDescriptor($job, $jobDescriptor);
        $jobDescriptor->write();

        return $job;
    }

    /**
     * Given a {@link QueuedJobDescriptor} mark the job as initialised. Works sort of like a mutex.
     * Currently a database lock isn't entirely achievable, due to database adapters not supporting locks.
     * This may still have a race condition, but this should minimise the possibility.
     * Side effect is the job status will be changed to "Initialised".
     *
     * Assumption is the job has a status of "Queued" or "Wait".
     *
     * @param QueuedJobDescriptor $jobDescriptor
     *
     * @return boolean
     */
    protected function grabMutex(QueuedJobDescriptor $jobDescriptor)
    {
        // write the status and determine if any rows were affected, for protection against a
        // potential race condition where two or more processes init the same job at once.
        // This deliberately does not use write() as that would always update LastEdited
        // and thus the row would always be affected.
        try {
            DB::query(sprintf(
                'UPDATE "QueuedJobDescriptor" SET "JobStatus" = \'%s\' WHERE "ID" = %s',
                QueuedJob::STATUS_INIT,
                $jobDescriptor->ID
            ));
        } catch (Exception $e) {
            return false;
        }

        if (DB::get_conn()->affectedRows() === 0 && $jobDescriptor->JobStatus !== QueuedJob::STATUS_INIT) {
            return false;
        }

        return true;
    }

    /**
     * Start the actual execution of a job.
     * The assumption is the jobID refers to a {@link QueuedJobDescriptor} that is status set as "Queued".
     *
     * This method will continue executing until the job says it's completed
     *
     * @param int $jobId
     *          The ID of the job to start executing
     *
     * @return boolean
     * @throws Exception
     */
    public function runJob($jobId)
    {
        // first retrieve the descriptor
        /** @var QueuedJobDescriptor $jobDescriptor */
        $jobDescriptor = DataObject::get_by_id(
            QueuedJobDescriptor::class,
            (int)$jobId
        );
        if (!$jobDescriptor) {
            throw new Exception("$jobId is invalid");
        }

        // now lets see whether we have a current user to run as. Typically, if the job is executing via the CLI,
        // we want it to actually execute as the RunAs user - however, if running via the web (which is rare...), we
        // want to ensure that the current user has admin privileges before switching. Otherwise, we just run it
        // as the currently logged in user and hope for the best

        // We need to use $_SESSION directly because SS ties the session to a controller that no longer exists at
        // this point of execution in some circumstances
        $originalUserID = isset($_SESSION['loggedInAs']) ? $_SESSION['loggedInAs'] : 0;
        /** @var Member|null $originalUser */
        $originalUser = $originalUserID
            ? DataObject::get_by_id(Member::class, $originalUserID)
            : null;
        $runAsUser = null;

        // If the Job has requested that we run it as a particular user, then we should try and do that.
        if ($jobDescriptor->RunAs() !== null) {
            $runAsUser = $this->setRunAsUser($jobDescriptor->RunAs(), $originalUser);
        }

        // set up a custom error handler for this processing
        $errorHandler = new JobErrorHandler();

        $job = null;

        $broken = false;

        // Push a config context onto the stack for the duration of this job run.
        Config::nest();

        if ($this->grabMutex($jobDescriptor)) {
            try {
                $job = $this->initialiseJob($jobDescriptor);

                // get the job ready to begin.
                if (!$jobDescriptor->JobStarted) {
                    $jobDescriptor->JobStarted = DBDatetime::now()->Rfc2822();
                } else {
                    $jobDescriptor->JobRestarted = DBDatetime::now()->Rfc2822();
                }

                // Only write to job as "Running" if 'isComplete' was NOT set to true
                // during setup() or prepareForRestart()
                if (!$job->jobFinished()) {
                    $jobDescriptor->JobStatus = QueuedJob::STATUS_RUN;
                    $jobDescriptor->write();
                }

                $lastStepProcessed = 0;
                // have we stalled at all?
                $stallCount = 0;

                if (class_exists(Subsite::class) && !empty($job->SubsiteID)) {
                    Subsite::changeSubsite($job->SubsiteID);

                    // lets set the base URL as far as Director is concerned so that our URLs are correct
                    /** @var Subsite $subsite */
                    $subsite = DataObject::get_by_id(Subsite::class, $job->SubsiteID);
                    if ($subsite && $subsite->exists()) {
                        $domain = $subsite->domain();
                        $base = rtrim(Director::protocol() . $domain, '/') . '/';

                        Config::modify()->set(Director::class, 'alternate_base_url', $base);
                    }
                }

                // while not finished
                while (!$job->jobFinished() && !$broken) {
                    // see that we haven't been set to 'paused' or otherwise by another process
                    /** @var QueuedJobDescriptor $jobDescriptor */
                    $jobDescriptor = DataObject::get_by_id(
                        QueuedJobDescriptor::class,
                        (int)$jobId
                    );
                    if (!$jobDescriptor || !$jobDescriptor->exists()) {
                        $broken = true;
                        $this->getLogger()->error(
                            print_r(
                                [
                                    'errno' => 0,
                                    'errstr' => 'Job descriptor ' . $jobId . ' could not be found',
                                    'errfile' => __FILE__,
                                    'errline' => __LINE__,
                                    'errcontext' => [],
                                ],
                                true
                            ),
                            [
                                'file' => __FILE__,
                                'line' => __LINE__,
                            ]
                        );
                        break;
                    }
                    if ($jobDescriptor->JobStatus != QueuedJob::STATUS_RUN) {
                        // we've been paused by something, so we'll just exit
                        $job->addMessage(_t(
                            __CLASS__ . '.JOB_PAUSED',
                            'Job paused at {time}',
                            ['time' => DBDatetime::now()->Rfc2822()]
                        ));
                        $broken = true;
                    }

                    if (!$broken) {
                        // Collect output as job messages as well as sending it to the screen
                        $obLogger = function ($buffer, $phase) use ($job, $jobDescriptor) {
                            $job->addMessage($buffer);
                            if ($jobDescriptor) {
                                $this->copyJobToDescriptor($job, $jobDescriptor);
                                $jobDescriptor->write();
                            }
                            return $buffer;
                        };
                        ob_start($obLogger, 256);

                        try {
                            $job->process();
                        } catch (Exception $e) {
                            // okay, we'll just catch this exception for now
                            $job->addMessage(
                                _t(
                                    __CLASS__ . '.JOB_EXCEPT',
                                    'Job caused exception {message} in {file} at line {line}',
                                    [
                                        'message' => $e->getMessage(),
                                        'file' => $e->getFile(),
                                        'line' => $e->getLine(),
                                    ]
                                )
                            );
                            $this->getLogger()->error(
                                $e->getMessage(),
                                [
                                    'exception' => $e,
                                ]
                            );
                            $jobDescriptor->JobStatus =  QueuedJob::STATUS_BROKEN;
                            $this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
                        }

                        ob_end_flush();

                        // now check the job state
                        $data = $job->getJobData();
                        if ($data->currentStep == $lastStepProcessed) {
                            $stallCount++;
                        }

                        if ($stallCount > static::config()->get('stall_threshold')) {
                            $broken = true;
                            $job->addMessage(
                                _t(
                                    __CLASS__ . '.JOB_STALLED',
                                    'Job stalled after {attempts} attempts - please check',
                                    ['attempts' => $stallCount]
                                )
                            );
                            $jobDescriptor->JobStatus = QueuedJob::STATUS_BROKEN;
                        }

                        // now we'll be good and check our memory usage. If it is too high, we'll set the job to
                        // a 'Waiting' state, and let the next processing run pick up the job.
                        if ($this->isMemoryTooHigh()) {
                            $job->addMessage(
                                _t(
                                    __CLASS__ . '.MEMORY_RELEASE',
                                    'Job releasing memory and waiting ({used} used)',
                                    ['used' => $this->humanReadable($this->getMemoryUsage())]
                                )
                            );
                            if ($jobDescriptor->JobStatus != QueuedJob::STATUS_BROKEN) {
                                $jobDescriptor->JobStatus = QueuedJob::STATUS_WAIT;
                            }
                            $broken = true;
                        }

                        // Also check if we are running too long
                        if ($this->hasPassedTimeLimit()) {
                            $job->addMessage(_t(
                                __CLASS__ . '.TIME_LIMIT',
                                'Queue has passed time limit and will restart before continuing'
                            ));
                            if ($jobDescriptor->JobStatus != QueuedJob::STATUS_BROKEN) {
                                $jobDescriptor->JobStatus = QueuedJob::STATUS_WAIT;
                            }
                            $broken = true;
                        }
                    }

                    if ($jobDescriptor) {
                        $this->copyJobToDescriptor($job, $jobDescriptor);
                        $jobDescriptor->write();
                    } else {
                        $this->getLogger()->error(
                            print_r(
                                [
                                    'errno' => 0,
                                    'errstr' => 'Job descriptor has been set to null',
                                    'errfile' => __FILE__,
                                    'errline' => __LINE__,
                                    'errcontext' => [],
                                ],
                                true
                            ),
                            [
                                'file' => __FILE__,
                                'line' => __LINE__,
                            ]
                        );
                        $broken = true;
                    }
                }

                // a last final save. The job is complete by now
                if ($jobDescriptor) {
                    $jobDescriptor->write();
                }

                if ($job->jobFinished()) {
                    /** @var AbstractQueuedJob|QueuedJob $job */
                    $job->afterComplete();
                    $jobDescriptor->cleanupJob();
                }
            } catch (Exception $e) {
                // okay, we'll just catch this exception for now
                $this->getLogger()->error(
                    $e->getMessage(),
                    [
                        'exception' => $e,
                    ]
                );
                $jobDescriptor->JobStatus =  QueuedJob::STATUS_BROKEN;
                $this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
                $jobDescriptor->write();
                $broken = true;
            }
        }

        $errorHandler->clear();

        Config::unnest();

        $this->unsetRunAsUser($runAsUser, $originalUser);

        return !$broken;
    }

    /**
     * @param Member $runAsUser
     * @param Member|null $originalUser
     * @return null|Member
     */
    protected function setRunAsUser(Member $runAsUser, Member $originalUser = null)
    {
        // Sanity check. Can't set the user if they don't exist.
        if ($runAsUser === null || !$runAsUser->exists()) {
            return null;
        }

        // Don't need to set Security user if we're already logged in as that same user.
        if ($originalUser && $originalUser->ID === $runAsUser->ID) {
            return null;
        }

        // We are currently either not logged in at all, or we're logged in as a different user. We should switch users
        // so that the context within the Job is correct.
        if (Controller::has_curr()) {
            Security::setCurrentUser($runAsUser);
        } else {
            $_SESSION['loggedInAs'] = $runAsUser->ID;
        }

        // This is an explicit coupling brought about by SS not having a nice way of mocking a user, as it requires
        // session nastiness
        if (class_exists('SecurityContext')) {
            singleton('SecurityContext')->setMember($runAsUser);
        }

        return $runAsUser;
    }

    /**
     * @param Member|null $runAsUser
     * @param Member|null $originalUser
     */
    protected function unsetRunAsUser(Member $runAsUser = null, Member $originalUser = null)
    {
        // No runAsUser was set, so we don't need to do anything.
        if ($runAsUser === null) {
            return;
        }

        // There was no originalUser, so we should make sure that we set the user back to null.
        if (!$originalUser) {
            if (Controller::has_curr()) {
                Security::setCurrentUser(null);
            } else {
                $_SESSION['loggedInAs'] = null;
            }

            return;
        }

        // Okay let's reset our user.
        if (Controller::has_curr()) {
            Security::setCurrentUser($originalUser);
        } else {
            $_SESSION['loggedInAs'] = $originalUser->ID;
        }
    }

    /**
     * Start timer
     */
    protected function markStarted()
    {
        if (!$this->startedAt) {
            $this->startedAt = DBDatetime::now()->getTimestamp();
        }
    }

    /**
     * Is execution time too long?
     *
     * @return bool True if the script has passed the configured time_limit
     */
    protected function hasPassedTimeLimit()
    {
        // Ensure a limit exists
        $limit = static::config()->get('time_limit');
        if (!$limit) {
            return false;
        }

        // Ensure started date is set
        $this->markStarted();

        // Check duration
        $now = DBDatetime::now()->getTimestamp();
        return $now > $this->startedAt + $limit;
    }

    /**
     * Is memory usage too high?
     *
     * @return bool
     */
    protected function isMemoryTooHigh()
    {
        $used = $this->getMemoryUsage();
        $limit = $this->getMemoryLimit();
        return $limit && ($used > $limit);
    }

    /**
     * Get peak memory usage of this application
     *
     * @return float
     */
    protected function getMemoryUsage()
    {
        // Note we use real_usage = false
        // http://stackoverflow.com/questions/15745385/memory-get-peak-usage-with-real-usage
        // Also we use the safer peak memory usage
        return (float)memory_get_peak_usage(false);
    }

    /**
     * Determines the memory limit (in bytes) for this application
     * Limits to the smaller of memory_limit configured via php.ini or silverstripe config
     *
     * @return float Memory limit in bytes
     */
    protected function getMemoryLimit()
    {
        // Limit to smaller of explicit limit or php memory limit
        $limit = $this->parseMemory(static::config()->get('memory_limit'));
        if ($limit) {
            return $limit;
        }

        // Fallback to php memory limit
        $phpLimit = $this->getPHPMemoryLimit();
        if ($phpLimit) {
            return $phpLimit;
        }
    }

    /**
     * Calculate the current memory limit of the server
     *
     * @return float
     */
    protected function getPHPMemoryLimit()
    {
        return $this->parseMemory(trim(ini_get("memory_limit")));
    }

    /**
     * Convert memory limit string to bytes.
     * Based on implementation in install.php5
     *
     * @param string $memString
     *
     * @return float
     */
    protected function parseMemory($memString)
    {
        switch (strtolower(substr($memString, -1))) {
            case "b":
                return round(substr($memString, 0, -1));
            case "k":
                return round(substr($memString, 0, -1) * 1024);
            case "m":
                return round(substr($memString, 0, -1) * 1024 * 1024);
            case "g":
                return round(substr($memString, 0, -1) * 1024 * 1024 * 1024);
            default:
                return round($memString);
        }
    }

    protected function humanReadable($size)
    {
        $filesizename = [" Bytes", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"];
        return $size ? round($size / pow(1024, ($i = floor(log($size, 1024)))), 2) . $filesizename[$i] : '0 Bytes';
    }

    /**
     * Gets a list of all the current jobs (or jobs that have recently finished)
     *
     * @param string $type
     *          if we're after a particular job list
     * @param int $includeUpUntil
     *          The number of seconds to include jobs that have just finished, allowing a job list to be built that
     *          includes recently finished jobs
     *
     * @return DataList|QueuedJobDescriptor[]
     */
    public function getJobList($type = null, $includeUpUntil = 0)
    {
        return DataObject::get(
            QueuedJobDescriptor::class,
            $this->getJobListFilter($type, $includeUpUntil)
        );
    }

    /**
     * Return the SQL filter used to get the job list - this is used by the UI for displaying the job list...
     *
     * @param string $type
     *          if we're after a particular job list
     * @param int $includeUpUntil
     *          The number of seconds to include jobs that have just finished, allowing a job list to be built that
     *          includes recently finished jobs
     *
     * @return string
     */
    public function getJobListFilter($type = null, $includeUpUntil = 0)
    {
        $util = singleton(QJUtils::class);

        $filter = ['JobStatus <>' => QueuedJob::STATUS_COMPLETE];
        if ($includeUpUntil) {
            $filter['JobFinished > '] = DBDatetime::create()->setValue(
                DBDatetime::now()->getTimestamp() - $includeUpUntil
            )->Rfc2822();
        }

        $filter = $util->dbQuote($filter, ' OR ');

        if ($type) {
            $filter = $util->dbQuote(['JobType =' => (string)$type]) . ' AND (' . $filter . ')';
        }

        return $filter;
    }

    /**
     * Process the job queue with the current queue runner
     *
     * @param string $queue
     */
    public function runQueue($queue)
    {
        if (!self::config()->get('disable_health_check')) {
            $this->checkJobHealth($queue);
        }
        $this->checkdefaultJobs($queue);
        $this->queueRunner->runQueue($queue);
    }

    /**
     * Process all jobs from a given queue
     *
     * @param string $name The job queue to completely process
     */
    public function processJobQueue($name)
    {
        // Start timer to measure lifetime
        $this->markStarted();

        // Begin main loop
        do {
            if (class_exists(Subsite::class)) {
                // clear subsite back to default to prevent any subsite changes from leaking to
                // subsequent actions
                Subsite::changeSubsite(0);
            }

            $job = $this->getNextPendingJob($name);
            if ($job) {
                $success = $this->runJob($job->ID);
                if (!$success) {
                    // make sure job is null so it doesn't continue the current
                    // processing loop. Next queue executor can pick up where
                    // things left off
                    $job = null;
                }
            }
        } while ($job);
    }

    /**
     * When PHP shuts down, we want to process all of the immediate queue items
     *
     * We use the 'getNextPendingJob' method, instead of just iterating the queue, to ensure
     * we ignore paused or stalled jobs.
     */
    public function onShutdown()
    {
        $this->processJobQueue(QueuedJob::IMMEDIATE);
    }

    /**
     * Get a logger
     *
     * @return LoggerInterface
     */
    public function getLogger()
    {
        return Injector::inst()->get(LoggerInterface::class);
    }
}


<?php

namespace Symbiote\QueuedJobs\Controllers;

use SilverStripe\Admin\AdminRootController;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Convert;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Dev\DebugView;
use SilverStripe\Dev\TaskRunner;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;
use Symbiote\QueuedJobs\Jobs\RunBuildTaskJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;
use Symbiote\QueuedJobs\Tasks\CreateQueuedJobTask;
use Symbiote\QueuedJobs\Tasks\DeleteAllJobsTask;
use Symbiote\QueuedJobs\Tasks\ProcessJobQueueChildTask;
use Symbiote\QueuedJobs\Tasks\ProcessJobQueueTask;

class QueuedTaskRunner extends TaskRunner
{

    private static $url_handlers = [
        'queue/$TaskName' => 'queueTask',
    ];

    private static $allowed_actions = [
        'queueTask',
    ];

    private static $task_blacklist = [
        ProcessJobQueueTask::class,
        ProcessJobQueueChildTask::class,
        CreateQueuedJobTask::class,
        DeleteAllJobsTask::class,
    ];

    public function index()
    {
        $tasks = $this->getTasks();

        $blacklist = (array)$this->config()->task_blacklist;
        $backlistedTasks = [];

        // Web mode
        if (!Director::is_cli()) {
            $renderer = new DebugView();
            echo $renderer->renderHeader();
            echo $renderer->renderInfo(
                "SilverStripe Development Tools: Tasks (QueuedJobs version)",
                Director::absoluteBaseURL()
            );
            $base = Director::absoluteBaseURL();

            echo "
";
            echo "
Queueable jobs
\n";
            echo "
By default these jobs will be added the job queue, rather than run immediately
\n";
            echo "
";
            foreach ($tasks as $task) {
                if (in_array($task['class'], $blacklist)) {
                    $backlistedTasks[] = $task;
                    continue;
                }

                $queueLink = $base . "dev/tasks/queue/" . $task['segment'];
                $immediateLink = $base . "dev/tasks/" . $task['segment'];

                echo "

";
                echo "" . $task['title'] . " [run immediately]
";
                echo "" . $task['description'] . "";
                echo "

\n";
            }
            echo "

";

            echo "
";
            echo "
Non-queueable tasks
\n";
            echo "
These tasks shouldn't be added the queuejobs queue, but you can run them immediately.
\n";
            echo "
";
            foreach ($backlistedTasks as $task) {
                $immediateLink = $base . "dev/tasks/" . $task['segment'];

                echo "

";
                echo "" . $task['title'] . "
";
                echo "" . $task['description'] . "";
                echo "

\n";
            }
            echo "

";

            echo $renderer->renderFooter();

            // CLI mode - revert to default behaviour
        } else {
            return parent::index();
        }
    }

    /**
     * Adds a RunBuildTaskJob to the job queue for a given task
     *
     * @param HTTPRequest $request
     */
    public function queueTask($request)
    {
        $name = $request->param('TaskName');
        $tasks = $this->getTasks();

        $variables = $request->getVars();
        unset($variables['url']);
        unset($variables['flush']);
        unset($variables['flushtoken']);
        unset($variables['isDev']);
        $querystring = http_build_query($variables);

        $title = function ($content) {
            printf(Director::is_cli() ? "%s\n\n" : '
%s
', $content);
        };

        $message = function ($content) {
            printf(Director::is_cli() ? "%s\n" : '
%s
', $content);
        };

        foreach ($tasks as $task) {
            if ($task['segment'] == $name) {
                /** @var BuildTask $inst */
                $inst = Injector::inst()->create($task['class']);
                if (!$inst->isEnabled()) {
                    $message('The task is disabled');
                    return;
                }

                $title(sprintf('Queuing Task %s', $inst->getTitle()));

                $job = new RunBuildTaskJob($task['class'], $querystring);
                $jobID = Injector::inst()->get(QueuedJobService::class)->queueJob($job);

                $message('Done: queued with job ID ' . $jobID);
                $adminUrl = Director::baseURL() . AdminRootController::config()->get('url_base');
                $adminLink = $adminUrl . "/queuedjobs/" . str_replace('\\', '-', QueuedJobDescriptor::class);
                $message("Visit queued jobs admin to see job status");
                return;
            }
        }

        $message(sprintf('The build task "%s" could not be found', Convert::raw2xml($name)));
    }
}


<?php

namespace Symbiote\QueuedJobs\Tasks\Engines;

use SilverStripe\Control\Director;
use SilverStripe\Core\Convert;
use SilverStripe\ORM\FieldType\DBDatetime;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
 * Class BaseRunner
 *
 * @author dmooyman
 */
class BaseRunner
{
    /**
     * Returns an instance of the QueuedJobService.
     *
     * @return QueuedJobService
     */
    public function getService()
    {
        return QueuedJobService::singleton();
    }

    /**
     * Write in a format expected by the output medium (CLI/HTML).
     *
     * @param string $line Line to be written out, without the newline character.
     * @param null|string $prefix
     */
    protected function writeLogLine($line, $prefix = null)
    {
        if (!$prefix) {
            $prefix = '[' . DBDatetime::now()->Rfc2822() . '] ';
        }

        if (Director::is_cli()) {
            echo $prefix . $line . "\n";
        } else {
            echo Convert::raw2xml($prefix . $line) . "<br>";
        }
    }

    /**
     * Logs the status of the queued job descriptor.
     *
     * @param bool|null|QueuedJobDescriptor $descriptor
     * @param string $queue
     */
    protected function logDescriptorStatus($descriptor, $queue)
    {
        if (is_null($descriptor)) {
            $this->writeLogLine('No new jobs on queue ' . $queue);
        }

        if ($descriptor === false) {
            $this->writeLogLine('Job is still running on queue ' . $queue);
        }

        if ($descriptor instanceof QueuedJobDescriptor) {
            $this->writeLogLine('Running ' . $descriptor->JobTitle . ' and others from queue ' . $queue . '.');
        }
    }

    /**
     * Logs the number of current jobs per queue
     */
    public function listJobs()
    {
        $service = $this->getService();
        for ($i = 1; $i <= 3; $i++) {
            $jobs = $service->getJobList($i);
            $num = $jobs ? $jobs->Count() : 0;
            $this->writeLogLine('Found ' . $num . ' jobs for mode ' . $i . '.');
        }
    }
}


<?php

namespace Symbiote\QueuedJobs\Jobs;

use AsyncPHP\Doorman\Cancellable;
use AsyncPHP\Doorman\Expires;
use AsyncPHP\Doorman\Process;
use AsyncPHP\Doorman\Task;
use InvalidArgumentException;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;
use Symbiote\QueuedJobs\Services\QueuedJob;

class DoormanQueuedJobTask implements Task, Expires, Process, Cancellable
{
    /**
     * @var int
     */
    protected $id;

    /**
     * @var QueuedJobDescriptor
     */
    protected $descriptor;

    /**
     * Reload descriptor from DB
     */
    protected function refreshDescriptor()
    {
        if ($this->descriptor) {
            $this->descriptor = QueuedJobDescriptor::get()->byID($this->descriptor->ID);
        }
    }

    /**
     * @inheritdoc
     *
     * @return null|int
     */
    public function getId()
    {
        return $this->id;
    }

    /**
     * @inheritdoc
     *
     * @param int $id
     *
     * @return $this
     */
    public function setId($id)
    {
        $this->id = $id;

        return $this;
    }

    /**
     * @return QueuedJobDescriptor
     */
    public function getDescriptor()
    {
        return $this->descriptor;
    }

    /**
     * @param QueuedJobDescriptor $descriptor
     */
    public function __construct(QueuedJobDescriptor $descriptor)
    {
        $this->descriptor = $descriptor;
    }

    /**
     * @inheritdoc
     *
     * @return string
     */
    public function serialize()
    {
        return serialize(array(
            'descriptor' => $this->descriptor->ID,
        ));
    }

    /**
     * @inheritdoc
     *
     * @throws InvalidArgumentException
     * @param string
     */
    public function unserialize($serialized)
    {
        $data = unserialize($serialized);

        if (!isset($data['descriptor'])) {
            throw new InvalidArgumentException('Malformed data');
        }

        $descriptor = QueuedJobDescriptor::get()
            ->filter('ID', $data['descriptor'])
            ->first();

        if (!$descriptor) {
            throw new InvalidArgumentException('Descriptor not found');
        }

        $this->descriptor = $descriptor;
    }

    /**
     * @return string
     */
    public function getHandler()
    {
        return 'DoormanQueuedJobHandler';
    }

    /**
     * @return array
     */
    public function getData()
    {
        return array(
            'descriptor' => $this->descriptor,
        );
    }

    /**
     * @return bool
     */
    public function ignoresRules()
    {
        if ($this->descriptor->hasMethod('ignoreRules')) {
            return $this->descriptor->ignoreRules();
        }

        return false;
    }

    /**
     * @return bool
     */
    public function stopsSiblings()
    {
        if ($this->descriptor->hasMethod('stopsSiblings')) {
            return $this->descriptor->stopsSiblings();
        }

        return false;
    }

    /**
     * @inheritdoc
     *
     * @return int
     */
    public function getExpiresIn()
    {
        if ($this->descriptor->hasMethod('getExpiresIn')) {
            return $this->descriptor->getExpiresIn();
        }

        return -1;
    }

    /**
     * @inheritdoc
     *
     * @param int $startedAt
     * @return bool
     */
    public function shouldExpire($startedAt)
    {
        if ($this->descriptor->hasMethod('shouldExpire')) {
            return $this->descriptor->shouldExpire($startedAt);
        }

        return true;
    }

    /**
     * @inheritdoc
     *
     * @return bool
     */
    public function canRunTask()
    {
        $this->refreshDescriptor();
        return in_array(
            $this->descriptor->JobStatus,
            array(
                QueuedJob::STATUS_NEW,
                QueuedJob::STATUS_INIT,
                QueuedJob::STATUS_WAIT
            )
        );
    }

    /**
     * @inheritdoc
     *
     * @return bool
     */
    public function isCancelled()
    {
        $this->refreshDescriptor();

        // Treat completed jobs as cancelled when it comes to how Doorman handles picking up jobs to run
        $cancelledStates = [
            QueuedJob::STATUS_CANCELLED,
            QueuedJob::STATUS_COMPLETE,
        ];

        return in_array($this->descriptor->JobStatus, $cancelledStates, true);
    }
}


<?php
namespace Symbiote\QueuedJobs\Tasks;

use SilverStripe\Control\HTTPRequest;
use SilverStripe\Dev\BuildTask;
use Symbiote\QueuedJobs\Services\QueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;

class CheckJobHealthTask extends BuildTask
{
    /**
     * {@inheritDoc}
     * @var string
     */
    private static $segment = 'CheckJobHealthTask';

    /**
     * {@inheritDoc}
     * @return string
     */
    public function getDescription()
    {
        return _t(
            __CLASS__ . '.Description',
            'A task used to check the health of jobs that are "running". Pass a specific queue as the "queue" ' .
            'parameter or otherwise the "Queued" queue will be checked'
        );
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return
     */
    public function run($request)
    {
        $queue = $request->requestVar('queue') ?: QueuedJob::QUEUED;

        $stalledJobCount = $this->getService()->checkJobHealth($queue);

        echo $stalledJobCount === 0 ? 'All jobs are healthy' : 'Detected and attempted restart on ' . $stalledJobCount .
            ' stalled jobs';
    }

    protected function getService()
    {
        return QueuedJobService::singleton();
    }
}


<?php

namespace Symbiote\QueuedJobs\DataObjects;

use SilverStripe\Assets\Filesystem;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Convert;
use SilverStripe\Forms\DropdownField;
use SilverStripe\Forms\FieldList;
use SilverStripe\Forms\LiteralField;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\FieldType\DBField;
use SilverStripe\Security\Member;
use SilverStripe\Security\Permission;
use Symbiote\QueuedJobs\Services\QueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
 * A QueuedJobDescriptor is the stored representation of a piece of work that could take a while to execute,
 * because of which it is desirable to not have it executing in parallel to other jobs.
 *
 * A queued job should always attempt to report how many potential dataobjects will be affected by being executed;
 * this will determine which queue it is placed within so that some shorter jobs can execute immediately without needing
 * to wait for a potentially long running job.
 *
 * @property string $JobTitle Name of job
 * @property string $Signature Unique identifier for this job instance
 * @property string $Implementation Classname of underlying job
 * @property string $StartAfter Don't start until this date, if set
 * @property string $JobStarted When this job was started
 * @property string $JobFinished When this job was finished
 * @property int $TotalSteps Number of steps
 * @property int $StepsProcessed Number of completed steps
 * @property int $LastProcessedCount Number at which StepsProcessed was last checked for stalled jobs
 * @property int $ResumeCounts Number of times this job has been resumed
 * @property string $SavedJobData serialised data for the job to use as storage
 * @property string $SavedJobMessages List of messages saved for this job
 * @property string $JobStatus Status of this job
 * @property string $JobType Type of job
 *
 * @method Member RunAs() Member to run this job as
 *
 * @author Marcus Nyeholt <marcus@symbiote.com.au>
 * @license BSD http://silverstripe.org/bsd-license/
 */
class QueuedJobDescriptor extends DataObject
{
    /**
     * {@inheritDoc}
     * @var string
     */
    private static $table_name = 'QueuedJobDescriptor';

    /**
     * @var array
     */
    private static $db = [
        'JobTitle' => 'Varchar(255)',
        'Signature' => 'Varchar(64)',
        'Implementation' => 'Varchar(255)',
        'StartAfter' => 'DBDatetime',
        'JobStarted' => 'DBDatetime',
        'JobRestarted' => 'DBDatetime',
        'JobFinished' => 'DBDatetime',
        'TotalSteps' => 'Int',
        'StepsProcessed' => 'Int',
        'LastProcessedCount' => 'Int(-1)', // -1 means never checked, 0 means checked but no work is done
        'ResumeCounts' => 'Int',
        'SavedJobData' => 'Text',
        'SavedJobMessages' => 'Text',
        'JobStatus' => 'Varchar(16)',
        'JobType' => 'Varchar(16)',
    ];

    /**
     * @var array
     */
    private static $has_one = [
        'RunAs' => Member::class,
    ];

    /**
     * @var array
     */
    private static $defaults = [
        'JobStatus' => 'New',
        'ResumeCounts' => 0,
        'LastProcessedCount' => -1 // -1 means never checked, 0 means checked and none were processed
    ];

    /**
     * @var array
     */
    private static $indexes = [
        'JobStatus' => true,
        'StartAfter' => true,
        'Signature' => true,
    ];

    /**
     * @var array
     */
    private static $casting = [
        'Messages' => 'HTMLText',
    ];

    /**
     * @var array
     */
    private static $searchable_fields = [
        'JobTitle',
    ];

    /**
     * @var string
     */
    private static $default_sort = 'Created DESC';

    public function requireDefaultRecords()
    {
        parent::requireDefaultRecords();
        $this->getJobDir();
    }

    /**
     * @return array
     */
    public function summaryFields()
    {
        return [
            'JobTitle' => _t(__CLASS__ . '.TABLE_TITLE', 'Title'),
            'Created' => _t(__CLASS__ . '.TABLE_ADDE', 'Added'),
            'JobStarted' => _t(__CLASS__ . '.TABLE_STARTED', 'Started'),
            'JobFinished' => _t(__CLASS__ . '.TABLE_FINISHED', 'Finished'),
//          'JobRestarted' => _t(__CLASS__ . '.TABLE_RESUMED', 'Resumed'),
            'StartAfter' => _t(__CLASS__ . '.TABLE_START_AFTER', 'Start After'),
            'JobTypeString' => _t(__CLASS__ . '.JOB_TYPE', 'Job Type'),
            'JobStatus' => _t(__CLASS__ . '.TABLE_STATUS', 'Status'),
            'LastMessage' => _t(__CLASS__ . '.TABLE_MESSAGES', 'Message'),
            'StepsProcessed' => _t(__CLASS__ . '.TABLE_NUM_PROCESSED', 'Done'),
            'TotalSteps' => _t(__CLASS__ . '.TABLE_TOTAL', 'Total'),
        ];
    }

    /**
     * Pause this job, but only if it is waiting, running, or init
     *
     * @param bool $force Pause this job even if it's not waiting, running, or init
     *
     * @return bool Return true if this job was paused
     */
    public function pause($force = false)
    {
        if ($force || in_array(
            $this->JobStatus,
            [QueuedJob::STATUS_WAIT, QueuedJob::STATUS_RUN, QueuedJob::STATUS_INIT]
        )) {
            $this->JobStatus = QueuedJob::STATUS_PAUSED;
            $this->write();
            return true;
        }
        return false;
    }

    /**
     * Resume this job and schedules it for execution
     *
     * @param bool $force Resume this job even if it's not paused or broken
     *
     * @return bool Return true if this job was resumed
     */
    public function resume($force = false)
    {
        if ($force || in_array($this->JobStatus, [QueuedJob::STATUS_PAUSED, QueuedJob::STATUS_BROKEN])) {
            $this->JobStatus = QueuedJob::STATUS_WAIT;
            $this->ResumeCounts++;
            $this->write();
            QueuedJobService::singleton()->startJob($this);
            return true;
        }
        return false;
    }

    /**
     * Restarts this job via a forced resume
     */
    public function restart()
    {
        $this->resume(true);
    }

    /**
     * Called to indicate that the job is ready to be run on the queue. This is done either as the result of
     * creating the job and adding it, or when resuming.
     */
    public function activateOnQueue()
    {
        // if it's an immediate job, lets cache it to disk to be picked up later
        if ($this->JobType == QueuedJob::IMMEDIATE
            && !Config::inst()->get(QueuedJobService::class, 'use_shutdown_function')
        ) {
            touch($this->getJobDir() . '/queuedjob-' . $this->ID);
        }
    }

    /**
     * Gets the path to the queuedjob cache directory
     *
     * @return string
     */
    protected function getJobDir()
    {
        // make sure our temp dir is in place. This is what will be inotify watched
        $jobDir = Config::inst()->get(QueuedJobService::class, 'cache_dir');
        if ($jobDir{0} !== '/') {
            $jobDir = TEMP_FOLDER . '/' . $jobDir;
        }

        if (!is_dir($jobDir)) {
            Filesystem::makeFolder($jobDir);
        }
        return $jobDir;
    }

    public function execute()
    {
        $service = QueuedJobService::singleton();
        $service->runJob($this->ID);
    }

    /**
     * Called when the job has completed and we want to cleanup anything the descriptor has lying around
     * in caches or the like.
     */
    public function cleanupJob()
    {
        // remove the job's temp file if it exists
        $tmpFile = $this->getJobDir() . '/queuedjob-' . $this->ID;
        if (file_exists($tmpFile)) {
            unlink($tmpFile);
        }
    }

    public function onBeforeDelete()
    {
        parent::onBeforeDelete();
        $this->cleanupJob();
    }

    /**
     * Get all job messages as an HTML unordered list.
     *
     * @return string|null
     */
    public function getMessages()
    {
        if (strlen($this->SavedJobMessages)) {
            $messages = @unserialize($this->SavedJobMessages);
            if (!empty($messages)) {
                return DBField::create_field(
                    'HTMLText',
                    '

' . nl2br(implode('

', Convert::raw2xml($messages))) . '

'
                );
            }
            return '';
        }
    }

    /**
     * Get the last job message as a raw string
     *
     * @return string|null
     */
    public function getLastMessage()
    {
        if (strlen($this->SavedJobMessages)) {
            $msgs = @unserialize($this->SavedJobMessages);
            if (is_array($msgs) && sizeof($msgs)) {
                return array_pop($msgs);
            }
        }
    }

    /**
     * @return string
     */
    public function getTitle()
    {
        return $this->JobTitle;
    }

    /**
     * Return a string representation of the numeric JobType
     * @return string
     */
    public function getJobTypeString()
    {
        $map = $this->getJobTypeValues();
        return isset($map[$this->JobType]) ? $map[$this->JobType] : '(Unknown)';
    }

    /**
     * Return a map of numeric JobType values to localisable string representations.
     * @return array
     */
    public function getJobTypeValues()
    {
        return [
            QueuedJob::IMMEDIATE => _t(__CLASS__ . '.TYPE_IMMEDIATE', 'Immediate'),
            QueuedJob::QUEUED => _t(__CLASS__ . '.TYPE_QUEUED', 'Queued'),
            QueuedJob::LARGE => _t(__CLASS__ . '.TYPE_LARGE', 'Large'),
        ];
    }

    /**
     * @return FieldList
     */
    public function getCMSFields()
    {
        $fields = parent::getCMSFields();
        $fields->replaceField(
            'JobType',
            new DropdownField('JobType', $this->fieldLabel('JobType'), $this->getJobTypeValues())
        );
        $statuses = [
            QueuedJob::STATUS_NEW,
            QueuedJob::STATUS_INIT,
            QueuedJob::STATUS_RUN,
            QueuedJob::STATUS_WAIT,
            QueuedJob::STATUS_COMPLETE,
            QueuedJob::STATUS_PAUSED,
            QueuedJob::STATUS_CANCELLED,
            QueuedJob::STATUS_BROKEN,
        ];
        $fields->replaceField(
            'JobStatus',
            DropdownField::create('JobStatus', $this->fieldLabel('JobStatus'), array_combine($statuses, $statuses))
        );

        $fields->removeByName('SavedJobData');
        $fields->removeByName('SavedJobMessages');

        if (strlen($this->SavedJobMessages)) {
            $fields->addFieldToTab('Root.Messages', LiteralField::create('Messages', $this->getMessages()));
        }

        if (Permission::check('ADMIN')) {
            return $fields;
        }

        // Readonly CMS view is a lot more useful for debugging than no view at all
        return $fields->makeReadonly();
    }
}


<?php

namespace Symbiote\QueuedJobs\Jobs;

use Exception;
use Page;
use SilverStripe\Control\Director;
use SilverStripe\Core\Config\Configurable;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Core\TempFolder;
use SilverStripe\ErrorPage\ErrorPage;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\FieldType\DBDatetime;
use SilverStripe\Versioned\Versioned;
use Symbiote\QueuedJobs\Services\AbstractQueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
 * A job for generating a site's google sitemap
 *
 * If the sitemap module is installed, uses information from that to populate things
 *
 * @author marcus@symbiote.com.au
 * @license http://silverstripe.org/bsd-license/
 */
class GenerateGoogleSitemapJob extends AbstractQueuedJob
{
    use Configurable;

    /**
     * @config
     * @var int
     */
    private static $regenerate_time = 43200;

    public function __construct()
    {
        $this->pagesToProcess = DB::query('SELECT ID FROM "SiteTree_Live" WHERE "ShowInSearch"=1')->column();
        $this->currentStep = 0;
        $this->totalSteps = count($this->pagesToProcess);
    }

    /**
     * Sitemap job is going to run for a while...
     *
     * @return int
     */
    public function getJobType()
    {
        return QueuedJob::QUEUED;
    }

    /**
     * @return string
     */
    public function getTitle()
    {
        return _t(__CLASS__ . '.REGENERATE', 'Regenerate Google sitemap .xml file');
    }

    /**
     * Return a signature for this queued job
     *
     * For the generate sitemap job, we only ever want one instance running, so just use the class name
     *
     * @return string
     */
    public function getSignature()
    {
        return md5(get_class($this));
    }

    /**
     * Note that this is duplicated for backwards compatibility purposes...
     */
    public function setup()
    {
        parent::setup();
        Environment::increaseTimeLimitTo();

        $restart = $this->currentStep == 0;
        if (!$this->tempFile || !file_exists($this->tempFile)) {
            $tmpfile = tempnam(TempFolder::getTempFolder(BASE_PATH), 'sitemap');
            if (file_exists($tmpfile)) {
                $this->tempFile = $tmpfile;
            }
            $restart = true;
        }

        if ($restart) {
            $this->pagesToProcess = DB::query('SELECT ID FROM SiteTree_Live WHERE ShowInSearch=1')->column();
        }
    }

    /**
     * On any restart, make sure to check that our temporary file is being created still.
     */
    public function prepareForRestart()
    {
        parent::prepareForRestart();
        // if the file we've been building is missing, lets fix it up
        if (!$this->tempFile || !file_exists($this->tempFile)) {
            $tmpfile = tempnam(TempFolder::getTempFolder(BASE_PATH), 'sitemap');
            if (file_exists($tmpfile)) {
                $this->tempFile = $tmpfile;
            }
            $this->currentStep = 0;
            $this->pagesToProcess = DB::query('SELECT ID FROM SiteTree_Live WHERE ShowInSearch=1')->column();
        }
    }

    public function process()
    {
        if (!$this->tempFile) {
            throw new Exception("Temporary sitemap file has not been set");
        }

        if (!file_exists($this->tempFile)) {
            throw new Exception("Temporary file $this->tempFile has been deleted!");
        }

        $remainingChildren = $this->pagesToProcess;

        // if there's no more, we're done!
        if (!count($remainingChildren)) {
            $this->completeJob();
            $this->isComplete = true;
            return;
        }

        // lets process our first item - note that we take it off the list of things left to do
        $ID = array_shift($remainingChildren);

        // get the page
        $page = Versioned::get_by_stage(Page::class, Versioned::LIVE, '"SiteTree_Live"."ID" = ' . $ID);

        if (!$page || !$page->Count()) {
            $this->addMessage("Page ID #$ID could not be found, skipping");
        } else {
            $page = $page->First();
        }

        if ($page && $page instanceof Page && !($page instanceof ErrorPage)) {
            if ($page->canView() && (!isset($page->Priority) || $page->Priority > 0)) {
                /** @var DBDatetime $created */
                $created = $page->dbObject('Created');
                $now = DBDatetime::create();
                $now->setValue(DBDatetime::now()->Rfc2822());
                $versions = $page->Version;
                $timediff = $now->format('U') - $created->format('U');

                // Check how many revisions have been made over the lifetime of the
                // Page for a rough estimate of it's changing frequency.
                $period = $timediff / ($versions + 1);

                if ($period > 60 * 60 * 24 * 365) { // > 1 year
                    $page->ChangeFreq = 'yearly';
                } elseif ($period > 60 * 60 * 24 * 30) { // > ~1 month
                    $page->ChangeFreq = 'monthly';
                } elseif ($period > 60 * 60 * 24 * 7) { // > 1 week
                    $page->ChangeFreq = 'weekly';
                } elseif ($period > 60 * 60 * 24) { // > 1 day
                    $page->ChangeFreq = 'daily';
                } elseif ($period > 60 * 60) { // > 1 hour
                    $page->ChangeFreq = 'hourly';
                } else { // < 1 hour
                    $page->ChangeFreq = 'always';
                }

                // do the generation of the file in a temporary location
                $content = $page->renderWith('SitemapEntry');

                $fp = fopen($this->tempFile, "a");
                if (!$fp) {
                    throw new Exception("Could not open $this->tempFile for writing");
                }
                fputs($fp, $content, strlen($content));
                fclose($fp);
            }
        }

        // and now we store the new list of remaining children
        $this->pagesToProcess = $remainingChildren;
        $this->currentStep++;

        if (!count($remainingChildren)) {
            $this->completeJob();
            $this->isComplete = true;
            return;
        }
    }

    /**
     * Outputs the completed file to the site's webroot
     */
    protected function completeJob()
    {
        $content = '<?xml version="1.0" encoding="UTF-8"?>' .
                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
        $content .= file_get_contents($this->tempFile);
        $content .= '</urlset>';

        $sitemap = Director::baseFolder() . '/sitemap.xml';

        file_put_contents($sitemap, $content);

        if (file_exists($this->tempFile)) {
            unlink($this->tempFile);
        }

        $nextgeneration = Injector::inst()->create(GenerateGoogleSitemapJob::class);
        QueuedJobService::singleton()->queueJob(
            $nextgeneration,
            DBDatetime::create()->setValue(
                DBDatetime::now()->getTimestamp() + $this->config()->get('regenerate_time')
            )->Rfc2822()
        );
    }
}


<?php

namespace Symbiote\QueuedJobs\Tasks\Engines;

use AsyncPHP\Doorman\Manager\ProcessManager;
use SilverStripe\Core\Environment;
use SilverStripe\ORM\FieldType\DBDatetime;
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor;
use Symbiote\QueuedJobs\Jobs\DoormanQueuedJobTask;
use Symbiote\QueuedJobs\Services\QueuedJob;
use SilverStripe\Core\Injector\Injector;

/**
 * Runs all jobs through the doorman engine
 */
class DoormanRunner extends BaseRunner implements TaskRunnerEngine
{
    /**
     * @var string[]
     */
    protected $defaultRules = [];

    /**
     * Assign default rules for this task
     *
     * @param array $rules
     * @return $this
     */
    public function setDefaultRules($rules)
    {
        $this->defaultRules = $rules;
        return $this;
    }

    /**
     * @return array List of rules
     */
    public function getDefaultRules()
    {
        return $this->defaultRules;
    }

    /**
     * Run tasks on the given queue
     *
     * @param string $queue
     */
    public function runQueue($queue)
    {
        // split jobs out into multiple tasks...

        /** @var ProcessManager $manager */
        $manager = Injector::inst()->create(ProcessManager::class);
        $manager->setWorker(
            BASE_PATH . "/vendor/silverstripe/framework/cli-script.php dev/tasks/ProcessJobQueueChildTask"
        );
        $logPath = Environment::getEnv('SS_DOORMAN_LOGPATH');
        if ($logPath) {
            $manager->setLogPath($logPath);
        }

        // Assign default rules
        $defaultRules = $this->getDefaultRules();
        if ($defaultRules) {
            foreach ($defaultRules as $rule) {
                $manager->addRule($rule);
            }
        }

        $descriptor = $this->getNextJobDescriptorWithoutMutex($queue);

        while ($manager->tick() || $descriptor) {
            $this->logDescriptorStatus($descriptor, $queue);

            if ($descriptor instanceof QueuedJobDescriptor) {
                $descriptor->JobStatus = QueuedJob::STATUS_INIT;
                $descriptor->write();

                $manager->addTask(new DoormanQueuedJobTask($descriptor));
            }

            sleep(1);

            $descriptor = $this->getNextJobDescriptorWithoutMutex($queue);
        }
    }

    /**
     * @param string $queue
     * @return null|QueuedJobDescriptor
     */
    protected function getNextJobDescriptorWithoutMutex($queue)
    {
        $list = QueuedJobDescriptor::get()
            ->filter('JobType', $queue)
            ->sort('ID', 'ASC');

        $descriptor = $list
            ->filter('JobStatus', QueuedJob::STATUS_WAIT)
            ->first();

        if ($descriptor) {
            return $descriptor;
        }

        return $list
            ->filter('JobStatus', QueuedJob::STATUS_NEW)
            ->where(sprintf(
                '"StartAfter" < \'%s\' OR "StartAfter" IS NULL',
                DBDatetime::now()->getValue()
            ))
            ->first();
    }
}


<?php

namespace Symbiote\QueuedJobs\Forms;

use SilverStripe\Forms\GridField\GridField;
use SilverStripe\Forms\GridField\GridField_ActionProvider;
use SilverStripe\Forms\GridField\GridField_ColumnProvider;
use SilverStripe\Forms\GridField\GridField_FormAction;
use SilverStripe\ORM\DataObject;
use SilverStripe\View\Requirements;
use Symbiote\QueuedJobs\Services\QueuedJob;

class GridFieldQueuedJobExecute implements GridField_ColumnProvider, GridField_ActionProvider
{

    protected $action = 'execute';

    /**
     * CSS icon class names for each action (see silverstripe-admin fonts)
     *
     * @var array
     */
    protected $icons = [
        'execute' => 'font-icon-block-media',
        'pause'   => 'font-icon-cancel-circled',
        'resume'  => 'font-icon-sync',
    ];

    /**
     * Call back to see if the record's action icon should be shown.
     *
     * @var callable
     */
    protected $viewCheck;

    /**
     * @param string   $action
     * @param callable $check
     */
    public function __construct($action = 'execute', $check = null)
    {
        $this->action = $action;
        if (!$check) {
            $check = function ($record) {
                return $record->JobStatus == QueuedJob::STATUS_WAIT || $record->JobStatus == QueuedJob::STATUS_NEW;
            };
        }

        $this->viewCheck = $check;
    }

    /**
     * Add a column 'Delete'
     *
     * @param GridField $gridField
     * @param array     $columns
     */
    public function augmentColumns($gridField, &$columns)
    {
        if (!in_array('Actions', $columns)) {
            $columns[] = 'Actions';
        }
    }

    /**
     * Return any special attributes that will be used for FormField::createTag()
     *
     * @param GridField $gridField
     * @param DataObject $record
     * @param string $columnName
     * @return array
     */
    public function getColumnAttributes($gridField, $record, $columnName)
    {
        return array('class' => 'grid-field__col-compact');
    }

    /**
     * Add the title
     *
     * @param GridField $gridField
     * @param string $columnName
     * @return array
     */
    public function getColumnMetadata($gridField, $columnName)
    {
        if ($columnName == 'Actions') {
            return array('title' => '');
        }
    }

    /**
     * Which columns are handled by this component
     *
     * @param GridField $gridField
     * @return array
     */
    public function getColumnsHandled($gridField)
    {
        return array('Actions');
    }

    /**
     * Which GridField actions are this component handling
     *
     * @param GridField $gridField
     * @return array
     */
    public function getActions($gridField)
    {
        return array('execute', 'pause', 'resume');
    }

    /**
     * @param GridField $gridField
     * @param DataObject $record
     * @param string $columnName
     * @return string|void - the HTML for the column
     */
    public function getColumnContent($gridField, $record, $columnName)
    {
        $icon = $this->icons[$this->action];

        if ($this->viewCheck) {
            $func = $this->viewCheck;
            if (!$func($record)) {
                return;
            }
        }

        $field = GridField_FormAction::create(
            $gridField,
            'ExecuteJob' . $record->ID,
            false,
            $this->action,
            array('RecordID' => $record->ID)
        );

        $humanTitle = ucfirst($this->action);
        $title = _t(__CLASS__ . '.' . $humanTitle, $humanTitle);

        $field
            ->addExtraClass('gridfield-button-job' . $this->action)
            ->addExtraClass($icon)
            ->addExtraClass('btn--icon-md btn--no-text grid-field__icon-action')
            ->setAttribute('title', $title)
            ->setDescription($title);

        return $field->Field();
    }

    /**
     * Handle the actions and apply any changes to the GridField
     *
     * @param GridField $gridField
     * @param string $actionName
     * @param mixed $arguments
     * @param array $data - form data
     * @return void
     */
    public function handleAction(GridField $gridField, $actionName, $arguments, $data)
    {
        $actions = $this->getActions(null);
        if (in_array($actionName, $actions)) {
            $item = $gridField->getList()->byID($arguments['RecordID']);
            if (!$item) {
                return;
            }
            $item->$actionName();
            Requirements::clear();
        }
    }
}


<?php

namespace Symbiote\QueuedJobs\Jobs;

use Page;
use SilverStripe\ORM\DataObject;
use Symbiote\QueuedJobs\Services\AbstractQueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJob;

/**
 * An example queued job
 *
 * Use this as an example of how you can write your own jobs
 *
 * @author Marcus Nyeholt <marcus@symbiote.com.au>
 * @license BSD http://silverstripe.org/bsd-license/
 */
class PublishItemsJob extends AbstractQueuedJob implements QueuedJob
{
    /**
     * @param DataObject $rootNodeID
     */
    public function __construct($rootNodeID = null)
    {
        // this value is automatically persisted between processing requests for
        // this job
        if ($rootNodeID) {
            $this->rootID = $rootNodeID;
        }
    }

    protected function getRoot()
    {
        return DataObject::get_by_id(Page::class, $this->rootID);
    }

    /**
     * Defines the title of the job
     *
     * @return string
     */
    public function getTitle()
    {
        $title = 'Unknown';

        if ($root = $this->getRoot()) {
            $title = $root->Title;
        }

        return _t(
            __CLASS__ . '.Title',
            "Publish items beneath {title}",
            ['title' => $title]
        );
    }

    /**
     * Indicate to the system which queue we think we should be in based
     * on how many objects we're going to touch on while processing.
     *
     * We want to make sure we also set how many steps we think we might need to take to
     * process everything - note that this does not need to be 100% accurate, but it's nice
     * to give a reasonable approximation
     *
     * @return int
     */
    public function getJobType()
    {
        $this->totalSteps = 'Lots';
        return QueuedJob::QUEUED;
    }

    /**
     * This is called immediately before a job begins - it gives you a chance
     * to initialise job data and make sure everything's good to go
     *
     * What we're doing in our case is to queue up the list of items we know we need to
     * process still (it's not everything - just the ones we know at the moment)
     *
     * When we go through, we'll constantly add and remove from this queue, meaning
     * we never overload it with content
     */
    public function setup()
    {
        if (!$this->getRoot()) {
            // we're missing for some reason!
            $this->isComplete = true;
            $this->remainingChildren = array();
            return;
        }
        $remainingChildren = array();
        $remainingChildren[] = $this->getRoot()->ID;
        $this->remainingChildren = $remainingChildren;

        // we reset this to 1; this is because we only know for sure about 1 item remaining
        // as time goes by, this will increase as we discover more items that need processing
        $this->totalSteps = 1;
    }

    /**
     * Lets process a single node, and publish it if necessary
     */
    public function process()
    {
        $remainingChildren = $this->remainingChildren;

        // if there's no more, we're done!
        if (!count($remainingChildren)) {
            $this->isComplete = true;
            return;
        }

        // we need to always increment! This is important, because if we don't then our container
        // that executes around us thinks that the job has died, and will stop it running.
        $this->currentStep++;

        // lets process our first item - note that we take it off the list of things left to do
        $ID = array_shift($remainingChildren);

        // get the page
        $page = DataObject::get_by_id(Page::class, $ID);
        if ($page) {
            // publish it
            $page->doPublish();

            // and add its children to the list to be published
            foreach ($page->Children() as $child) {
                $remainingChildren[] = $child->ID;
                // we increase how many steps we need to do - this means our total steps constantly rises,
                // but it gives users an idea of exactly how many more we know about
                $this->totalSteps++;
            }
            $page->destroy();
            unset($page);
        }

        // and now we store the new list of remaining children
        $this->remainingChildren = $remainingChildren;

        if (!count($remainingChildren)) {
            $this->isComplete = true;
            return;
        }
    }
}


<?php

namespace Symbiote\QueuedJobs\Tasks;

use SilverStripe\Control\HTTPRequest;
use SilverStripe\Dev\BuildTask;
use Symbiote\QueuedJobs\Services\QueuedJob;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
 * Task used to process the job queue
 *
 * @author Marcus Nyeholt <marcus@symbiote.com.au>
 * @license BSD http://silverstripe.org/bsd-license/
 */
class ProcessJobQueueTask extends BuildTask
{
    /**
     * {@inheritDoc}
     * @var string
     */
    private static $segment = 'ProcessJobQueueTask';

    /**
     * @return string
     */
    public function getDescription()
    {
        return _t(
            __CLASS__ . '.Description',
            'Used via a cron job to execute queued jobs that need to be run.'
        );
    }

    /**
     * @param HTTPRequest $request
     */
    public function run($request)
    {
        $service = $this->getService();

        if ($request->getVar('list')) {
            // List helper
            $service->queueRunner->listJobs();
            return;
        }

        // Check if there is a job to run
        if (($job = $request->getVar('job')) && strpos($job, '-')) {
            // Run from a isngle job
            $parts = explode('-', $job);
            $id = $parts[1];
            $service->runJob($id);
            return;
        }

        // Run the queue
        $queue = $this->getQueue($request);
        $service->runQueue($queue);
    }

    /**
     * Resolves the queue name to one of a few aliases.
     *
     * @todo Solve the "Queued"/"queued" mystery!
     *
     * @param HTTPRequest $request
     * @return string
     */
    protected function getQueue($request)
    {
        $queue = $request->getVar('queue');

        if (!$queue) {
            $queue = 'Queued';
        }

        switch (strtolower($queue)) {
            case 'immediate':
                $queue = QueuedJob::IMMEDIATE;
                break;
            case 'queued':
                $queue = QueuedJob::QUEUED;
                break;
            case 'large':
                $queue = QueuedJob::LARGE;
                break;
            default:
                break;
        }

        return $queue;
    }

    /**
     * Returns an instance of the QueuedJobService.
     *
     * @return QueuedJobService
     */
    public function getService()
    {
        return QueuedJobService::singleton();
    }
}