From a4cd53ba8d188090e1510b88087f686fabea217b Mon Sep 17 00:00:00 2001 From: Russell Smith Date: Wed, 17 Apr 2013 11:06:38 +1000 Subject: [PATCH] MDL-17783 cron: Apply exclusive lock to each part of cron It is common for certain operations to take a long time to run. Cron is set to run at regular intervals and needs to be able to skips tasks that are currently being undertaken by another process. This provides certainty for module developers that only a single copy of their cron function will be running at any one time. Once clear example is forum digest processing which can take a long time on large sites. With the locks you can leave that job running and complete all of the other cron tasks. Conflicts: lib/cronlib.php lib/dml/mssql_native_moodle_database.php lib/dml/mysqli_native_moodle_database.php lib/dml/oci_native_moodle_database.php lib/dml/pgsql_native_moodle_database.php lib/dml/sqlsrv_native_moodle_database.php --- lib/cronlib.php | 661 +++++++++++++++++++----------- lib/dml/moodle_database.php | 44 +- lib/dml/mssql_native_moodle_database.php | 22 +- lib/dml/mysqli_native_moodle_database.php | 23 +- lib/dml/oci_native_moodle_database.php | 21 +- lib/dml/pgsql_native_moodle_database.php | 33 +- lib/dml/sqlsrv_native_moodle_database.php | 23 +- 7 files changed, 541 insertions(+), 286 deletions(-) diff --git a/lib/cronlib.php b/lib/cronlib.php index 0f33af8..e3a0d3a 100644 --- a/lib/cronlib.php +++ b/lib/cronlib.php @@ -29,6 +29,15 @@ function cron_run() { global $DB, $CFG, $OUTPUT; + /* + * Each part of the cron needs a lock number that will be the same during + * every run of the cron. We much use a number as PostgreSQL cannot handle + * advisory locks with text in them. To ensure we meet that requirement + * we start with an id of 1 and increment after each lock operation has + * been completed. + */ + $lockid = 1; + if (CLI_MAINTENANCE) { echo "CLI maintenance mode active, cron execution suspended.\n"; exit(1); @@ -67,110 +76,118 @@ function cron_run() { // Run cleanup core cron jobs, but not every time since they aren't too important. // These don't have a timer to reduce load, so we'll use a random number // to randomly choose the percentage of times we should run these jobs. - srand ((double) microtime() * 10000000); - $random100 = rand(0,100); - if ($random100 < 20) { // Approximately 20% of the time. - mtrace("Running clean-up tasks..."); - cron_trace_time_and_memory(); - - // Delete users who haven't confirmed within required period - if (!empty($CFG->deleteunconfirmed)) { - $cuttime = $timenow - ($CFG->deleteunconfirmed * 3600); - $rs = $DB->get_recordset_sql ("SELECT * - FROM {user} - WHERE confirmed = 0 AND firstaccess > 0 - AND firstaccess < ?", array($cuttime)); - foreach ($rs as $user) { - delete_user($user); // we MUST delete user properly first - $DB->delete_records('user', array('id'=>$user->id)); // this is a bloody hack, but it might work - mtrace(" Deleted unconfirmed user for ".fullname($user, true)." ($user->id)"); + try { + $DB->get_cron_lock($lockid); + srand ((double) microtime() * 10000000); + $random100 = rand(0,100); + if ($random100 < 20) { // Approximately 20% of the time. + mtrace("Running clean-up tasks..."); + cron_trace_time_and_memory(); + + // Delete users who haven't confirmed within required period + if (!empty($CFG->deleteunconfirmed)) { + $cuttime = $timenow - ($CFG->deleteunconfirmed * 3600); + $rs = $DB->get_recordset_sql ("SELECT * + FROM {user} + WHERE confirmed = 0 AND firstaccess > 0 + AND firstaccess < ?", array($cuttime)); + foreach ($rs as $user) { + delete_user($user); // we MUST delete user properly first + $DB->delete_records('user', array('id'=>$user->id)); // this is a bloody hack, but it might work + mtrace(" Deleted unconfirmed user for ".fullname($user, true)." ($user->id)"); + } + $rs->close(); } - $rs->close(); - } - // Delete users who haven't completed profile within required period - if (!empty($CFG->deleteincompleteusers)) { - $cuttime = $timenow - ($CFG->deleteincompleteusers * 3600); - $rs = $DB->get_recordset_sql ("SELECT * - FROM {user} - WHERE confirmed = 1 AND lastaccess > 0 - AND lastaccess < ? AND deleted = 0 - AND (lastname = '' OR firstname = '' OR email = '')", - array($cuttime)); - foreach ($rs as $user) { - delete_user($user); - mtrace(" Deleted not fully setup user $user->username ($user->id)"); + // Delete users who haven't completed profile within required period + if (!empty($CFG->deleteincompleteusers)) { + $cuttime = $timenow - ($CFG->deleteincompleteusers * 3600); + $rs = $DB->get_recordset_sql ("SELECT * + FROM {user} + WHERE confirmed = 1 AND lastaccess > 0 + AND lastaccess < ? AND deleted = 0 + AND (lastname = '' OR firstname = '' OR email = '')", + array($cuttime)); + foreach ($rs as $user) { + delete_user($user); + mtrace(" Deleted not fully setup user $user->username ($user->id)"); + } + $rs->close(); } - $rs->close(); - } - // Delete old logs to save space (this might need a timer to slow it down...) - if (!empty($CFG->loglifetime)) { // value in days - $loglifetime = $timenow - ($CFG->loglifetime * 3600 * 24); - $DB->delete_records_select("log", "time < ?", array($loglifetime)); - mtrace(" Deleted old log records"); - } + // Delete old logs to save space (this might need a timer to slow it down...) + if (!empty($CFG->loglifetime)) { // value in days + $loglifetime = $timenow - ($CFG->loglifetime * 3600 * 24); + $DB->delete_records_select("log", "time < ?", array($loglifetime)); + mtrace(" Deleted old log records"); + } - // Delete old backup_controllers and logs. - $loglifetime = get_config('backup', 'loglifetime'); - if (!empty($loglifetime)) { // Value in days. - $loglifetime = $timenow - ($loglifetime * 3600 * 24); - // Delete child records from backup_logs. - $DB->execute("DELETE FROM {backup_logs} - WHERE EXISTS ( - SELECT 'x' - FROM {backup_controllers} bc - WHERE bc.backupid = {backup_logs}.backupid - AND bc.timecreated < ?)", array($loglifetime)); - // Delete records from backup_controllers. - $DB->execute("DELETE FROM {backup_controllers} - WHERE timecreated < ?", array($loglifetime)); - mtrace(" Deleted old backup records"); - } + // Delete old backup_controllers and logs. + $loglifetime = get_config('backup', 'loglifetime'); + if (!empty($loglifetime)) { // Value in days. + $loglifetime = $timenow - ($loglifetime * 3600 * 24); + // Delete child records from backup_logs. + $DB->execute("DELETE FROM {backup_logs} + WHERE EXISTS ( + SELECT 'x' + FROM {backup_controllers} bc + WHERE bc.backupid = {backup_logs}.backupid + AND bc.timecreated < ?)", array($loglifetime)); + // Delete records from backup_controllers. + $DB->execute("DELETE FROM {backup_controllers} + WHERE timecreated < ?", array($loglifetime)); + mtrace(" Deleted old backup records"); + } - // Delete old cached texts - if (!empty($CFG->cachetext)) { // Defined in config.php - $cachelifetime = time() - $CFG->cachetext - 60; // Add an extra minute to allow for really heavy sites - $DB->delete_records_select('cache_text', "timemodified < ?", array($cachelifetime)); - mtrace(" Deleted old cache_text records"); - } + // Delete old cached texts + if (!empty($CFG->cachetext)) { // Defined in config.php + $cachelifetime = time() - $CFG->cachetext - 60; // Add an extra minute to allow for really heavy sites + $DB->delete_records_select('cache_text', "timemodified < ?", array($cachelifetime)); + mtrace(" Deleted old cache_text records"); + } - if (!empty($CFG->usetags)) { - require_once($CFG->dirroot.'/tag/lib.php'); - tag_cron(); - mtrace(' Executed tag cron'); - } + if (!empty($CFG->usetags)) { + require_once($CFG->dirroot.'/tag/lib.php'); + tag_cron(); + mtrace(' Executed tag cron'); + } - // Context maintenance stuff - context_helper::cleanup_instances(); - mtrace(' Cleaned up context instances'); - context_helper::build_all_paths(false); - // If you suspect that the context paths are somehow corrupt - // replace the line below with: context_helper::build_all_paths(true); - mtrace(' Built context paths'); + // Context maintenance stuff + context_helper::cleanup_instances(); + mtrace(' Cleaned up context instances'); + context_helper::build_all_paths(false); + // If you suspect that the context paths are somehow corrupt + // replace the line below with: context_helper::build_all_paths(true); + mtrace(' Built context paths'); - // Remove expired cache flags - gc_cache_flags(); - mtrace(' Cleaned cache flags'); + // Remove expired cache flags + gc_cache_flags(); + mtrace(' Cleaned cache flags'); - // Cleanup messaging - if (!empty($CFG->messagingdeletereadnotificationsdelay)) { - $notificationdeletetime = time() - $CFG->messagingdeletereadnotificationsdelay; - $DB->delete_records_select('message_read', 'notification=1 AND timeread<:notificationdeletetime', array('notificationdeletetime'=>$notificationdeletetime)); - mtrace(' Cleaned up read notifications'); - } + // Cleanup messaging + if (!empty($CFG->messagingdeletereadnotificationsdelay)) { + $notificationdeletetime = time() - $CFG->messagingdeletereadnotificationsdelay; + $DB->delete_records_select('message_read', 'notification=1 AND timeread<:notificationdeletetime', array('notificationdeletetime'=>$notificationdeletetime)); + mtrace(' Cleaned up read notifications'); + } - mtrace("...finished clean-up tasks"); + mtrace("...finished clean-up tasks"); + + } // End of occasional clean-up tasks + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for cleanup tasks, another copy of this is currently running"); + } + $lockid++; - } // End of occasional clean-up tasks // Send login failures notification - brute force protection in moodle is weak, @@ -189,68 +206,88 @@ function cron_run() { session_gc(); mtrace("Cleaned up stale user sessions"); - + mtrace("Running auth crons if required..."); // Run the auth cron, if any before enrolments // because it might add users that will be needed in enrol plugins - $auths = get_enabled_auth_plugins(); - mtrace("Running auth crons if required..."); - cron_trace_time_and_memory(); - foreach ($auths as $auth) { - $authplugin = get_auth_plugin($auth); - if (method_exists($authplugin, 'cron')) { - mtrace("Running cron for auth/$auth..."); - $authplugin->cron(); - if (!empty($authplugin->log)) { - mtrace($authplugin->log); + try { + $DB->get_cron_lock($lockid); + $auths = get_enabled_auth_plugins(); + mtrace("Running auth crons if required..."); + cron_trace_time_and_memory(); + foreach ($auths as $auth) { + $authplugin = get_auth_plugin($auth); + if (method_exists($authplugin, 'cron')) { + mtrace("Running cron for auth/$auth..."); + $authplugin->cron(); + if (!empty($authplugin->log)) { + mtrace($authplugin->log); + } } + unset($authplugin); } - unset($authplugin); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for auth crons, another copy of this is currently running"); } - // Generate new password emails for users - ppl expect these generated asap - if ($DB->count_records('user_preferences', array('name'=>'create_password', 'value'=>'1'))) { - mtrace('Creating passwords for new users...'); - $newusers = $DB->get_recordset_sql("SELECT u.id as id, u.email, u.firstname, + $lockid++; + + mtrace('Creating passwords for new users...'); + try { + $DB->get_cron_lock($lockid); + // Generate new password emails for users - ppl expect these generated asap + if ($DB->count_records('user_preferences', array('name'=>'create_password', 'value'=>'1'))) { + $newusers = $DB->get_recordset_sql("SELECT u.id as id, u.email, u.firstname, u.lastname, u.username, u.lang, p.id as prefid FROM {user} u JOIN {user_preferences} p ON u.id=p.userid WHERE p.name='create_password' AND p.value='1' AND u.email !='' AND u.suspended = 0 AND u.auth != 'nologin' AND u.deleted = 0"); - // note: we can not send emails to suspended accounts - foreach ($newusers as $newuser) { - // Use a low cost factor when generating bcrypt hash otherwise - // hashing would be slow when emailing lots of users. Hashes - // will be automatically updated to a higher cost factor the first - // time the user logs in. - if (setnew_password_and_mail($newuser, true)) { - unset_user_preference('create_password', $newuser); - set_user_preference('auth_forcepasswordchange', 1, $newuser); - } else { - trigger_error("Could not create and mail new user password!"); + // note: we can not send emails to suspended accounts + foreach ($newusers as $newuser) { + // Use a low cost factor when generating bcrypt hash otherwise + // hashing would be slow when emailing lots of users. Hashes + // will be automatically updated to a higher cost factor the first + // time the user logs in. + if (setnew_password_and_mail($newuser)) { + unset_user_preference('create_password', $newuser); + set_user_preference('auth_forcepasswordchange', 1, $newuser); + } else { + trigger_error("Could not create and mail new user password!"); + } } + $newusers->close(); } - $newusers->close(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for user password task, another copy of this is currently running"); } + $lockid++; - - // It is very important to run enrol early - // because other plugins depend on correct enrolment info. mtrace("Running enrol crons if required..."); - $enrols = enrol_get_plugins(true); - foreach($enrols as $ename=>$enrol) { - // do this for all plugins, disabled plugins might want to cleanup stuff such as roles - if (!$enrol->is_cron_required()) { - continue; + try { + $DB->get_cron_lock($lockid); + // It is very important to run enrol early + // because other plugins depend on correct enrolment info. + $enrols = enrol_get_plugins(true); + foreach($enrols as $ename=>$enrol) { + // do this for all plugins, disabled plugins might want to cleanup stuff such as roles + if (!$enrol->is_cron_required()) { + continue; + } + mtrace("Running cron for enrol_$ename..."); + cron_trace_time_and_memory(); + $enrol->cron(); + $enrol->set_config('lastcron', time()); } - mtrace("Running cron for enrol_$ename..."); - cron_trace_time_and_memory(); - $enrol->cron(); - $enrol->set_config('lastcron', time()); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for enrol crons, another copy of this is currently running"); } + $lockid++; - - // Run all cron jobs for each module mtrace("Starting activity modules"); + // Run all cron jobs for each module get_mailer('buffer'); if ($mods = $DB->get_records_select("modules", "cron > 0 AND ((? - lastcron) > cron) AND visible = 1", array($timenow))) { foreach ($mods as $mod) { @@ -259,21 +296,27 @@ function cron_run() { include_once($libfile); $cron_function = $mod->name."_cron"; if (function_exists($cron_function)) { - mtrace("Processing module function $cron_function ...", ''); - cron_trace_time_and_memory(); - $pre_dbqueries = null; - $pre_dbqueries = $DB->perf_get_queries(); - $pre_time = microtime(1); - if ($cron_function()) { - $DB->set_field("modules", "lastcron", $timenow, array("id"=>$mod->id)); - } - if (isset($pre_dbqueries)) { - mtrace("... used " . ($DB->perf_get_queries() - $pre_dbqueries) . " dbqueries"); - mtrace("... used " . (microtime(1) - $pre_time) . " seconds"); + try { + mtrace("Processing module function $cron_function ...", ''); + cron_trace_time_and_memory(); + $DB->get_cron_lock($mod->id, LOCK_TIMEOUT, CRON_MODULES_LOCK); + $pre_dbqueries = null; + $pre_dbqueries = $DB->perf_get_queries(); + $pre_time = microtime(1); + if ($cron_function()) { + $DB->set_field("modules", "lastcron", $timenow, array("id"=>$mod->id)); + } + if (isset($pre_dbqueries)) { + mtrace("... used " . ($DB->perf_get_queries() - $pre_dbqueries) . " dbqueries"); + mtrace("... used " . (microtime(1) - $pre_time) . " seconds"); + } + // Reset possible changes by modules to time_limit. MDL-11597 + @set_time_limit(0); + $DB->release_cron_lock($mod->id, CRON_MODULES_LOCK); + mtrace("done."); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for $cron_function, another copy of this is currently running"); } - // Reset possible changes by modules to time_limit. MDL-11597 - @set_time_limit(0); - mtrace("done."); } } } @@ -293,75 +336,138 @@ function cron_run() { $classname = 'block_'.$block->name; $blockobj = new $classname; if (method_exists($blockobj,'cron')) { - mtrace("Processing cron function for ".$block->name.'....',''); - cron_trace_time_and_memory(); - if ($blockobj->cron()) { - $DB->set_field('block', 'lastcron', $timenow, array('id'=>$block->id)); + try { + mtrace("Processing cron function for ".$block->name.'....',''); + cron_trace_time_and_memory(); + $DB->get_cron_lock($block->id, LOCK_TIMEOUT, CRON_BLOCKS_LOCK); + if ($blockobj->cron()) { + $DB->set_field('block', 'lastcron', $timenow, array('id'=>$block->id)); + } + // Reset possible changes by blocks to time_limit. MDL-11597 + @set_time_limit(0); + $DB->release_cron_lock($block->id, CRON_BLOCKS_LOCK); + mtrace('done.'); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for ".$block->name." cron, another copy of this is currently running"); } - // Reset possible changes by blocks to time_limit. MDL-11597 - @set_time_limit(0); - mtrace('done.'); } } - } } mtrace('Finished blocks'); mtrace('Starting admin reports'); - cron_execute_plugin_type('report'); + try { + $DB->get_cron_lock($lockid); + cron_execute_plugin_type('report'); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for admin report cron, another copy of this is currently running"); + } mtrace('Finished admin reports'); + $lockid++; mtrace('Starting main gradebook job...'); - cron_trace_time_and_memory(); - grade_cron(); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + grade_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for main gradebook cron, another copy of this is currently running"); + } mtrace('done.'); + $lockid++; mtrace('Starting processing the event queue...'); - cron_trace_time_and_memory(); - events_cron(); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + events_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for events cron, another copy of this is currently running"); + } mtrace('done.'); + $lockid++; if ($CFG->enablecompletion) { - // Completion cron mtrace('Starting the completion cron...'); - cron_trace_time_and_memory(); - require_once($CFG->dirroot.'/completion/cron.php'); - completion_cron(); + // Completion cron + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + require_once($CFG->libdir . '/completion/cron.php'); + completion_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for completion cron, another copy of this is currently running"); + } mtrace('done'); } + $lockid++; if ($CFG->enableportfolios) { // Portfolio cron mtrace('Starting the portfolio cron...'); - cron_trace_time_and_memory(); - require_once($CFG->libdir . '/portfoliolib.php'); - portfolio_cron(); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + require_once($CFG->libdir . '/portfoliolib.php'); + portfolio_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for portfolio cron, another copy of this is currently running"); + } mtrace('done'); } - - - //now do plagiarism checks - require_once($CFG->libdir.'/plagiarismlib.php'); - plagiarism_cron(); + $lockid++; + + + // now do plagiarism checks + mtrace('Starting the plagiarism cron...'); + try { + $DB->get_cron_lock($lockid); + require_once($CFG->libdir.'/plagiarismlib.php'); + plagiarism_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for plagiarism cron, another copy of this is currently running"); + } + mtrace('Finished the plagiarism cron'); + $lockid++; mtrace('Starting course reports'); - cron_execute_plugin_type('coursereport'); + try { + $DB->get_cron_lock($lockid); + cron_execute_plugin_type('coursereport'); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for course report cron, another copy of this is currently running"); + } mtrace('Finished course reports'); + $lockid++; // run gradebook import/export/report cron mtrace('Starting gradebook plugins'); - cron_execute_plugin_type('gradeimport'); - cron_execute_plugin_type('gradeexport'); - cron_execute_plugin_type('gradereport'); + try { + $DB->get_cron_lock($lockid); + cron_execute_plugin_type('gradeimport'); + cron_execute_plugin_type('gradeexport'); + cron_execute_plugin_type('gradereport'); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for gradebook plugin crons, another copy of this is currently running"); + } mtrace('Finished gradebook plugins'); + $lockid++; // run calendar cron require_once "{$CFG->dirroot}/calendar/lib.php"; @@ -371,51 +477,89 @@ function cron_run() { if (!empty($CFG->enableblogs) && $CFG->useexternalblogs) { require_once($CFG->dirroot . '/blog/lib.php'); mtrace("Fetching external blog entries...", ''); - cron_trace_time_and_memory(); - $sql = "timefetched < ? OR timefetched = 0"; - $externalblogs = $DB->get_records_select('blog_external', $sql, array(time() - $CFG->externalblogcrontime)); - - foreach ($externalblogs as $eb) { - blog_sync_external_entries($eb); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + $sql = "timefetched < ? OR timefetched = 0"; + $externalblogs = $DB->get_records_select('blog_external', $sql, array(time() - $CFG->externalblogcrontime)); + foreach ($externalblogs as $eb) { + blog_sync_external_entries($eb); + } + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for blogs cron, another copy of this is currently running"); } mtrace('done.'); } - // Run blog associations cleanup - if (!empty($CFG->enableblogs) && $CFG->useblogassociations) { + $lockid++; + + + if ($CFG->useblogassociations) { require_once($CFG->dirroot . '/blog/lib.php'); - // delete entries whose contextids no longer exists mtrace("Deleting blog associations linked to non-existent contexts...", ''); - cron_trace_time_and_memory(); - $DB->delete_records_select('blog_association', 'contextid NOT IN (SELECT id FROM {context})'); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + // Run blog associations cleanup + // delete entries whose contextids no longer exists + $DB->delete_records_select('blog_association', 'contextid NOT IN (SELECT id FROM {context})'); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for blog associations cron, another copy of this is currently running"); + } mtrace('done.'); } + $lockid++; // Run question bank clean-up. mtrace("Starting the question bank cron...", ''); - cron_trace_time_and_memory(); - require_once($CFG->libdir . '/questionlib.php'); - question_bank::cron(); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + require_once($CFG->libdir . '/questionlib.php'); + question_bank::cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for questions cron, another copy of this is currently running"); + } mtrace('done.'); + $lockid++; - //Run registration updated cron + // Run registration updated cron mtrace(get_string('siteupdatesstart', 'hub')); - cron_trace_time_and_memory(); - require_once($CFG->dirroot . '/' . $CFG->admin . '/registration/lib.php'); - $registrationmanager = new registration_manager(); - $registrationmanager->cron(); + try { + cron_trace_time_and_memory(); + $DB->get_cron_lock($lockid); + require_once($CFG->dirroot . '/' . $CFG->admin . '/registration/lib.php'); + $registrationmanager = new registration_manager(); + $registrationmanager->cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for registration cron, another copy of this is currently running"); + } mtrace(get_string('siteupdatesend', 'hub')); + $lockid++; + // If enabled, fetch information about available updates and eventually notify site admins if (empty($CFG->disableupdatenotifications)) { require_once($CFG->libdir.'/pluginlib.php'); - $updateschecker = available_update_checker::instance(); - $updateschecker->cron(); + try { + $DB->get_cron_lock($lockid); + $updateschecker = available_update_checker::instance(); + $updateschecker->cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for update checker cron, another copy of this is currently running"); + } } + $lockid++; + - //cleanup old session linked tokens - //deletes the session linked tokens that are over a day old. + // cleanup old session linked tokens + // deletes the session linked tokens that are over a day old. mtrace("Deleting session linked tokens more than one day old...", ''); cron_trace_time_and_memory(); $DB->delete_records_select('external_tokens', 'lastaccess < :onedayago AND tokentype = :tokentype', @@ -424,73 +568,118 @@ function cron_run() { // all other plugins - cron_execute_plugin_type('message', 'message plugins'); - cron_execute_plugin_type('filter', 'filters'); - cron_execute_plugin_type('editor', 'editors'); - cron_execute_plugin_type('format', 'course formats'); - cron_execute_plugin_type('profilefield', 'profile fields'); - cron_execute_plugin_type('webservice', 'webservices'); - cron_execute_plugin_type('repository', 'repository plugins'); - cron_execute_plugin_type('qbehaviour', 'question behaviours'); - cron_execute_plugin_type('qformat', 'question import/export formats'); - cron_execute_plugin_type('qtype', 'question types'); - cron_execute_plugin_type('plagiarism', 'plagiarism plugins'); - cron_execute_plugin_type('theme', 'themes'); - cron_execute_plugin_type('tool', 'admin tools'); + $other_plugins = array( + 'message' => 'message plugins', + 'filter' => 'filters', + 'editor' => 'editors', + 'format' => 'course formats', + 'profilefield' => 'profile fields', + 'webservice' => 'webservices', + 'repository' => 'repository plugins', + 'qbehaviour' => 'question behaviours', + 'qformat' => 'question import/export formats', + 'qtype' => 'question types', + 'plagiarism' => 'plagiarism plugins', + 'theme' => 'themes', + 'tool' => 'admin tools' + ); + + foreach ($other_plugins as $plugintype => $description) { + try { + $DB->get_cron_lock($lockid); + cron_execute_plugin_type($plugintype, $description); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for $description cron, another copy of this is currently running"); + } + $lockid++; + } // and finally run any local cronjobs, if any if ($locals = get_plugin_list('local')) { mtrace('Processing customized cron scripts ...', ''); - // new cron functions in lib.php first - cron_execute_plugin_type('local'); - // legacy cron files are executed directly - foreach ($locals as $local => $localdir) { - if (file_exists("$localdir/cron.php")) { - include("$localdir/cron.php"); + try { + $DB->get_cron_lock($lockid); + // new cron functions in lib.php first + cron_execute_plugin_type('local'); + // legacy cron files are executed directly + foreach ($locals as $local => $localdir) { + if (file_exists("$localdir/cron.php")) { + include("$localdir/cron.php"); + } } + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for local crons, another copy of this is currently running"); } mtrace('done.'); } - - - // Run automated backups if required - these may take a long time to execute - require_once($CFG->dirroot.'/backup/util/includes/backup_includes.php'); - require_once($CFG->dirroot.'/backup/util/helper/backup_cron_helper.class.php'); - backup_cron_automated_helper::run_automated_backup(); + $lockid++; + + try { + $DB->get_cron_lock($lockid); + // Run automated backups if required - these may take a long time to execute + require_once($CFG->dirroot.'/backup/util/includes/backup_includes.php'); + require_once($CFG->dirroot.'/backup/util/helper/backup_cron_helper.class.php'); + backup_cron_automated_helper::run_automated_backup(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for backup cron, another copy of this is currently running"); + } + $lockid++; // Run stats as at the end because they are known to take very long time on large sites if (!empty($CFG->enablestats) and empty($CFG->disablestatsprocessing)) { - require_once($CFG->dirroot.'/lib/statslib.php'); - // check we're not before our runtime - $timetocheck = stats_get_base_daily() + $CFG->statsruntimestarthour*60*60 + $CFG->statsruntimestartminute*60; - - if (time() > $timetocheck) { - // process configured number of days as max (defaulting to 31) - $maxdays = empty($CFG->statsruntimedays) ? 31 : abs($CFG->statsruntimedays); - if (stats_cron_daily($maxdays)) { - if (stats_cron_weekly()) { - if (stats_cron_monthly()) { - stats_clean_old(); + try { + $DB->get_cron_lock($lockid); + require_once($CFG->dirroot.'/lib/statslib.php'); + // check we're not before our runtime + $timetocheck = stats_get_base_daily() + $CFG->statsruntimestarthour*60*60 + $CFG->statsruntimestartminute*60; + + if (time() > $timetocheck) { + // process configured number of days as max (defaulting to 31) + $maxdays = empty($CFG->statsruntimedays) ? 31 : abs($CFG->statsruntimedays); + if (stats_cron_daily($maxdays)) { + if (stats_cron_weekly()) { + if (stats_cron_monthly()) { + stats_clean_old(); + } } } + @set_time_limit(0); + } else { + mtrace('Next stats run after:'. userdate($timetocheck)); } - @set_time_limit(0); - } else { - mtrace('Next stats run after:'. userdate($timetocheck)); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for stats cron, another copy of this is currently running"); } } + $lockid++; // Run badges review cron. - mtrace("Starting badges cron..."); - require_once($CFG->dirroot . '/badges/cron.php'); - badge_cron(); + try { + mtrace("Starting badges cron..."); + require_once($CFG->dirroot . '/badges/cron.php'); + badge_cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for badges cron, another copy of this is currently running"); + } mtrace('done.'); + $lockid++; // cleanup file trash - not very important - $fs = get_file_storage(); - $fs->cron(); + try { + $DB->get_cron_lock($lockid); + $fs = get_file_storage(); + $fs->cron(); + $DB->release_cron_lock($lockid); + } catch (dml_sessionwait_exception $dse) { + mtrace("Could not get lock for file storage cron, another copy of this is currently running"); + } mtrace("Cron script completed correctly"); diff --git a/lib/dml/moodle_database.php b/lib/dml/moodle_database.php index c7753a5..6a21611 100644 --- a/lib/dml/moodle_database.php +++ b/lib/dml/moodle_database.php @@ -52,6 +52,23 @@ define('SQL_QUERY_STRUCTURE', 4); /** SQL_QUERY_AUX - Auxiliary query done by driver, setting connection config, getting table info, etc. */ define('SQL_QUERY_AUX', 5); +/** LOCK_TIMEOUT - How much time to wait for lock before continuing to next process - 1 second */ +if (!defined('LOCK_TIMEOUT')) { + define('LOCK_TIMEOUT', 1); +} + +/** SESSION_LOCK - Lock for a session object */ +define('SESSION_LOCK', 1); + +/** CRON_LOCK - Lock for a cron process */ +define('CRON_LOCK', 2); + +/** SESSION_LOCK - Lock for a module cron process */ +define('CRON_MODULES_LOCK', 3); + +/** SESSION_LOCK - Lock for a block cron process */ +define('CRON_BLOCKS_LOCK', 4); + /** * Abstract class representing moodle database interface. * @link http://docs.moodle.org/dev/DML_functions @@ -2305,20 +2322,41 @@ abstract class moodle_database { * Obtains the session lock. * @param int $rowid The id of the row with session record. * @param int $timeout The maximum allowed time to wait for the lock in seconds. - * @return void * @throws dml_exception A DML specific exception is thrown for any errors. */ - public function get_session_lock($rowid, $timeout) { + public function get_session_lock($rowid, $timeout = LOCK_TIMEOUT) { $this->used_for_db_sessions = true; + $this->get_lock($rowid, $timeout, SESSION_LOCK); + } + + /** + * Obtains the cron lock. + * @param int $rowid The id of the row with session record. + * @param int $timeout The maximum allowed time to wait for the lock in seconds. + * @param int $locktype The type of cron lock in order to avoid lock id clashes. + * @throws dml_exception A DML specific exception is thrown for any errors. + */ + public function get_cron_lock($rowid, $timeout = LOCK_TIMEOUT, $locktype = CRON_LOCK) { + $this->get_lock($rowid, $timeout, $locktype); } /** * Releases the session lock. * @param int $rowid The id of the row with session record. - * @return void * @throws dml_exception A DML specific exception is thrown for any errors. */ public function release_session_lock($rowid) { + $this->release_lock($rowid, SESSION_LOCK); + } + + /** + * Releases the cron lock. + * @param int $rowid The lockid + * @param int $locktype The type of cron lock in order to avoid lock id clashes. + * @throws dml_exception A DML specific exception is thrown for any errors. + */ + public function release_cron_lock($rowid, $locktype = CRON_LOCK) { + $this->release_lock($rowid, $locktype); } /** diff --git a/lib/dml/mssql_native_moodle_database.php b/lib/dml/mssql_native_moodle_database.php index 3191d5f..3f3888a 100644 --- a/lib/dml/mssql_native_moodle_database.php +++ b/lib/dml/mssql_native_moodle_database.php @@ -1254,20 +1254,19 @@ s only returning name of SQL substring function, it now requires all parameters. } /** - * Obtain session lock - * @param int $rowid id of the row with session record + * Obtain lock + * @param int $rowid id of the row with session record or lock id * @param int $timeout max allowed time to wait for the lock in seconds - * @return bool success + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock */ - public function get_session_lock($rowid, $timeout) { + protected function get_lock($rowid, $timeout, $locktype) { if (!$this->session_lock_supported()) { return; } - parent::get_session_lock($rowid, $timeout); $timeoutmilli = $timeout * 1000; - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; // There is one bug in PHP/freetds (both reproducible with mssql_query() // and its mssql_init()/mssql_bind()/mssql_execute() alternative) for // stored procedures, causing scalar results of the execution @@ -1296,17 +1295,22 @@ s only returning name of SQL substring function, it now requires all parameters. $this->free_result($result); } - public function release_session_lock($rowid) { + /** + * Release lock + * @param int $rowid id of the row with session record or lock id + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock + */ + protected function release_lock($rowid, $locktype) { if (!$this->session_lock_supported()) { return; } - if (!$this->used_for_db_sessions) { + if (!$this->used_for_db_sessions && $locktype == SESSION_LOCK) { return; } parent::release_session_lock($rowid); - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; $sql = "sp_releaseapplock '$fullname', 'Session'"; $this->query_start($sql, null, SQL_QUERY_AUX); $result = mssql_query($sql, $this->mssql); diff --git a/lib/dml/mysqli_native_moodle_database.php b/lib/dml/mysqli_native_moodle_database.php index e819824..225d046 100644 --- a/lib/dml/mysqli_native_moodle_database.php +++ b/lib/dml/mysqli_native_moodle_database.php @@ -1406,15 +1406,13 @@ class mysqli_native_moodle_database extends moodle_database { } /** - * Obtain session lock - * @param int $rowid id of the row with session record + * Obtain lock + * @param int $rowid id of the row with session record or lock id * @param int $timeout max allowed time to wait for the lock in seconds - * @return void + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock */ - public function get_session_lock($rowid, $timeout) { - parent::get_session_lock($rowid, $timeout); - - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + protected function get_lock($rowid, $timeout, $locktype) { + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; $sql = "SELECT GET_LOCK('$fullname', $timeout)"; $this->query_start($sql, null, SQL_QUERY_AUX); $result = $this->mysqli->query($sql); @@ -1432,13 +1430,18 @@ class mysqli_native_moodle_database extends moodle_database { } } - public function release_session_lock($rowid) { - if (!$this->used_for_db_sessions) { + /** + * Release lock + * @param int $rowid id of the row with session record or lock id + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock + */ + protected function release_lock($rowid, $locktype) { + if (!$this->used_for_db_sessions && $locktype == SESSION_LOCK) { return; } parent::release_session_lock($rowid); - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; $sql = "SELECT RELEASE_LOCK('$fullname')"; $this->query_start($sql, null, SQL_QUERY_AUX); $result = $this->mysqli->query($sql); diff --git a/lib/dml/oci_native_moodle_database.php b/lib/dml/oci_native_moodle_database.php index f5cce9d..6f58218 100644 --- a/lib/dml/oci_native_moodle_database.php +++ b/lib/dml/oci_native_moodle_database.php @@ -1648,19 +1648,19 @@ class oci_native_moodle_database extends moodle_database { } /** - * Obtain session lock - * @param int $rowid id of the row with session record + * Obtain lock + * @param int $rowid id of the row with session record or lock id * @param int $timeout max allowed time to wait for the lock in seconds - * @return void + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock */ - public function get_session_lock($rowid, $timeout) { + protected function get_lock($rowid, $timeout, $locktype) { if (!$this->oci_package_installed()) { return; } parent::get_session_lock($rowid, $timeout); - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; - $sql = 'SELECT MOODLELIB.GET_LOCK(:lockname, :locktimeout) FROM DUAL'; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; + $sql = 'SELECT MOODLE_LOCKS.GET_LOCK(:lockname, :locktimeout) FROM DUAL'; $params = array('lockname' => $fullname , 'locktimeout' => $timeout); $this->query_start($sql, $params, SQL_QUERY_AUX); $stmt = $this->parse_query($sql); @@ -1673,7 +1673,12 @@ class oci_native_moodle_database extends moodle_database { oci_free_statement($stmt); } - public function release_session_lock($rowid) { + /** + * Release lock + * @param int $rowid id of the row with session record or lock id + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock + */ + protected function release_lock($rowid, $locktype) { if (!$this->oci_package_installed()) { return; } @@ -1683,7 +1688,7 @@ class oci_native_moodle_database extends moodle_database { parent::release_session_lock($rowid); - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; $params = array('lockname' => $fullname); $sql = 'SELECT MOODLELIB.RELEASE_LOCK(:lockname) FROM DUAL'; $this->query_start($sql, $params, SQL_QUERY_AUX); diff --git a/lib/dml/pgsql_native_moodle_database.php b/lib/dml/pgsql_native_moodle_database.php index 5783972..236a761 100644 --- a/lib/dml/pgsql_native_moodle_database.php +++ b/lib/dml/pgsql_native_moodle_database.php @@ -1230,12 +1230,12 @@ class pgsql_native_moodle_database extends moodle_database { } /** - * Obtain session lock - * @param int $rowid id of the row with session record + * Obtain lock + * @param int $rowid id of the row with session record or lock id * @param int $timeout max allowed time to wait for the lock in seconds - * @return bool success + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock */ - public function get_session_lock($rowid, $timeout) { + protected function get_lock($rowid, $timeout, $locktype) { // NOTE: there is a potential locking problem for database running // multiple instances of moodle, we could try to use pg_advisory_lock(int, int), // luckily there is not a big chance that they would collide @@ -1243,8 +1243,6 @@ class pgsql_native_moodle_database extends moodle_database { return; } - parent::get_session_lock($rowid, $timeout); - $timeoutmilli = $timeout * 1000; $sql = "SET statement_timeout TO $timeoutmilli"; @@ -1256,7 +1254,12 @@ class pgsql_native_moodle_database extends moodle_database { pg_free_result($result); } - $sql = "SELECT pg_advisory_lock($rowid)"; + if ($locktype == SESSION_LOCK) { + $sql = "SELECT pg_advisory_lock($rowid)"; + } else { + $sql = "SELECT pg_advisory_lock($locktype, $rowid)"; + } + $this->query_start($sql, null, SQL_QUERY_AUX); $start = time(); $result = pg_query($this->pgsql, $sql); @@ -1285,17 +1288,27 @@ class pgsql_native_moodle_database extends moodle_database { } } - public function release_session_lock($rowid) { + /** + * Release lock + * @param int $rowid id of the row with session record or lock id + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock + */ + protected function release_lock($rowid, $locktype) { if (!$this->session_lock_supported()) { return; } - if (!$this->used_for_db_sessions) { + if (!$this->used_for_db_sessions && $locktype == SESSION_LOCK) { return; } parent::release_session_lock($rowid); - $sql = "SELECT pg_advisory_unlock($rowid)"; + if ($locktype == SESSION_LOCK) { + $sql = "SELECT pg_advisory_unlock($rowid)"; + } else { + $sql = "SELECT pg_advisory_unlock($locktype, $rowid)"; + } + $this->query_start($sql, null, SQL_QUERY_AUX); $result = pg_query($this->pgsql, $sql); $this->query_end($result); diff --git a/lib/dml/sqlsrv_native_moodle_database.php b/lib/dml/sqlsrv_native_moodle_database.php index 09fa92e..0ac0403 100644 --- a/lib/dml/sqlsrv_native_moodle_database.php +++ b/lib/dml/sqlsrv_native_moodle_database.php @@ -1318,20 +1318,18 @@ class sqlsrv_native_moodle_database extends moodle_database { } /** - * Obtain session lock - * @param int $rowid id of the row with session record + * Obtain lock + * @param int $rowid id of the row with session record or lock id * @param int $timeout max allowed time to wait for the lock in seconds - * @return void + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock */ - public function get_session_lock($rowid, $timeout) { + protected function get_lock($rowid, $timeout, $locktype) { if (!$this->session_lock_supported()) { return; } - parent::get_session_lock($rowid, $timeout); $timeoutmilli = $timeout * 1000; - - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; // While this may work using proper {call sp_...} calls + binding + // executing + consuming recordsets, the solution used for the mssql // driver is working perfectly, so 100% mimic-ing that code. @@ -1358,17 +1356,22 @@ class sqlsrv_native_moodle_database extends moodle_database { $this->free_result($result); } - public function release_session_lock($rowid) { + /** + * Release lock + * @param int $rowid id of the row with session record or lock id + * @param int $locktype whether it is a session lock or cron lock, if cron lock what type of cron lock + */ + protected function release_lock($rowid, $locktype) { if (!$this->session_lock_supported()) { return; } - if (!$this->used_for_db_sessions) { + if (!$this->used_for_db_sessions && $locktype == SESSION_LOCK) { return; } parent::release_session_lock($rowid); - $fullname = $this->dbname.'-'.$this->prefix.'-session-'.$rowid; + $fullname = $this->dbname.'-'.$this->prefix.'-'.$locktype."-".$rowid; $sql = "sp_releaseapplock '$fullname', 'Session'"; $this->query_start($sql, null, SQL_QUERY_AUX); $result = sqlsrv_query($this->sqlsrv, $sql); -- 1.7.11.4