diff --git a/lib/statslib.php b/lib/statslib.php index 7c6c53d..f77f475 100644 --- a/lib/statslib.php +++ b/lib/statslib.php @@ -75,123 +75,188 @@ function stats_cron_daily () { $return = STATS_RUN_COMPLETE; // optimistic - static $daily_modules; - - if (empty($daily_modules)) { - $daily_modules = array(); - $mods = get_records("modules"); - foreach ($mods as $mod) { - $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; - if (!is_readable($file)) { - continue; - } - require_once($file); - $fname = $mod->name.'_get_daily_stats'; - if (function_exists($fname)) { - $daily_modules[$mod] = $fname; - } - } - } + /* Disabled - see note further down + * about module's stat processing + * + * static $daily_modules; + * if (empty($daily_modules)) { + * $daily_modules = array(); + * $mods = get_records("modules"); + * foreach ($mods as $mod) { + * $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; + * if (!is_readable($file)) { + * continue; + * } + * require_once($file); + * $fname = $mod->name.'_get_daily_stats'; + * if (function_exists($fname)) { + * $daily_modules[$mod] = $fname; + * } + * } + *} + */ $nextmidnight = stats_get_next_dayend($timestart); - - if (!$courses = get_records('course','','','','id,1')) { - return STATS_RUN_ABORTED; - } $days = 0; + + // + // The SITEID case - count non-deleted users vs active users + // + $defaultfprole = $CFG->defaultcourseroleid; + if (!empty($CFG->defaultfrontpageroleid)) { // 1.9 only, so far + $defaultfprole = $CFG->defaultfrontpageroleid; + } + + $primary_roles = sql_primary_role_subselect(); // In dmllib.php + mtrace("starting at $timestart"); while ($midnight > $nextmidnight && $timestart < $nextmidnight) { + mtrace("processing for day between $timestart and $nextmidnight"); $timesql = " (l.time > $timestart AND l.time < $nextmidnight) "; begin_sql(); - foreach ($courses as $course) { - //do this first. - if ($course->id == SITEID) { - $stat = new StdClass; - $stat->courseid = $course->id; - $stat->timeend = $nextmidnight; - $stat->roleid = 0; // all users - $stat->stattype = 'logins'; - $sql = 'SELECT count(l.id) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '.$timesql; - $stat->stat1 = count_records_sql($sql); - $sql = 'SELECT COUNT(DISTINCT(l.userid)) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '.$timesql; - $stat->stat2 = count_records_sql($sql); - insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it. - - // and now user logins... - $sql = 'SELECT l.userid,count(l.id) as count FROM '.$CFG->prefix.'log l WHERE action = \'login\' AND '.$timesql.' GROUP BY userid'; - - if ($logins = get_records_sql($sql)) { - foreach ($logins as $l) { - $stat->statsreads = $l->count; - $stat->userid = $l->userid; - $stat->timeend = $nextmidnight; - $stat->courseid = SITEID; - $stat->statswrites = 0; - $stat->stattype = 'logins'; - $stat->roleid = 0; - insert_record('stats_user_daily',$stat,false); - } - } - } - - $context = get_context_instance(CONTEXT_COURSE, $course->id); - if (!$roles = get_roles_on_exact_context($context)) { - // no roles.. nothing to log. - continue; - } - - $primary_roles = sql_primary_role_subselect(); // In dmllib.php - foreach ($roles as $role) { - // ENROLMENT FIRST.... - // ALL users with this role... - $stat = new StdClass; - $stat->courseid = $course->id; - $stat->roleid = $role->id; - $stat->timeend = $nextmidnight; - $stat->stattype = 'enrolments'; - $sql = 'SELECT COUNT(DISTINCT prs.userid) FROM ('.$primary_roles.') prs WHERE prs.primary_roleid='.$role->id. - ' AND prs.courseid='.$course->id.' AND prs.contextlevel = '.CONTEXT_COURSE; - $stat->stat1 = count_records_sql($sql); - - $sql = 'SELECT COUNT(DISTINCT prs.userid) FROM ('.$primary_roles.') prs - INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid AND l.course=prs.courseid) - WHERE prs.primary_roleid='.$role->id.' AND prs.courseid='.$course->id.' - AND prs.contextlevel = '.CONTEXT_COURSE.' AND '.$timesql; - $stat->stat2 = count_records_sql($sql); - insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it. + // Sitecourse logins for the day + $sql = "INSERT INTO {$CFG->prefix}stats_daily (courseid,timeend,roleid,stattype,stat1,stat2) + SELECT " . SITEID . ", $nextmidnight, 0, 'logins', + COUNT(l.userid), COUNT(DISTINCT(l.userid)) + FROM {$CFG->prefix}log l + WHERE l.action='login' AND $timesql"; + execute_sql($sql, false); + $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (userid,courseid,timeend,roleid,stattype,statsreads,statswrites) + SELECT l.userid, " . SITEID . ", $nextmidnight, 0, 'logins', COUNT(l.id), 0 + FROM {$CFG->prefix}log l + WHERE l.action='login' AND $timesql + GROUP BY l.userid"; + execute_sql($sql, false); + + // Enrolments vs active users + // - stat1: enrolled users + // - stat2: active users + // - This is not "exact" because it follows a pre-roles metaphor. + // When we look at the activity here, we only look at activity + // from formally enrolled users (because of the LEFT OUTER JOIN). + // This means that default-enrolment access isn't counted. + // We could do a purely outer join here, I suspect, but it's not + // easily portable (IIRC) and the participation numbers will look funny. + // - SITEID is specialcased here, because it's all about default enrolment + // in that case, we'll count non-deleted users. + // + $sql = "INSERT INTO {$CFG->prefix}stats_daily + (courseid, roleid, timeend, stattype, stat1, stat2) + SELECT prs.courseid, prs.primary_roleid, + $nextmidnight, 'enrolments', + COUNT(DISTINCT prs.userid), COUNT(DISTINCT l.userid) + FROM ($primary_roles) prs + LEFT OUTER JOIN {$CFG->prefix}log l + ON (prs.courseid=l.course AND prs.userid=l.userid) + WHERE prs.courseid != ".SITEID." AND (l.id IS NULL OR ($timesql)) + GROUP BY prs.courseid, prs.primary_roleid"; + execute_sql($sql, false); + + $sql = "INSERT INTO {$CFG->prefix}stats_daily + (courseid, roleid, timeend, stattype, stat1, stat2) + SELECT ctx.instanceid, COALESCE(ra.roleid, {$defaultfprole}), + $nextmidnight, 'enrolments', + (SELECT COUNT(id) FROM {$CFG->prefix}user where deleted=0), + COUNT(DISTINCT l.userid) + FROM {$CFG->prefix}user u + JOIN {$CFG->prefix}context ctx + ON (ctx.instanceid = ".SITEID." AND ctx.contextlevel=".CONTEXT_COURSE.") + LEFT OUTER JOIN {$CFG->prefix}role_assignments ra + ON (ctx.id=ra.contextid AND u.id=ra.userid) + LEFT OUTER JOIN {$CFG->prefix}log l + ON (l.course=".SITEID." AND u.id=l.userid) + WHERE u.deleted=0 AND (l.id IS NULL OR ($timesql)) + GROUP BY ctx.instanceid,ra.roleid"; + execute_sql($sql, false); + + // These are strings we can put in an IN clauses + // where needed... + $viewactions = implode(',', stats_get_action_names('view')); + $postactions = implode(',', stats_get_action_names('post')); + + + // Activities + // + // - stat1 stores 'view' actions + // - stat2 stores 'post' actions + // - pl stands for 'parsed log' or processed log + // - avoid "view" - may be a reserved word + // + // Very heavy query - on large sites, we have seen it + // take around 30s on Pg 80s and 4 minutes on MySQL + // + $sql = "INSERT INTO {$CFG->prefix}stats_daily + (timeend, stattype, courseid, roleid, stat1, stat2) + SELECT $nextmidnight, 'activity', + pl.course, pl.roleid, + SUM(pl.viewaction), SUM(pl.postaction) + FROM ( + SELECT l.course, COALESCE(prs.primary_roleid, {$CFG->defaultcourseroleid}) AS roleid, + CASE + WHEN l.action IN ($viewactions) THEN 1 + ELSE 0 + END AS viewaction, + CASE + WHEN l.action IN ($postactions) THEN 1 + ELSE 0 + END AS postaction + FROM {$CFG->prefix}log l + LEFT OUTER JOIN ($primary_roles) prs + ON (l.userid=prs.userid AND l.course=prs.courseid) + WHERE $timesql + ) as pl + GROUP BY pl.course,pl.roleid"; + execute_sql($sql, false); + + // + // Daily user cron + // driven by the log table - we'll only insert stats where + // the user has done something in the course. Outer join + // with PRS to get rolename + // Similar to the activity cron - + // + $sql = "INSERT INTO {$CFG->prefix}stats_user_daily + (timeend, stattype, userid, courseid, roleid, + statsreads, statswrites) + SELECT $nextmidnight, 'activity', + pl.userid, pl.course, pl.roleid, + SUM(pl.viewaction), SUM(pl.postaction) + FROM ( + SELECT l.course, l.userid, + CASE + WHEN l.course = ".SITEID." + THEN COALESCE(prs.primary_roleid, {$defaultfprole}) + ELSE + COALESCE(prs.primary_roleid, {$CFG->defaultcourseroleid}) + END AS roleid, + CASE + WHEN l.action IN ($viewactions) THEN 1 + ELSE 0 + END AS viewaction, + CASE + WHEN l.action IN ($postactions) THEN 1 + ELSE 0 + END AS postaction + FROM {$CFG->prefix}log l + LEFT OUTER JOIN ($primary_roles) prs + ON (l.userid=prs.userid AND l.course=prs.courseid) + WHERE $timesql + ) as pl + GROUP BY pl.userid, pl.course,pl.roleid "; + execute_sql($sql, false); + + /* Per-module stats are disabled -- if we want to support them + * we will need a new calling convention -- roughly as follows. + * foreach ($mods as $mod => $fname) { + * mtrace(' doing daily statistics for '.$mod->name); + * $fname($timeend); + * } + * + */ - // ACTIVITY - - $stat = new StdClass; - $stat->courseid = $course->id; - $stat->roleid = $role->id; - $stat->timeend = $nextmidnight; - $stat->stattype = 'activity'; - - $sql = 'SELECT COUNT(DISTINCT l.id) FROM ('.$primary_roles.') prs - INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid - AND l.course=prs.courseid) WHERE prs.primary_roleid='.$role->id.' - AND prs.courseid='.$course->id.' AND prs.contextlevel = '.CONTEXT_COURSE.' - AND '.$timesql.' '.stats_get_action_sql_in('view'); - $stat->stat1 = count_records_sql($sql); - - $sql = 'SELECT COUNT(DISTINCT l.id) FROM ('.$primary_roles.') prs - INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid AND l.course=prs.courseid) - WHERE prs.primary_roleid='.$role->id.' AND prs.courseid='.$course->id.' - AND prs.contextlevel = '.CONTEXT_COURSE.' AND '.$timesql.' '.stats_get_action_sql_in('post'); - $stat->stat2 = count_records_sql($sql); - - insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it. - } - - $users = stats_get_course_users($course,$timesql); - foreach ($users as $user) { - stats_do_daily_user_cron($course,$user,$user->primaryrole,$timesql,$nextmidnight,$daily_modules); - } - } commit_sql(); $timestart = $nextmidnight; $nextmidnight = stats_get_next_dayend($nextmidnight); @@ -235,22 +300,27 @@ function stats_cron_weekly () { $return = STATS_RUN_COMPLETE; // optimistic static $weekly_modules; - - if (empty($weekly_modules)) { - $weekly_modules = array(); - $mods = get_records("modules"); - foreach ($mods as $mod) { - $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; - if (!is_readable($file)) { - continue; - } - require_once($file); - $fname = $mod->name.'_get_weekly_stats'; - if (function_exists($fname)) { - $weekly_modules[$mod] = $fname; - } - } - } + + /* Disabled - see note in stats_cron_daily() + * If we want to re-enable it, the calling convention + * must change to be called once per day/week/month process + * loop only. The goal is fixed-number of SQL queries. + * if (empty($weekly_modules)) { + * $weekly_modules = array(); + * $mods = get_records("modules"); + * foreach ($mods as $mod) { + * $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; + * if (!is_readable($file)) { + * continue; + * } + * require_once($file); + * $fname = $mod->name.'_get_weekly_stats'; + * if (function_exists($fname)) { + * $weekly_modules[$mod] = $fname; + * } + * } + *} + */ $nextsunday = stats_get_next_weekend($timestart); @@ -367,23 +437,26 @@ function stats_cron_monthly () { $return = STATS_RUN_COMPLETE; // optimistic - static $monthly_modules; - - if (empty($monthly_modules)) { - $monthly_modules = array(); - $mods = get_records("modules"); - foreach ($mods as $mod) { - $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; - if (!is_readable($file)) { - continue; - } - require_once($file); - $fname = $mod->name.'_get_monthly_stats'; - if (function_exists($fname)) { - $monthly_modules[$mod] = $fname; - } - } - } + /* Disabled - see notes in stats_cron_daily() + * and stats_cron_weekly. + * static $monthly_modules; + * + *if (empty($monthly_modules)) { + * $monthly_modules = array(); + * $mods = get_records("modules"); + * foreach ($mods as $mod) { + * $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php'; + * if (!is_readable($file)) { + * continue; + * } + * require_once($file); + * $fname = $mod->name.'_get_monthly_stats'; + * if (function_exists($fname)) { + * $monthly_modules[$mod] = $fname; + * } + * } + *} + */ $nextmonthend = stats_get_next_monthend($timestart); @@ -456,7 +529,7 @@ function stats_cron_monthly () { $users = stats_get_course_users($course,$timesql); foreach ($users as $user) { - stats_do_aggregate_user_cron($course,$user,$user->primaryrole,$timesql,$nextmonthend,'monthly',$monthly_modules); + stats_do_aggregate_user_cron($course,$user,$user->primaryrole,$timesql,$nextmonthend,'monthly'); } } @@ -772,7 +845,7 @@ function stats_get_post_actions() { return array('add','delete','edit','add mod','delete mod','edit section'.'enrol','loginas','new','unenrol','update','update mod'); } -function stats_get_action_sql_in($str) { +function stats_get_action_names($str) { global $CFG; $mods = get_records('modules'); @@ -789,17 +862,17 @@ function stats_get_action_sql_in($str) { $actions = array_merge($actions,$function()); } } - $actions = array_unique($actions); - if (empty($actions)) { - return ' '; - } else if (count($actions) == 1) { - return ' AND l.action = '.array_pop($actions).' '; - } else { - return ' AND l.action IN (\''.implode('\',\'',$actions).'\') '; + + // The array_values() forces a stack-like array + // so we can later loop over safely... + $actions = array_values(array_unique($actions)); + $c = count($actions); + for ($n=0;$n<$c;$n++) { + $actions[$n] = "'" . $actions[$n] . "'"; // quote them for SQL } + return $actions; } - function stats_get_course_users($course,$timesql) { global $CFG; @@ -828,39 +901,7 @@ function stats_get_course_users($course,$timesql) { } -function stats_do_daily_user_cron($course,$user,$roleid,$timesql,$timeend,$mods) { - - global $CFG; - - $stat = new StdClass; - $stat->userid = $user->userid; - $stat->roleid = $roleid; - $stat->courseid = $course->id; - $stat->stattype = 'activity'; - $stat->timeend = $timeend; - - $sql = 'SELECT COUNT(l.id) FROM '.$CFG->prefix.'log l WHERE l.userid = '.$user->userid - .' AND l.course = '.$course->id - .' AND '.$timesql .' '.stats_get_action_sql_in('view'); - - $stat->statsreads = count_records_sql($sql); - - $sql = 'SELECT COUNT(l.id) FROM '.$CFG->prefix.'log l WHERE l.userid = '.$user->userid - .' AND l.course = '.$course->id - .' AND '.$timesql.' '.stats_get_action_sql_in('post'); - - $stat->statswrites = count_records_sql($sql); - - insert_record('stats_user_daily',$stat,false); - - // now ask the modules if they want anything. - foreach ($mods as $mod => $fname) { - mtrace(' doing daily statistics for '.$mod->name); - $fname($course,$user,$timeend,$roleid); - } -} - -function stats_do_aggregate_user_cron($course,$user,$roleid,$timesql,$timeend,$timestr,$mods) { +function stats_do_aggregate_user_cron($course,$user,$roleid,$timesql,$timeend,$timestr) { global $CFG; @@ -880,11 +921,12 @@ function stats_do_aggregate_user_cron($course,$user,$roleid,$timesql,$timeend,$t insert_record('stats_user_'.$timestr,$stat,false); + // Disabled - see stats_cron_daily() for details // now ask the modules if they want anything. - foreach ($mods as $mod => $fname) { - mtrace(' doing '.$timestr.' statistics for '.$mod->name); - $fname($course,$user,$timeend,$roleid); - } + //foreach ($mods as $mod => $fname) { + // mtrace(' doing '.$timestr.' statistics for '.$mod->name); + // $fname($course,$user,$timeend,$roleid); + //} } function stats_do_aggregate_user_login_cron($timesql,$timeend,$timestr) {