### Eclipse Workspace Patch 1.0
#P moodle19
Index: lib/adminlib.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/adminlib.php,v
retrieving revision 1.153.2.41
diff -u -r1.153.2.41 adminlib.php
--- lib/adminlib.php	5 Feb 2008 11:42:51 -0000	1.153.2.41
+++ lib/adminlib.php	15 Feb 2008 22:51:15 -0000
@@ -455,45 +455,41 @@
 }
 
 /**
- * This function will return FALSE if the lock fails to be set (ie, if it's already locked)
+ * Try to obtain or release the cron lock.
  *
- * @param string  $name ?
- * @param bool  $value ?
- * @param int  $staleafter ?
- * @param bool  $clobberstale ?
- * @todo Finish documenting this function
+ * @param string  $name  name of lock
+ * @param int  $until timestamp when this lock considered stale, null means remove lock unconditionaly, 1 means locked always
+ * @param bool $ignorecurrent ignore current lock state, usually entend previous lock
+ * @return bool true if lock obtained
  */
-function set_cron_lock($name,$value=true,$staleafter=7200,$clobberstale=false) {
-
+function set_cron_lock($name, $until=1, $ignorecurrent=false) {
     if (empty($name)) {
-        mtrace("Tried to get a cron lock for a null fieldname");
+        debugging("Tried to get a cron lock for a null fieldname");
         return false;
     }
 
-    if (empty($value)) {
-        set_config($name,0);
+    // remove lock by force == remove from config table
+    if (is_null($until)) {
+        set_config($name, null);
         return true;
     }
 
-    if ($config = get_record('config','name',$name)) {
-        if (empty($config->value)) {
-            set_config($name,time());
-        } else {
-            // check for stale.
-            if ((time() - $staleafter) > $config->value) {
-                mtrace("STALE LOCKFILE FOR $name - was $config->value");
-                if (!empty($clobberstale)) {
-                    set_config($name,time());
-                    return true;
-                }
-            } else {
-                return false; // was not stale - ie, we're ok to still be running.
-            }
+    if (!$ignorecurrent) {
+        // read value from db - other processes might have changed it
+        $value = get_field('config', 'value', 'name', $name);
+
+        // remove stale lock
+        if ($value != 1 and $value < time()) {
+            $value = false;
+        }
+
+        if ($value) {
+            // lock already held by somebody else
+            return false;
         }
     }
-    else {
-        set_config($name,time());
-    }
+
+    set_config($name, $until);
     return true;
 }
 
Index: lib/statslib.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/statslib.php,v
retrieving revision 1.54
diff -u -r1.54 statslib.php
--- lib/statslib.php	11 Apr 2007 23:57:43 -0000	1.54
+++ lib/statslib.php	15 Feb 2008 22:51:25 -0000
@@ -3,11 +3,11 @@
     // THESE CONSTANTS ARE USED FOR THE REPORTING PAGE.
 
     define('STATS_REPORT_LOGINS',1); // double impose logins and unqiue logins on a line graph. site course only.
-    define('STATS_REPORT_READS',2); // double impose student reads and teacher reads on a line graph. 
+    define('STATS_REPORT_READS',2); // double impose student reads and teacher reads on a line graph.
     define('STATS_REPORT_WRITES',3); // double impose student writes and teacher writes on a line graph.
     define('STATS_REPORT_ACTIVITY',4); // 2+3 added up, teacher vs student.
     define('STATS_REPORT_ACTIVITYBYROLE',5); // all activity, reads vs writes, seleted by role.
-    
+
     // user level stats reports.
     define('STATS_REPORT_USER_ACTIVITY',7);
     define('STATS_REPORT_USER_ALLACTIVITY',8);
@@ -47,516 +47,917 @@
     define('STATS_MODE_DETAILED',2);
     define('STATS_MODE_RANKED',3); // admins only - ranks courses
 
-    // return codes - whether to rerun
-    define('STATS_RUN_COMPLETE',1);
-    define('STATS_RUN_ABORTED',0);
-
-function stats_cron_daily () {
+/**
+ * Execute daily statistics gathering
+ * @param int $maxdays maximum number of days to be processed
+ * @return boolean success
+ */
+function stats_cron_daily($maxdays=1) {
     global $CFG;
-    
-    if (empty($CFG->enablestats)) {
-        return STATS_RUN_ABORTED;
+
+    $now = time();
+
+    // read last execution date from db
+    if (!$timestart = get_config(NULL, 'statslastdaily')) {
+        $timestart = stats_get_base_daily(stats_get_start_from('daily'));
+        set_config('statslastdaily', $timestart);
     }
 
-    if (!$timestart = stats_get_start_from('daily')) {
-        return STATS_RUN_ABORTED;
+    $nextmidnight = stats_get_next_day_start($timestart);
+
+    // are there any days that need to be processed?
+    if ($now < $nextmidnight) {
+        return true; // everything ok and up-to-date
     }
 
+    $timeout = empty($CFG->statsmaxruntime) ? 60*60*24 : $CFG->statsmaxruntime;
 
-    $midnight = stats_getmidnight(time());
-    
-    // check to make sure we're due to run, at least one day after last run
-    if (isset($CFG->statslastdaily) and ((time() - 24*60*60) < $CFG->statslastdaily)) {
-        return STATS_RUN_ABORTED;
+    if (!set_cron_lock('statsrunning', $now + $timeout)) {
+        return false;
     }
 
-    mtrace("Running daily statistics gathering...");
-    set_config('statslastdaily',time());
+    // fisrt delete entries that should not be there yet
+    delete_records_select('stats_daily',      "timeend > $timestart");
+    delete_records_select('stats_user_daily', "timeend > $timestart");
 
-    $return = STATS_RUN_COMPLETE; // optimistic
+    // Read in a few things we'll use later
+    $viewactions = implode(',', stats_get_action_names('view'));
+    $postactions = implode(',', stats_get_action_names('post'));
 
-    static $daily_modules;
-    
-    if (empty($daily_modules)) {
-        $daily_modules = array();
-        $mods = get_records("modules");
-        foreach ($mods as $mod) {
-            $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php';
-            if (!is_readable($file)) {
-                continue;
-            }
-            require_once($file);
-            $fname = $mod->name.'_get_daily_stats';
-            if (function_exists($fname)) {
-                $daily_modules[$mod] = $fname;
-            }
-        }
-    }
+    $guest     = get_guest();
+    $guestrole = get_guest_role();
 
-    $nextmidnight = stats_get_next_dayend($timestart);
+    list($enrolfrom, $enrolwhere) = stats_get_enrolled_sql($CFG->statscatdepth);
+    list($fpfrom, $fpwhere)       = stats_get_enrolled_sql(0);
+
+    mtrace("Running daily statistics gathering, starting at $timestart...");
 
-    if (!$courses = get_records('course','','','','id,1')) {
-        return STATS_RUN_ABORTED;
-    }
-    
     $days = 0;
-    mtrace("starting at $timestart");
-    while ($midnight > $nextmidnight && $timestart < $nextmidnight) {
+    $failed = false; // failed stats flag
 
-        $timesql = " (l.time > $timestart AND l.time < $nextmidnight) ";
-        begin_sql();
-        foreach ($courses as $course) {
-            //do this first.
-            if ($course->id == SITEID) {
-                $stat = new StdClass;
-                $stat->courseid = $course->id;
-                $stat->timeend = $nextmidnight;
-                $stat->roleid = 0; // all users
-                $stat->stattype = 'logins';
-                $sql = 'SELECT count(l.id) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '.$timesql;
-                $stat->stat1 = count_records_sql($sql);
-                $sql = 'SELECT COUNT(DISTINCT(l.userid)) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '.$timesql;
-                $stat->stat2 = count_records_sql($sql);
-                insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it.
-
-                // and now user logins...
-                $sql = 'SELECT l.userid,count(l.id) as count FROM '.$CFG->prefix.'log l WHERE action = \'login\' AND '.$timesql.' GROUP BY userid';
-                
-                if ($logins = get_records_sql($sql)) {
-                    foreach ($logins as $l) {
-                        $stat->statsreads = $l->count;
-                        $stat->userid = $l->userid;
-                        $stat->timeend = $nextmidnight;
-                        $stat->courseid = SITEID;
-                        $stat->statswrites = 0;
-                        $stat->stattype = 'logins';
-                        $stat->roleid = 0;
-                        insert_record('stats_user_daily',$stat,false);
-                    }
-                }
-            }
+    while ($now > $nextmidnight) {
+        if ($days >= $maxdays) {
+            mtrace("...stopping early, reached maximum number of $maxdays days - will continue next time.");
+            set_cron_lock('statsrunning', null);
+            return false;
+        }
 
-            $context = get_context_instance(CONTEXT_COURSE, $course->id); 
-            if (!$roles = get_roles_on_exact_context($context)) {
-                // no roles.. nothing to log.
-                continue;
-            }
-            
-            $primary_roles = sql_primary_role_subselect();  // In dmllib.php
-            foreach ($roles as $role) {
-                // ENROLMENT FIRST....
-                // ALL users with this role...
-                $stat = new StdClass;
-                $stat->courseid = $course->id;
-                $stat->roleid = $role->id;
-                $stat->timeend = $nextmidnight;
-                $stat->stattype = 'enrolments';
-                $sql = 'SELECT COUNT(DISTINCT prs.userid) FROM ('.$primary_roles.') prs WHERE prs.primary_roleid='.$role->id.
-                    ' AND prs.courseid='.$course->id.' AND prs.contextlevel = '.CONTEXT_COURSE;
-                $stat->stat1 = count_records_sql($sql);               
-                
-                $sql = 'SELECT COUNT(DISTINCT prs.userid) FROM ('.$primary_roles.') prs 
-                        INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid AND l.course=prs.courseid) 
-                        WHERE prs.primary_roleid='.$role->id.' AND prs.courseid='.$course->id.' 
-                        AND prs.contextlevel = '.CONTEXT_COURSE.' AND '.$timesql;
-
-                $stat->stat2 = count_records_sql($sql);               
-                insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it.
-
-                // ACTIVITY
-                
-                $stat = new StdClass;
-                $stat->courseid = $course->id;
-                $stat->roleid = $role->id;
-                $stat->timeend = $nextmidnight;
-                $stat->stattype = 'activity';
-                
-                $sql = 'SELECT COUNT(DISTINCT l.id) FROM ('.$primary_roles.') prs 
-                        INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid
-                        AND l.course=prs.courseid) WHERE prs.primary_roleid='.$role->id.' 
-                        AND prs.courseid='.$course->id.' AND prs.contextlevel = '.CONTEXT_COURSE.'
-                         AND '.$timesql.' '.stats_get_action_sql_in('view');
-                $stat->stat1 = count_records_sql($sql);       
-
-                $sql = 'SELECT COUNT(DISTINCT l.id) FROM ('.$primary_roles.') prs 
-                        INNER JOIN '.$CFG->prefix.'log l ON (prs.userid=l.userid  AND l.course=prs.courseid) 
-                        WHERE prs.primary_roleid='.$role->id.' AND prs.courseid='.$course->id.' 
-                        AND prs.contextlevel = '.CONTEXT_COURSE.' AND '.$timesql.' '.stats_get_action_sql_in('post');
-                $stat->stat2 = count_records_sql($sql);       
+        $days++;
+        @set_time_limit($timeout - 200);
 
-                insert_record('stats_daily',$stat,false); // don't worry about the return id, we don't need it.
-            }
-            
-            $users = stats_get_course_users($course,$timesql);
-            foreach ($users as $user) {
-                stats_do_daily_user_cron($course,$user,$user->primaryrole,$timesql,$nextmidnight,$daily_modules);
-            }
+        if ($days > 1) {
+            // move the lock
+            set_cron_lock('statsrunning', time() + $timeout, true);
         }
-        commit_sql();
-        $timestart = $nextmidnight;
-        $nextmidnight = stats_get_next_dayend($nextmidnight);
-        $days++;
 
-        if (!stats_check_runtime()) {
-            mtrace("Stopping early! reached maxruntime");
-            $return = STATS_RUN_ABORTED;
+        $timesql  = "l.time >= $timestart  AND l.time  < $nextmidnight";
+        $timesql1 = "l1.time >= $timestart AND l1.time < $nextmidnight";
+        $timesql2 = "l2.time >= $timestart AND l2.time < $nextmidnight";
+
+        $start = time();
+        $timers = array();
+
+    /// process login info first
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads)
+
+                SELECT 'logins' AS stattype, $nextmidnight AS timeend, ".SITEID." AS courseid,
+                       l.userid, count(l.id) AS statsreads
+                  FROM {$CFG->prefix}log l
+                 WHERE action = 'login' AND $timesql
+              GROUP BY stattype, timeend, courseid, userid
+                HAVING count(l.id) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
             break;
         }
-    }
-    mtrace("got up to ".$timestart);
-    mtrace("Completed $days days");
-    return $return;
+        mtrace('1:'.(time() - $start).' ', '');
+        $start = time();
 
-}
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
 
+                SELECT 'logins' AS stattype, $nextmidnight AS timeend, ".SITEID." as courseid, 0,
+                       COALESCE((SELECT SUM(statsreads)
+                                       FROM {$CFG->prefix}stats_user_daily s1
+                                      WHERE s1.stattype = 'logins' AND timeend = $nextmidnight), 0) AS stat1,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}stats_user_daily s2
+                                  WHERE s2.stattype = 'logins' AND timeend = $nextmidnight), 0) AS stat2";
 
-function stats_cron_weekly () {
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('2:'.(time() - $start).' ', '');
+        $start = time();
 
-    global $CFG;
 
-    if (empty($CFG->enablestats)) {
-        STATS_RUN_ABORTED;
-    }
+        // Enrolments and active enrolled users
+        //
+        // Unfortunately, we do not know how many users were registered
+        // at given times in history :-(
+        // - stat1: enrolled users
+        // - stat2: enrolled users active in this period
+        // - enrolment is defined now as having course:view capability in
+        //   course context or above, we look 3 cats upwards only and ignore prevent
+        //   and prohibit caps to simplify it
+        // - SITEID is specialcased here, because it's all about default enrolment
+        //   in that case, we'll count non-deleted users.
+        //
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'enrolments' AS stattype, $nextmidnight AS timeend,
+                        c.id AS courseid, ra.roleid, COUNT(DISTINCT ra.userid) AS stat1, 0 AS stat2
+                  FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                       {$CFG->prefix}course c, $enrolfrom
+                 WHERE $enrolwhere
+                GROUP BY stattype, timeend, c.id, ra.roleid, stat2
+                HAVING COUNT(DISTINCT ra.userid) > 0";
 
-    if (!$timestart = stats_get_start_from('weekly')) {
-        return STATS_RUN_ABORTED;
-    }
-    
-    // check to make sure we're due to run, at least one week after last run
-    $sunday = stats_get_base_weekly(); 
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('3:'.(time() - $start).' ', '');
+        $start = time();
 
-    if (isset($CFG->statslastweekly) and ((time() - (7*24*60*60)) <= $CFG->statslastweekly)) {
-        return STATS_RUN_ABORTED;
-    }
+        // using table alias in UPDATE does not work in pg < 8.2
+        $sql = "UPDATE {$CFG->prefix}stats_daily
+                   SET stat2 = (SELECT COUNT(DISTINCT ra.userid)
+                                  FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                                       {$CFG->prefix}course c, $enrolfrom
+                                 WHERE ra.roleid = {$CFG->prefix}stats_daily.roleid AND
+                                       c.id = {$CFG->prefix}stats_daily.courseid AND
+                                       $enrolwhere AND
+                                       EXISTS (SELECT 'x'
+                                                 FROM {$CFG->prefix}log l
+                                                WHERE l.course = {$CFG->prefix}stats_daily.courseid AND
+                                                      l.userid = ra.userid AND $timesql))
+                 WHERE {$CFG->prefix}stats_daily.stattype = 'enrolments' AND
+                       {$CFG->prefix}stats_daily.timeend = $nextmidnight AND
+                       {$CFG->prefix}stats_daily.courseid IN
+                          (SELECT DISTINCT l.course
+                             FROM {$CFG->prefix}log l
+                            WHERE $timesql)";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('4:'.(time() - $start).' ', '');
+        $start = time();
 
-    mtrace("Running weekly statistics gathering...");
-    set_config('statslastweekly',time());
+    /// now get course total enrolments (roleid==0) - except frontpage
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
 
-    $return = STATS_RUN_COMPLETE; // optimistic
+                SELECT 'enrolments' AS stattype, $nextmidnight AS timeend,
+                       c.id, 0 AS nroleid, COUNT(DISTINCT ra.userid) AS stat1, 0 AS stat2
+                  FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                       {$CFG->prefix}course c, $enrolfrom
+                 WHERE c.id != ".SITEID." AND $enrolwhere
+                GROUP BY stattype, timeend, c.id, nroleid, stat2
+                HAVING COUNT(DISTINCT ra.userid) > 0";
 
-    static $weekly_modules;
-    
-    if (empty($weekly_modules)) {
-        $weekly_modules = array();
-        $mods = get_records("modules");
-        foreach ($mods as $mod) {
-            $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php';
-            if (!is_readable($file)) {
-                continue;
-            }
-            require_once($file);
-            $fname = $mod->name.'_get_weekly_stats';
-            if (function_exists($fname)) {
-                $weekly_modules[$mod] = $fname;
-            }
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
         }
-    }
+        mtrace('5:'.(time() - $start).' ', '');
+        $start = time();
 
-    $nextsunday = stats_get_next_weekend($timestart);
+        $sql = "UPDATE {$CFG->prefix}stats_daily
+                   SET stat2 = (SELECT COUNT(DISTINCT ra.userid)
+                                  FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                                       {$CFG->prefix}course c, $enrolfrom
+                                 WHERE c.id = {$CFG->prefix}stats_daily.courseid AND
+                                       $enrolwhere AND
+                                       EXISTS (SELECT 'x'
+                                                 FROM {$CFG->prefix}log l
+                                                WHERE l.course = {$CFG->prefix}stats_daily.courseid AND
+                                                      l.userid = ra.userid AND $timesql))
+                 WHERE {$CFG->prefix}stats_daily.stattype = 'enrolments' AND
+                       {$CFG->prefix}stats_daily.timeend = $nextmidnight AND
+                       {$CFG->prefix}stats_daily.roleid = 0 AND
+                       {$CFG->prefix}stats_daily.courseid IN
+                          (SELECT l.course
+                             FROM {$CFG->prefix}log l
+                            WHERE $timesql AND l.course != ".SITEID.")";
 
-    if (!$courses = get_records('course','','','','id,1')) {
-        return STATS_RUN_ABORTED;
-    }
-    
-    $weeks = 0;
-    mtrace("starting at $timestart");
-    while ($sunday > $nextsunday && $timestart < $nextsunday) {
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('6:'.(time() - $start).' ', '');
+        $start = time();
 
-        $timesql = " (timeend > $timestart AND timeend < $nextsunday) ";
-        begin_sql();
-        foreach ($courses as $course) {
-            
-            // enrolment first
-            $sql = 'SELECT roleid, ceil(avg(stat1)) AS stat1, ceil(avg(stat2)) AS stat2
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'enrolments\'
-                    GROUP BY roleid';
-            
-            if ($rolestats = get_records_sql($sql)) {
-                foreach ($rolestats as $stat) {
-                    $stat->courseid = $course->id;
-                    $stat->timeend = $nextsunday;
-                    $stat->stattype = 'enrolments';
-                    
-                    insert_record('stats_weekly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
-            
-            // activity
-            $sql = 'SELECT roleid, sum(stat1) AS stat1, sum(stat2) as stat2
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'activity\'
-                    GROUP BY roleid';
-            
-            if ($rolestats = get_records_sql($sql)) {
-                foreach ($rolestats as $stat) {
-                    $stat->courseid = $course->id;
-                    $stat->timeend = $nextsunday;
-                    $stat->stattype = 'activity';
-                    unset($stat->id);
-                    
-                    insert_record('stats_weekly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
-            
-            // logins
-            if ($course->id == SITEID) {
-                $sql = 'SELECT sum(stat1) AS stat1
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'logins\'';
-                
-                if ($stat = get_record_sql($sql)) {
-                    if (empty($stat->stat1)) {
-                        $stat->stat1 = 0;
-                    }
-                    $stat->courseid = $course->id;
-                    $stat->roleid = 0;
-                    $stat->timeend = $nextsunday;
-                    $stat->stattype = 'logins';
-                    $sql = 'SELECT COUNT(DISTINCT(l.userid)) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '
-                        .str_replace('timeend','time',$timesql);
-                    $stat->stat2 = count_records_sql($sql);
-                    
-                    insert_record('stats_weekly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
+    /// frontapge(==site) enrolments total
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
 
-            $users = stats_get_course_users($course,$timesql);
-            foreach ($users as $user) {
-                stats_do_aggregate_user_cron($course,$user,$user->primaryrole,$timesql,$nextsunday,'weekly',$weekly_modules);
+                SELECT 'enrolments', $nextmidnight, ".SITEID.", 0,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}user u
+                                  WHERE u.deleted = 0), 0) AS stat1,
+                       COALESCE((SELECT COUNT(DISTINCT u.id)
+                                   FROM {$CFG->prefix}user u
+                                        JOIN {$CFG->prefix}log l ON l.userid = u.id
+                                  WHERE u.deleted = 0 AND $timesql), 0) AS stat2";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('7:'.(time() - $start).' ', '');
+        $start = time();
+
+        if (empty($CFG->defaultfrontpageroleid)) { // 1.9 only, so far
+            $defaultfproleid = 0;
+        } else {
+            $defaultfproleid = $CFG->defaultfrontpageroleid;
+        }
+
+    /// Default frontpage role enrolments are all site users (not deleted)
+        if ($defaultfproleid) {
+            // first remove default frontpage role counts if created by previous query
+            $sql = "DELETE
+                      FROM {$CFG->prefix}stats_daily
+                     WHERE stattype = 'enrolments' AND courseid = ".SITEID." AND
+                           roleid = $defaultfproleid AND timeend = $nextmidnight";
+            if (!execute_sql($sql, false)) {
+                $failed = true;
+                break;
+            }
+            mtrace('8:'.(time() - $start).' ', '');
+            $start = time();
+
+            $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                    SELECT 'enrolments', $nextmidnight, ".SITEID.", $defaultfproleid,
+                           COALESCE((SELECT COUNT('x')
+                                       FROM {$CFG->prefix}user u
+                                      WHERE u.deleted = 0), 0) AS stat1,
+                           COALESCE((SELECT COUNT(DISTINCT u.id)
+                                       FROM {$CFG->prefix}user u
+                                            JOIN {$CFG->prefix}log l ON l.userid = u.id
+                                      WHERE u.deleted = 0 AND $timesql), 0) AS stat2";
+
+            if (!execute_sql($sql, false)) {
+                $failed = true;
+                break;
             }
+            mtrace('9:'.(time() - $start).' ', '');
+            $start = time();
+
+        } else {
+            mtrace('8 (x)', '');
+            mtrace('9 (X)', '');
         }
-        stats_do_aggregate_user_login_cron($timesql,$nextsunday,'weekly');
-        commit_sql();
-        $timestart = $nextsunday;
-        $nextsunday = stats_get_next_weekend($nextsunday);
-        $weeks++;
 
-        if (!stats_check_runtime()) {
-            mtrace("Stopping early! reached maxruntime");
-            $return = STATS_RUN_ABORTED;
+
+
+    /// individual user stats (including not-logged-in) in each course, this is slow - reuse this data if possible
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads, statswrites)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, d.courseid, d.userid,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}log l
+                                  WHERE l.userid = d.userid AND
+                                        l.course = d.courseid AND $timesql AND
+                                        l.action IN ($viewactions)), 0) AS statsreads,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}log l
+                                  WHERE l.userid = d.userid AND
+                                        l.course = d.courseid AND $timesql AND
+                                        l.action IN ($postactions)), 0) AS statswrites
+                  FROM (SELECT DISTINCT u.id AS userid, l.course AS courseid
+                          FROM {$CFG->prefix}user u, {$CFG->prefix}log l
+                         WHERE u.id = l.userid AND $timesql
+                       UNION
+                        SELECT 0 AS userid, ".SITEID." AS courseid) d";
+                        // can not use group by here because pg can not handle it :-(
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
             break;
         }
+        mtrace('10:'.(time() - $start).' ', '');
+        $start = time();
+
+
+    /// how many view/post actions in each course total
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, c.id AS courseid, 0,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}log l1
+                                  WHERE l1.course = c.id AND l1.action IN ($viewactions) AND
+                                        $timesql1), 0) AS stat1,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}log l2
+                                  WHERE l2.course = c.id AND l2.action IN ($postactions) AND
+                                        $timesql2), 0) AS stat2
+                  FROM {$CFG->prefix}course c
+                 WHERE EXISTS (SELECT 'x'
+                                 FROM {$CFG->prefix}log l
+                                WHERE l.course = c.id and $timesql)";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('11:'.(time() - $start).' ', '');
+        $start = time();
+
+
+    /// how many view actions for each course+role - excluding guests and frontpage
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, pl.courseid, pl.roleid,
+                       SUM(pl.statsreads) AS stat1, SUM(pl.statswrites) AS stat2
+                  FROM (SELECT DISTINCT ra.roleid, c.id AS courseid, sud.statsreads, sud.statswrites
+                          FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx, {$CFG->prefix}course c,
+                               {$CFG->prefix}stats_user_daily sud, $enrolfrom
+                         WHERE c.id != ".SITEID." AND
+                               ra.roleid != $guestrole->id AND ra.userid != $guest->id AND
+                               sud.timeend = $nextmidnight AND sud.userid = ra.userid AND
+                               sud.courseid = c.id AND $enrolwhere
+                        ) pl
+              GROUP BY stattype, timeend, pl.courseid, pl.roleid
+                HAVING SUM(pl.statsreads) > 0 OR SUM(pl.statswrites) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('12:'.(time() - $start).' ', '');
+        $start = time();
+
+    /// how many view actions from guests only in each course - excluding frontpage
+    /// (guest is anybody with guest role or no role with course:view in course - this may not work properly if category limit too low)
+    /// normal users may enter course with temporary guest acces too
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, sud.courseid, $guestrole->id AS nroleid,
+                       SUM(sud.statsreads) AS stat1, SUM(sud.statswrites) AS stat2
+                  FROM {$CFG->prefix}stats_user_daily sud
+                 WHERE sud.timeend = $nextmidnight AND sud.courseid != ".SITEID." AND
+                       (sud.userid = $guest->id OR sud.userid
+                         NOT IN (SELECT ra.userid
+                                   FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                                        {$CFG->prefix}course c, $enrolfrom
+                                  WHERE c.id != ".SITEID." AND  ra.roleid != $guestrole->id AND
+                                        $enrolwhere))
+              GROUP BY stattype, timeend, sud.courseid, nroleid
+                HAVING SUM(sud.statsreads) > 0 OR SUM(sud.statswrites) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('13:'.(time() - $start).' ', '');
+        $start = time();
+
+
+    /// how many view actions for each role on frontpage - excluding guests, not-logged-in and default frontpage role
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, pl.courseid, pl.roleid,
+                       SUM(pl.statsreads) AS stat1, SUM(pl.statswrites) AS stat2
+                  FROM (SELECT DISTINCT ra.roleid, c.id AS courseid, sud.statsreads, sud.statswrites
+                          FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx, {$CFG->prefix}course c,
+                               {$CFG->prefix}stats_user_daily sud, $fpfrom
+                         WHERE c.id = ".SITEID." AND ra.roleid != $defaultfproleid AND
+                               ra.roleid != $guestrole->id AND ra.userid != $guest->id AND
+                               sud.timeend = $nextmidnight AND sud.userid = ra.userid AND
+                               sud.courseid = c.id AND $fpwhere
+                        ) pl
+              GROUP BY stattype, timeend, pl.courseid, pl.roleid
+                HAVING SUM(pl.statsreads) > 0 OR SUM(pl.statswrites) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('14:'.(time() - $start).' ', '');
+        $start = time();
+
+
+    /// how many view actions for default frontpage role on frontpage only
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, sud.courseid, $defaultfproleid AS nroleid,
+                       SUM(sud.statsreads) AS stat1, SUM(sud.statswrites) AS stat2
+                  FROM {$CFG->prefix}stats_user_daily sud
+                 WHERE sud.timeend = $nextmidnight AND sud.courseid = ".SITEID." AND
+                       sud.userid != $guest->id AND sud.userid != 0 AND sud.userid
+                         NOT IN (SELECT ra.userid
+                                   FROM {$CFG->prefix}role_assignments ra, {$CFG->prefix}context ctx,
+                                        {$CFG->prefix}course c, $fpfrom
+                                  WHERE c.id = ".SITEID." AND  ra.roleid != $guestrole->id AND
+                                        ra.roleid != $defaultfproleid AND $fpwhere)
+              GROUP BY stattype, timeend, sud.courseid, nroleid
+                HAVING SUM(sud.statsreads) > 0 OR SUM(sud.statswrites) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('15:'.(time() - $start).' ', '');
+        $start = time();
+
+    /// how many view actions for guests or not-logged-in on frontpage
+        $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS stattype, $nextmidnight AS timeend, ".SITEID." AS courseid, $guestrole->id AS nroleid,
+                       SUM(pl.statsreads) AS stat1, SUM(pl.statswrites) AS stat2
+                  FROM (SELECT sud.statsreads, sud.statswrites
+                          FROM {$CFG->prefix}stats_user_daily sud
+                         WHERE (sud.userid = $guest->id OR sud.userid = 0) AND
+                               sud.timeend = $nextmidnight AND sud.courseid = ".SITEID."
+                        ) pl
+              GROUP BY stattype, timeend, courseid, nroleid
+                HAVING SUM(pl.statsreads) > 0 OR SUM(pl.statswrites) > 0";
+
+        if (!execute_sql($sql, false)) {
+            $failed = true;
+            break;
+        }
+        mtrace('16:'.(time() - $start).' ', '');
+
+        // remember processed days
+        set_config('statslastdaily', $nextmidnight);
+        mtrace(" - finished until $nextmidnight: ".userdate($nextmidnight));
+
+        $timestart    = $nextmidnight;
+        $nextmidnight = stats_get_next_day_start($nextmidnight);
+    }
+
+    set_cron_lock('statsrunning', null);
+
+    if ($failed) {
+        $days--;
+        mtrace("...error occured, completed $days days of statistics.");
+        return false;
+
+    } else {
+        mtrace("...completed $days days of statistics.");
+        return false;
     }
-    mtrace("got up to ".$timestart);
-    mtrace("Completed $weeks weeks");
-    return $return;
 }
-    
 
-function stats_cron_monthly () {
+
+/**
+ * Execute weekly statistics gathering
+ * @return boolean success
+ */
+function stats_cron_weekly() {
     global $CFG;
 
-    if (empty($CFG->enablestats)) {
-        return STATS_RUN_ABORTED;
+    $now = time();
+
+    // read last execution date from db
+    if (!$timestart = get_config(NULL, 'statslastweekly')) {
+        $timestart = stats_get_base_daily(stats_get_start_from('weekly'));
+        set_config('statslastweekly', $timestart);
     }
 
-    if (!$timestart = stats_get_start_from('monthly')) {
-        return STATS_RUN_ABORTED;
+    $nextstartweek = stats_get_next_week_start($timestart);
+
+    // are there any weeks that need to be processed?
+    if ($now < $nextstartweek) {
+        return true; // everything ok and up-to-date
     }
-    
-    // check to make sure we're due to run, at least one month after last run
-    $monthend = stats_get_base_monthly();
-    
-    if (isset($CFG->statslastmonthly) and ((time() - (31*24*60*60)) <= $CFG->statslastmonthly)) {
-        return STATS_RUN_ABORTED;
-    }
-    
-    mtrace("Running monthly statistics gathering...");
-    set_config('statslastmonthly',time());
-
-    $return = STATS_RUN_COMPLETE; // optimistic
-
-    static $monthly_modules;
-    
-    if (empty($monthly_modules)) {
-        $monthly_modules = array();
-        $mods = get_records("modules");
-        foreach ($mods as $mod) {
-            $file = $CFG->dirroot.'/mod/'.$mod->name.'/lib.php';
-            if (!is_readable($file)) {
-                continue;
-            }
-            require_once($file);
-            $fname = $mod->name.'_get_monthly_stats';
-            if (function_exists($fname)) {
-                $monthly_modules[$mod] = $fname;
-            }
+
+    $timeout = empty($CFG->statsmaxruntime) ? 60*60*24 : $CFG->statsmaxruntime;
+
+    if (!set_cron_lock('statsrunning', $now + $timeout)) {
+        return false;
+    }
+
+    // fisrt delete entries that should not be there yet
+    delete_records_select('stats_weekly',      "timeend > $timestart");
+    delete_records_select('stats_user_weekly', "timeend > $timestart");
+
+    mtrace("Running weekly statistics gathering, starting at $timestart...");
+
+    $weeks = 0;
+    while ($now > $nextstartweek) {
+        @set_time_limit($timeout - 200);
+        $weeks++;
+
+        if ($weeks > 1) {
+            // move the lock
+            set_cron_lock('statsrunning', time() + $timeout, true);
         }
+
+        $logtimesql  = "l.time >= $timestart AND l.time < $nextstartweek";
+        $stattimesql = "timeend > $timestart AND timeend <= $nextstartweek";
+
+    /// process login info first
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_weekly (stattype, timeend, courseid, userid, statsreads)
+
+                SELECT 'logins' AS stattype, $nextstartweek AS timeend, ".SITEID." as courseid,
+                       l.userid, count(l.id) AS statsreads
+                  FROM {$CFG->prefix}log l
+                 WHERE action = 'login' AND $logtimesql
+              GROUP BY stattype, timeend, courseid, userid
+                HAVING count(l.id) > 0";
+
+        execute_sql($sql, false);
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_weekly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'logins' AS stattype, $nextstartweek AS timeend, ".SITEID." as courseid, 0,
+                       COALESCE((SELECT SUM(statsreads)
+                                   FROM {$CFG->prefix}stats_user_weekly s1
+                                  WHERE s1.stattype = 'logins' AND timeend = $nextstartweek), 0) AS nstat1,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}stats_user_weekly s2
+                                  WHERE s2.stattype = 'logins' AND timeend = $nextstartweek), 0) AS nstat2";
+
+        execute_sql($sql, false);
+
+
+    /// now enrolments averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_weekly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'enrolments' AS nstattype, $nextstartweek AS ntimeend, courseid, roleid,
+                       CEIL(AVG(stat1)) AS nstat1, CEIL(AVG(stat2)) AS nstat2
+                  FROM {$CFG->prefix}stats_daily sd
+                 WHERE stattype = 'enrolments' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, roleid";
+
+        execute_sql($sql, false);
+
+
+    /// activity read/write averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_weekly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS nstattype, $nextstartweek AS ntimeend, courseid, roleid,
+                       SUM(stat1) AS nstat1, SUM(stat2) AS nstat2
+                  FROM {$CFG->prefix}stats_daily
+                 WHERE stattype = 'activity' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, roleid";
+
+        execute_sql($sql, false);
+
+
+    /// user read/write averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_weekly (stattype, timeend, courseid, userid, statsreads, statswrites)
+
+                SELECT 'activity' AS nstattype, $nextstartweek AS ntimeend, courseid, userid,
+                       SUM(statsreads) AS nstatsreads, SUM(statswrites) AS nstatswrites
+                  FROM {$CFG->prefix}stats_user_daily
+                 WHERE stattype = 'activity' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, userid";
+
+        execute_sql($sql, false);
+
+        set_config('statslastweekly', $nextstartweek);
+        mtrace(" finished until $nextstartweek: ".userdate($nextstartweek));
+
+        $timestart     = $nextstartweek;
+        $nextstartweek = stats_get_next_week_start($nextstartweek);
+    }
+
+    set_cron_lock('statsrunning', null);
+    mtrace("...completed $weeks weeks of statistics.");
+    return true;
+}
+
+/**
+ * Execute monthly statistics gathering
+ * @return boolean success
+ */
+function stats_cron_monthly() {
+    global $CFG;
+
+    $now = time();
+
+    // read last execution date from db
+    if (!$timestart = get_config(NULL, 'statslastmonthly')) {
+        $timestart = stats_get_base_monthly(stats_get_start_from('monthly'));
+        set_config('statslastmonthly', $timestart);
     }
-    
-    $nextmonthend = stats_get_next_monthend($timestart);
 
-    if (!$courses = get_records('course','','','','id,1')) {
-        return STATS_RUN_ABORTED;
+    $nextstartmonth = stats_get_next_month_start($timestart);
+
+    // are there any months that need to be processed?
+    if ($now < $nextstartmonth) {
+        return true; // everything ok and up-to-date
     }
-    
-    $months = 0;
-    mtrace("starting from $timestart");
-    while ($monthend > $nextmonthend && $timestart < $nextmonthend) {
 
-        $timesql = " (timeend > $timestart AND timeend < $nextmonthend) ";
-        begin_sql();
-        foreach ($courses as $course) {
-            
-            // enrolment first
-            $sql = 'SELECT roleid, ceil(avg(stat1)) AS stat1, ceil(avg(stat2)) AS stat2
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'enrolments\'
-                    GROUP BY roleid';
-            
-            if ($rolestats = get_records_sql($sql)) {
-                foreach ($rolestats as $stat) {
-                    $stat->courseid = $course->id;
-                    $stat->timeend = $nextmonthend;
-                    $stat->stattype = 'enrolments';
-                    
-                    insert_record('stats_monthly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
-            
-            // activity
-            $sql = 'SELECT roleid, sum(stat1) AS stat1, sum(stat2) as stat2
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'activity\'
-                    GROUP BY roleid';
-            
-            if ($rolestats = get_records_sql($sql)) {
-                foreach ($rolestats as $stat) {
-                    $stat->courseid = $course->id;
-                    $stat->timeend = $nextmonthend;
-                    $stat->stattype = 'activity';
-                    unset($stat->id);
-                    
-                    insert_record('stats_monthly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
-            
-            // logins
-            if ($course->id == SITEID) {
-                $sql = 'SELECT sum(stat1) AS stat1
-                    FROM '.$CFG->prefix.'stats_daily 
-                    WHERE courseid = '.$course->id.' AND '.$timesql.' AND stattype = \'logins\'';
-                
-                if ($stat = get_record_sql($sql)) {
-                    if (empty($stat->stat1)) {
-                        $stat->stat1 = 0;
-                    }
-                    $stat->courseid = $course->id;
-                    $stat->roleid = 0;
-                    $stat->timeend = $nextmonthend;
-                    $stat->stattype = 'logins';
-                    $sql = 'SELECT COUNT(DISTINCT(l.userid)) FROM '.$CFG->prefix.'log l WHERE l.action = \'login\' AND '
-                        .str_replace('timeend','time',$timesql);
-                    $stat->stat2 = count_records_sql($sql);
-                    
-                    insert_record('stats_monthly',$stat,false); // don't worry about the return id, we don't need it.
-                }
-            }
+    $timeout = empty($CFG->statsmaxruntime) ? 60*60*24 : $CFG->statsmaxruntime;
 
-            $users = stats_get_course_users($course,$timesql);
-            foreach ($users as $user) {
-                stats_do_aggregate_user_cron($course,$user,$user->primaryrole,$timesql,$nextmonthend,'monthly',$monthly_modules);
-            }
+    if (!set_cron_lock('statsrunning', $now + $timeout)) {
+        return false;
+    }
 
-        }
-        stats_do_aggregate_user_login_cron($timesql,$nextmonthend,'monthly');
-        commit_sql();
-        $timestart = $nextmonthend;
-        $nextmonthend = stats_get_next_monthend($timestart);
+    // fisr delete entries that should not be there yet
+    delete_records_select('stats_monthly', "timeend > $timestart");
+    delete_records_select('stats_user_monthly', "timeend > $timestart");
+
+    $startmonth = stats_get_base_monthly($now);
+
+
+    mtrace("Running monthly statistics gathering, starting at $timestart...");
+
+    $months = 0;
+    while ($now > $nextstartmonth) {
+        @set_time_limit($timeout - 200);
         $months++;
-        if (!stats_check_runtime()) {
-            mtrace("Stopping early! reached maxruntime");
-            break;
-            $return = STATS_RUN_ABORTED;
+
+        if ($months > 1) {
+            // move the lock
+            set_cron_lock('statsrunning', time() + $timeout, true);
+        }
+
+        $logtimesql  = "l.time >= $timestart AND l.time < $nextstartmonth";
+        $stattimesql = "timeend > $timestart AND timeend <= $nextstartmonth";
+
+    /// process login info first
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_monthly (stattype, timeend, courseid, userid, statsreads)
+
+                SELECT 'logins' AS stattype, $nextstartmonth AS timeend, ".SITEID." as courseid,
+                       l.userid, count(l.id) AS statsreads
+                  FROM {$CFG->prefix}log l
+                 WHERE action = 'login' AND $logtimesql
+              GROUP BY stattype, timeend, courseid, userid";
+
+        execute_sql($sql, false);
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_monthly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'logins' AS stattype, $nextstartmonth AS timeend, ".SITEID." as courseid, 0,
+                       COALESCE((SELECT SUM(statsreads)
+                                   FROM {$CFG->prefix}stats_user_monthly s1
+                                  WHERE s1.stattype = 'logins' AND timeend = $nextstartmonth), 0) AS nstat1,
+                       COALESCE((SELECT COUNT('x')
+                                   FROM {$CFG->prefix}stats_user_monthly s2
+                                  WHERE s2.stattype = 'logins' AND timeend = $nextstartmonth), 0) AS nstat2";
+
+        execute_sql($sql, false);
+
+
+    /// now enrolments averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_monthly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'enrolments' AS nstattype, $nextstartmonth AS ntimeend, courseid, roleid,
+                       CEIL(AVG(stat1)) AS nstat1, CEIL(AVG(stat2)) AS nstat2
+                  FROM {$CFG->prefix}stats_daily sd
+                 WHERE stattype = 'enrolments' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, roleid";
+
+        execute_sql($sql, false);
+
+
+    /// activity read/write averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_monthly (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT 'activity' AS nstattype, $nextstartmonth AS ntimeend, courseid, roleid,
+                       SUM(stat1) AS nstat1, SUM(stat2) AS nstat2
+                  FROM {$CFG->prefix}stats_daily
+                 WHERE stattype = 'activity' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, roleid";
+
+        execute_sql($sql, false);
+
+
+    /// user read/write averages
+        $sql = "INSERT INTO {$CFG->prefix}stats_user_monthly (stattype, timeend, courseid, userid, statsreads, statswrites)
+
+                SELECT 'activity' AS nstattype, $nextstartmonth AS ntimeend, courseid, userid,
+                       SUM(statsreads) AS nstatsreads, SUM(statswrites) AS nstatswrites
+                  FROM {$CFG->prefix}stats_user_daily
+                 WHERE stattype = 'activity' AND $stattimesql
+              GROUP BY nstattype, ntimeend, courseid, userid";
+
+        execute_sql($sql, false);
+
+        set_config('statslastmonthly', $nextstartmonth);
+        mtrace(" finished until $nextstartmonth: ".userdate($nextstartmonth));
+
+        $timestart      = $nextstartmonth;
+        $nextstartmonth = stats_get_next_month_start($nextstartmonth);
+    }
+
+    set_cron_lock('statsrunning', null);
+    mtrace("...completed $months months of statistics.");
+    return true;
+}
+
+/**
+ * Returns simplified enrolment sql join data
+ * @param int $limit number of max parent course categories
+ * @return array from and where string
+ */
+function stats_get_enrolled_sql($limit) {
+    global $CFG;
+
+    $from  = "{$CFG->prefix}role_capabilities rc";
+    for($i=1; $i<=$limit; $i++) {
+        $from .= ", {$CFG->prefix}course_categories cc$i";
+    }
+
+    $where = " (rc.capability = 'moodle/course:view' AND
+                rc.permission = 1 AND rc.contextid = ".SYSCONTEXTID." AND
+                rc.roleid = ra.roleid AND
+                ( (ctx.id = ".SYSCONTEXTID." AND ctx.id = ra.contextid)
+                  OR (ctx.contextlevel = ".CONTEXT_COURSE." AND
+                   ctx.instanceid = c.id AND ctx.id = ra.contextid)";
+
+    for($i=1; $i<=$limit; $i++) {
+        $where .= " OR (ctx.contextlevel = ".CONTEXT_COURSECAT." AND ctx.id = ra.contextid AND ctx.instanceid = cc1.id";
+        for($j=2; $j<=$i; $j++) {
+            $k = $j -1;
+            $where .= " AND cc$j.parent = cc$k.id";
         }
+        $where .=  " AND c.category = cc$i.id)";
     }
-    mtrace("got up to $timestart");
-    mtrace("Completed $months months");
-    return $return;
+
+    $where .= ")) ";
+
+    return array($from, $where);
 }
 
+/**
+ * Return starting date of stats processing
+ * @param string $str name of table - daily, weekly or monthly
+ * @return int timestamp
+ */
 function stats_get_start_from($str) {
     global $CFG;
 
-    // if it's not our first run, just return the most recent.
+    // are there any data in stats table? Should not be...
     if ($timeend = get_field_sql('SELECT timeend FROM '.$CFG->prefix.'stats_'.$str.' ORDER BY timeend DESC')) {
         return $timeend;
     }
-    
     // decide what to do based on our config setting (either all or none or a timestamp)
-    $function = 'stats_get_base_'.$str;
     switch ($CFG->statsfirstrun) {
-        case 'all': 
-            return $function(get_field_sql('SELECT time FROM '.$CFG->prefix.'log ORDER BY time'));
-            break;
-        case 'none': 
-            return $function(strtotime('-1 day',time()));
-            break;
+        case 'all':
+            if ($firstlog = get_field_sql('SELECT time FROM '.$CFG->prefix.'log ORDER BY time ASC')) {
+                return $firstlog;
+            }
         default:
             if (is_numeric($CFG->statsfirstrun)) {
-                return $function(time() - $CFG->statsfirstrun);
+                return time() - $CFG->statsfirstrun;
             }
-            return false;
-            break;
+            // not a number? use next instead
+        case 'none':
+            return strtotime('-3 day', time());
     }
 }
 
+/**
+ * Start of day
+ * @param int $time timestamp
+ * @return start of day
+ */
 function stats_get_base_daily($time=0) {
+    global $CFG;
+
     if (empty($time)) {
         $time = time();
     }
-    return stats_getmidnight($time);
+    if ($CFG->timezone == 99) {
+        $time = strtotime(date('d-M-Y', $time));
+        return $time;
+    } else {
+        $offset = get_timezone_offset($CFG->timezone);
+        $gtime = $time + $offset;
+        $gtime = intval($gtime / (60*60*24)) * 60*60*24;
+        return $gtime - $offset;
+    }
 }
 
+/**
+ * Start of week
+ * @param int $time timestamp
+ * @return start of week
+ */
 function stats_get_base_weekly($time=0) {
-    if (empty($time)) {
-        $time = time();
-    }
-    // if we're currently a monday, last monday will take us back a week
-    $str = 'last monday';
-    if (date('D',$time) == 'Mon')
-        $str = 'now';
+    global $CFG;
 
-    return stats_getmidnight(strtotime($str,$time));
+    $time = stats_get_base_daily($time);
+    $startday = $CFG->calendar_startwday;
+    if ($CFG->timezone == 99) {
+        $thisday = date('w', $time);
+    } else {
+        $offset = get_timezone_offset($CFG->timezone);
+        $gtime = $time + $offset;
+        $thisday = gmdate('w', $gtime);
+    }
+    if ($thisday > $startday) {
+        $time = $time - (($thisday - $startday) * 60*60*24);
+    } else if ($thisday < $startday) {
+        $time = $time - ((7 + $thisday - $startday) * 60*60*24);
+    }
+    return $time;
 }
 
+/**
+ * Start of month
+ * @param int $time timestamp
+ * @return start of month
+ */
 function stats_get_base_monthly($time=0) {
+    global $CFG;
+
     if (empty($time)) {
         $time = time();
     }
-    return stats_getmidnight(strtotime(date('1-M-Y',$time)));
+    if ($CFG->timezone == 99) {
+        return strtotime(date('1-M-Y', $time));
+
+    } else {
+        $time = stats_get_base_daily($time);
+        $offset = get_timezone_offset($CFG->timezone);
+        $gtime = $time + $offset;
+        $day = gmdate('d', $gtime);
+        if ($day == 1) {
+            return $time;
+        }
+        return $gtime - (($day-1) * 60*60*24);
+    }
 }
 
-function stats_get_next_monthend($lastmonth) {
-    return stats_getmidnight(strtotime(date('1-M-Y',$lastmonth).' +1 month'));
+/**
+ * Start of next day
+ * @param int $time timestamp
+ * @return start of next day
+ */
+function stats_get_next_day_start($time) {
+    $next = stats_get_base_daily($time);
+    $next = $next + 60*60*26;
+    $next = stats_get_base_daily($next);
+    if ($next <= $time) {
+        //DST trouble - prevent infinite loops
+        $next = $next + 60*60*24;
+    }
+    return $next;
 }
 
-function stats_get_next_weekend($lastweek) {
-    return stats_getmidnight(strtotime('+1 week',$lastweek));
+/**
+ * Start of next week
+ * @param int $time timestamp
+ * @return start of next week
+ */
+function stats_get_next_week_start($time) {
+    $next = stats_get_base_weekly($time);
+    $next = $next + 60*60*24*9;
+    $next = stats_get_base_weekly($next);
+    if ($next <= $time) {
+        //DST trouble - prevent infinite loops
+        $next = $next + 60*60*24*7;
+    }
+    return $next;
 }
 
-function stats_get_next_dayend($lastday) {
-    return stats_getmidnight(strtotime('+1 day',$lastday));
+/**
+ * Start of next month
+ * @param int $time timestamp
+ * @return start of next month
+ */
+function stats_get_next_month_start($time) {
+    $next = stats_get_base_monthly($time);
+    $next = $next + 60*60*24*33;
+    $next = stats_get_base_monthly($next);
+    if ($next <= $time) {
+        //DST trouble - prevent infinite loops
+        $next = $next + 60*60*24*31;
+    }
+    return $next;
 }
 
+/**
+ * Remove old stats data
+ */
 function stats_clean_old() {
-    mtrace("Running stats cleanup tasks... ");
-    // delete dailies older than 2 months (to be safe)
-    $deletebefore = stats_get_next_monthend(strtotime('-2 months',time()));
-    delete_records_select('stats_daily',"timeend < $deletebefore");
-    delete_records_select('stats_user_daily',"timeend < $deletebefore");
-    
-    // delete weeklies older than 8 months (to be safe)
-    $deletebefore = stats_get_next_monthend(strtotime('-8 months',time()));
-    delete_records_select('stats_weekly',"timeend < $deletebefore");
-    delete_records_select('stats_user_weekly',"timeend < $deletebefore");
+    mtrace("Running stats cleanup tasks...");
+    $deletebefore =  stats_get_base_monthly();
+
+    // delete dailies older than 3 months (to be safe)
+    $deletebefore = strtotime('-3 months', $deletebefore);
+    delete_records_select('stats_daily',      "timeend < $deletebefore");
+    delete_records_select('stats_user_daily', "timeend < $deletebefore");
+
+    // delete weeklies older than 9  months (to be safe)
+    $deletebefore = strtotime('-6 months', $deletebefore);
+    delete_records_select('stats_weekly',      "timeend < $deletebefore");
+    delete_records_select('stats_user_weekly', "timeend < $deletebefore");
 
     // don't delete monthlies
+
+    mtrace("...stats cleanup finished");
 }
 
 function stats_get_parameters($time,$report,$courseid,$mode,$roleid=0) {
     global $CFG,$db;
+
+    $param = new object();
+
     if ($time < 10) { // dailies
         // number of days to go back = 7* time
         $param->table = 'daily';
@@ -605,7 +1006,7 @@
         }
         break;
 
-    case STATS_REPORT_WRITES: 
+    case STATS_REPORT_WRITES:
         $param->fields = $db->Concat('timeend','roleid').' AS uniqueid, timeend, roleid, stat2 as line1';
         $param->fieldscomplete = true; // set this to true to avoid anything adding stuff to the list and breaking complex queries.
         $param->aggregategroupby = 'roleid';
@@ -670,7 +1071,7 @@
         break;
 
     // ******************** STATS_MODE_RANKED ******************** //
-    case STATS_REPORT_ACTIVE_COURSES: 
+    case STATS_REPORT_ACTIVE_COURSES:
         $param->fields = 'sum(stat1+stat2) AS line1';
         $param->stattype = 'activity';
         $param->orderby = 'line1 DESC';
@@ -685,22 +1086,20 @@
         }
         $param->fields = '';
         $param->sql = 'SELECT activity.courseid, activity.all_activity AS line1, enrolments.highest_enrolments AS line2,
-                        activity.all_activity / enrolments.highest_enrolments as line3 
+                        activity.all_activity / enrolments.highest_enrolments as line3
                        FROM (
-                            SELECT courseid, sum(stat1+stat2) AS all_activity 
-                            FROM '.$CFG->prefix.'stats_'.$param->table.'
-                            WHERE stattype=\'activity\' AND timeend >= '.$param->timeafter.'
-                            GROUP BY courseid
+                            SELECT courseid, (stat1+stat2) AS all_activity
+                              FROM '.$CFG->prefix.'stats_'.$param->table.'
+                             WHERE stattype=\'activity\' AND timeend >= '.$param->timeafter.' AND roleid = 0
                        ) activity
-                       INNER JOIN 
+                       INNER JOIN
                             (
                             SELECT courseid, max(stat1) AS highest_enrolments 
-                            FROM '.$CFG->prefix.'stats_'.$param->table.'
-                            WHERE stattype=\'enrolments\' AND timeend >= '.$param->timeafter.'
-                            GROUP BY courseid
+                              FROM '.$CFG->prefix.'stats_'.$param->table.'
+                             WHERE stattype=\'enrolments\' AND timeend >= '.$param->timeafter.' AND stat1 > '.$threshold.' 
+                          GROUP BY courseid
                       ) enrolments
                       ON (activity.courseid = enrolments.courseid)
-                      WhERE enrolments.highest_enrolments > '.$threshold.'
                       ORDER BY line3 DESC';
         $param->line1 = get_string('activity');
         $param->line2 = get_string('users');
@@ -715,14 +1114,13 @@
         }
         $param->fields = '';
         $param->sql = 'SELECT courseid, ceil(avg(all_enrolments)) as line1,
-                         ceil(avg(active_enrolments)) as line2, avg(proportion_active) AS line3 
+                         ceil(avg(active_enrolments)) as line2, avg(proportion_active) AS line3
                        FROM (
-                           SELECT courseid, timeend, sum(stat2) as active_enrolments, 
-                              sum(stat1) as all_enrolments, sum(stat2)'.$real.'/sum(stat1)'.$real.' as proportion_active 
-                           FROM '.$CFG->prefix.'stats_'.$param->table.' WHERE stattype=\'enrolments\' 
-                           GROUP BY courseid, timeend
-                           HAVING sum(stat1) > '.$threshold.'
-                       ) aq 
+                           SELECT courseid, timeend, stat2 as active_enrolments,
+                                  stat1 as all_enrolments, stat2'.$real.'/stat1'.$real.' as proportion_active
+                             FROM '.$CFG->prefix.'stats_'.$param->table.'
+                            WHERE stattype=\'enrolments\' AND roleid = 0 AND stat1 > '.$threshold.'
+                       ) aq
                        WHERE timeend >= '.$param->timeafter.'
                        GROUP BY courseid
                        ORDER BY line3 DESC';
@@ -738,12 +1136,11 @@
         $param->sql =  'SELECT courseid, sum(views) AS line1, sum(posts) AS line2,
                            avg(proportion_active) AS line3
                          FROM (
-                           SELECT courseid, timeend,sum(stat1) as views, sum(stat2) AS posts,
-                            sum(stat2)'.$real.'/sum(stat1)'.$real.' as proportion_active 
-                           FROM '.$CFG->prefix.'stats_'.$param->table.' WHERE stattype=\'activity\' 
-                           GROUP BY courseid, timeend
-                           HAVING sum(stat1) > 0
-                       ) aq 
+                           SELECT courseid, timeend, stat1 as views, stat2 AS posts,
+                                  stat2'.$real.'/stat1'.$real.' as proportion_active
+                             FROM '.$CFG->prefix.'stats_'.$param->table.'
+                            WHERE stattype=\'activity\' AND roleid = 0 AND stat1 > 0
+                       ) aq
                        WHERE timeend >= '.$param->timeafter.'
                        GROUP BY courseid
                        ORDER BY line3 DESC';
@@ -762,7 +1159,7 @@
     */
     //TODO must add the SITEID reports to the rest of the reports.
     return $param;
-} 
+}
 
 function stats_get_view_actions() {
     return array('view','view all','history');
@@ -772,9 +1169,9 @@
     return array('add','delete','edit','add mod','delete mod','edit section'.'enrol','loginas','new','unenrol','update','update mod');
 }
 
-function stats_get_action_sql_in($str) {
+function stats_get_action_names($str) {
     global $CFG;
-    
+
     $mods = get_records('modules');
     $function = 'stats_get_'.$str.'_actions';
     $actions = $function();
@@ -789,121 +1186,17 @@
             $actions = array_merge($actions,$function());
         }
     }
-    $actions = array_unique($actions);
-    if (empty($actions)) {
-        return ' ';
-    } else if (count($actions) == 1) {
-        return ' AND l.action = '.array_pop($actions).' ';
-    } else {
-        return ' AND l.action IN (\''.implode('\',\'',$actions).'\') ';
-    }
-}
 
-
-function stats_get_course_users($course,$timesql) {
-    global $CFG;
-    
-    $timesql = str_replace('timeend','l.time',$timesql);
-
-    $sql = "SELECT userid, primaryrole FROM (
-                SELECT active_course_users.userid,
-                    (SELECT roleid FROM {$CFG->prefix}role_assignments outer_r_a INNER JOIN {$CFG->prefix}role outer_r ON outer_r_a.roleid=outer_r.id
-                        INNER JOIN {$CFG->prefix}context c ON outer_r_a.contextid = c.id
-                        WHERE c.instanceid=".$course->id." AND c.contextlevel = ".CONTEXT_COURSE." AND outer_r_a.userid=active_course_users.userid
-                        AND NOT EXISTS (SELECT 1 FROM {$CFG->prefix}role_assignments inner_r_a
-                            INNER JOIN {$CFG->prefix}role inner_r ON inner_r_a.roleid = inner_r.id
-                            WHERE inner_r.sortorder < outer_r.sortorder
-                            AND inner_r_a.userid = outer_r_a.userid
-                            AND inner_r_a.contextid = outer_r_a.contextid
-                        )
-                    ) AS primaryrole
-                    FROM (SELECT DISTINCT userid FROM {$CFG->prefix}log l WHERE course=".$course->id." AND ".$timesql." )
-                    active_course_users
-                ) foo WHERE primaryrole IS NOT NULL";
-    if (!$users = get_records_sql($sql)) {
-        $users = array();
-    } 
-
-    return $users;
-
-}
-
-function stats_do_daily_user_cron($course,$user,$roleid,$timesql,$timeend,$mods) {
-
-    global $CFG;
-
-    $stat = new StdClass;
-    $stat->userid   = $user->userid;
-    $stat->roleid   = $roleid;
-    $stat->courseid = $course->id;
-    $stat->stattype = 'activity';
-    $stat->timeend  = $timeend;
-    
-    $sql = 'SELECT COUNT(l.id) FROM '.$CFG->prefix.'log l WHERE l.userid = '.$user->userid
-        .' AND  l.course = '.$course->id
-        .' AND '.$timesql .' '.stats_get_action_sql_in('view');
-
-    $stat->statsreads  = count_records_sql($sql);
-    
-    $sql = 'SELECT COUNT(l.id) FROM '.$CFG->prefix.'log l WHERE l.userid = '.$user->userid
-        .' AND l.course = '.$course->id
-        .' AND '.$timesql.' '.stats_get_action_sql_in('post');
-
-    $stat->statswrites = count_records_sql($sql);
-                
-    insert_record('stats_user_daily',$stat,false);
-
-    // now ask the modules if they want anything.
-    foreach ($mods as $mod => $fname) {
-        mtrace('  doing daily statistics for '.$mod->name);
-        $fname($course,$user,$timeend,$roleid);
-    }
-}
-
-function stats_do_aggregate_user_cron($course,$user,$roleid,$timesql,$timeend,$timestr,$mods) {
-
-    global $CFG;
-
-    $stat = new StdClass;
-    $stat->userid   = $user->userid;
-    $stat->roleid   = $roleid;
-    $stat->courseid = $course->id;
-    $stat->stattype = 'activity';
-    $stat->timeend  = $timeend;
-
-    $sql = 'SELECT sum(statsreads) as statsreads, sum(statswrites) as statswrites FROM '.$CFG->prefix.'stats_user_daily WHERE courseid = '.$course->id.' AND '.$timesql
-        ." AND roleid=".$roleid." AND userid = ".$stat->userid." AND stattype='activity'"; // add on roleid in case they have teacher and student records.
-    
-    $r = get_record_sql($sql);
-    $stat->statsreads = (empty($r->statsreads)) ? 0 : $r->statsreads;
-    $stat->statswrites = (empty($r->statswrites)) ? 0 : $r->statswrites;
-    
-    insert_record('stats_user_'.$timestr,$stat,false);
-
-    // now ask the modules if they want anything.
-    foreach ($mods as $mod => $fname) {
-        mtrace('  doing '.$timestr.' statistics for '.$mod->name);
-        $fname($course,$user,$timeend,$roleid);
+    // The array_values() forces a stack-like array
+    // so we can later loop over safely...
+    $actions =  array_values(array_unique($actions));
+    $c = count($actions);
+    for ($n=0;$n<$c;$n++) {
+        $actions[$n] = "'" . $actions[$n] . "'"; // quote them for SQL
     }
+    return $actions;
 }
 
-function stats_do_aggregate_user_login_cron($timesql,$timeend,$timestr) {
-    global $CFG;
-    
-    $sql = 'SELECT userid,roleid,sum(statsreads) as statsreads, sum(statswrites) as writes FROM '.$CFG->prefix.'stats_user_daily WHERE stattype = \'logins\' AND '.$timesql.' GROUP BY userid,roleid';
-    
-    if ($users = get_records_sql($sql)) {
-        foreach ($users as $stat) {
-            $stat->courseid = SITEID;
-            $stat->timeend = $timeend;
-            $stat->stattype = 'logins';
-            
-            insert_record('stats_user_'.$timestr,$stat,false);
-        }
-    }
-}
-
-
 function stats_get_time_options($now,$lastweekend,$lastmonthend,$earliestday,$earliestweek,$earliestmonth) {
 
     $now = stats_get_base_daily(time());
@@ -920,7 +1213,7 @@
         $timeoptions[STATS_TIME_LAST2WEEKS] = get_string('numweeks','moodle',2);
     }
     if ($now - (60*60*24*21) >= $earliestday) {
-        $timeoptions[STATS_TIME_LAST3WEEKS] = get_string('numweeks','moodle',3); 
+        $timeoptions[STATS_TIME_LAST3WEEKS] = get_string('numweeks','moodle',3);
     }
     if ($now - (60*60*24*28) >= $earliestday) {
         $timeoptions[STATS_TIME_LAST4WEEKS] = get_string('numweeks','moodle',4);// show dailies up to (including) here.
@@ -964,7 +1257,7 @@
 
 function stats_get_report_options($courseid,$mode) {
     global $CFG;
-    
+
     $reportoptions = array();
 
     switch ($mode) {
@@ -983,7 +1276,7 @@
         if ($courseid == SITEID) {
             $reportoptions[STATS_REPORT_LOGINS] = get_string('statsreport'.STATS_REPORT_LOGINS);
         }
-        
+
         break;
     case STATS_MODE_DETAILED:
         $reportoptions[STATS_REPORT_USER_ACTIVITY] = get_string('statsreport'.STATS_REPORT_USER_ACTIVITY);
@@ -1002,7 +1295,7 @@
         }
      break;
     }
-  
+
     return $reportoptions;
 }
 
@@ -1014,13 +1307,17 @@
 
     $timestr = str_replace('user_','',$timestr); // just in case.
     $fun = 'stats_get_base_'.$timestr;
-    
+
     $now = $fun();
 
     $times = array();
     // add something to timeafter since it is our absolute base
     $actualtimes = array();
-    foreach ($stats as $s) {
+    foreach ($stats as $statid=>$s) {
+        //normalize the times in stats - those might have been created in different timezone, DST etc.
+        $s->timeend = $fun($s->timeend + 60*60*5);
+        $stats[$statid] = $s;
+
         $actualtimes[] = $s->timeend;
     }
 
@@ -1029,11 +1326,11 @@
     while ($timeafter < $now) {
         $times[] = $timeafter;
         if ($timestr == 'daily') {
-            $timeafter = stats_get_next_dayend($timeafter);
+            $timeafter = stats_get_next_day_start($timeafter);
         } else if ($timestr == 'weekly') {
-            $timeafter = stats_get_next_weekend($timeafter);
+            $timeafter = stats_get_next_week_start($timeafter);
         } else if ($timestr == 'monthly') {
-            $timeafter = stats_get_next_monthend($timeafter);
+            $timeafter = stats_get_next_month_start($timeafter);
         } else {
             return $stats; // this will put us in a never ending loop.
         }
@@ -1056,7 +1353,7 @@
             $stats[] = $newobj;
         }
     }
-    
+
     usort($stats,"stats_compare_times");
     return $stats;
 
@@ -1070,21 +1367,6 @@
    return ($a->timeend > $b->timeend) ? -1 : 1;
 }
 
-function stats_check_runtime() {
-    global $CFG;
-    
-    if (empty($CFG->statsmaxruntime)) {
-        return true;
-    }
-    
-    if ((time() - $CFG->statsrunning) < $CFG->statsmaxruntime) {
-        return true;
-    }
-    
-    return false; // we've gone over! 
-        
-}
-
 function stats_check_uptodate($courseid=0) {
     global $CFG;
 
@@ -1112,39 +1394,52 @@
     return get_string('statscatchupmode','error',$a);
 }
 
+/**
+ * Calculate missing course totals in stats
+ */
+function stats_upgrade_totals() {
+    global $CFG;
 
-// copied from usergetmidnight, but we ignore dst
-function stats_getmidnight($date, $timezone=99) {
-    $timezone = get_user_timezone_offset($timezone);
-    $userdate = getdate($date);
-    return make_timestamp($userdate['year'], $userdate['mon'], $userdate['mday'], 0, 0, 0, $timezone,false ); // ignore dst for this.
-}
-
-function stats_getdate($time, $timezone=99) {
-
-    $timezone = get_user_timezone_offset($timezone);
-
-    if (abs($timezone) > 13) {    // Server time
-        return getdate($time);
-    }
-
-    // There is no gmgetdate so we use gmdate instead
-    $time += intval((float)$timezone * HOURSECS);
-    $datestring = strftime('%S_%M_%H_%d_%m_%Y_%w_%j_%A_%B', $time);
-    list(
-        $getdate['seconds'],
-        $getdate['minutes'],
-        $getdate['hours'],
-        $getdate['mday'],
-        $getdate['mon'],
-        $getdate['year'],
-        $getdate['wday'],
-        $getdate['yday'],
-        $getdate['weekday'],
-        $getdate['month']
-    ) = explode('_', $datestring);
+    if (empty($CFG->statsrolesupgraded)) {
+        // stats not yet upgraded to cope with roles...
+        return;
+    }
+
+    $types = array('daily', 'weekly', 'monthly');
 
-    return $getdate;
+    $now = time();
+    $y30 = 60*60*24*365*30;              // 30 years ago :-O
+    $y20 = 60*60*24*365*20;              // 20 years ago :-O
+    $limit = $now - $y20;
+
+    foreach ($types as $i => $type) {
+        $type2 = $types[($i+1) % count($types)];
+
+        $sql = "DELETE FROM {$CFG->prefix}stats_$type2
+                      WHERE timeend < $limit";
+        execute_sql($sql);
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_$type2 (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT stattype, (timeend - $y30) AS ntimeend, courseid, 0, SUM(stat1), SUM(stat2)
+                  FROM {$CFG->prefix}stats_$type
+                 WHERE (stattype = 'enrolments' OR stattype = 'activity') AND
+                       roleid != 0
+              GROUP BY stattype, ntimeend, courseid";
+        execute_sql($sql);
+
+        $sql = "INSERT INTO {$CFG->prefix}stats_$type (stattype, timeend, courseid, roleid, stat1, stat2)
+
+                SELECT stattype, (timeend + $y30), courseid, roleid, stat1, stat2
+                  FROM {$CFG->prefix}stats_$type2
+                 WHERE (stattype = 'enrolments' OR stattype = 'activity') AND
+                       roleid = 0 AND timeend < $y20";
+        execute_sql($sql);
+
+        $sql = "DELETE FROM {$CFG->prefix}stats_$type2
+                      WHERE timeend < $limit";
+        execute_sql($sql);
+    }
 }
 
 
@@ -1155,22 +1450,25 @@
     }
 
     $result = begin_sql();
-    
+
     $result = $result && stats_upgrade_user_table_for_roles('daily');
     $result = $result && stats_upgrade_user_table_for_roles('weekly');
     $result = $result && stats_upgrade_user_table_for_roles('monthly');
-    
+
     $result = $result && stats_upgrade_table_for_roles('daily');
     $result = $result && stats_upgrade_table_for_roles('weekly');
     $result = $result && stats_upgrade_table_for_roles('monthly');
 
-    
+
     $result = $result && commit_sql();
 
     if (!empty($result)) {
         set_config('statsrolesupgraded',time());
     }
 
+    // finally upgade totals, no big deal if it fails
+    stats_upgrade_totals();
+
     return $result;
 }
 
@@ -1182,7 +1480,7 @@
  * @return boolean @todo maybe something else (error message) depending on
  * how this will be called.
  */
-function stats_upgrade_user_table_for_roles ($period) {
+function stats_upgrade_user_table_for_roles($period) {
     global $CFG;
     static $teacher_role_id, $student_role_id;
 
@@ -1230,7 +1528,7 @@
     if (!in_array($period, array('daily', 'weekly', 'monthly'))) {
         return false;
     }
-    
+
     if (!$teacher_role_id) {
         $role            = get_roles_with_capability('moodle/legacy:teacher', CAP_ALLOW);
         $role            = array_keys($role);
Index: lib/dmllib.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/dmllib.php,v
retrieving revision 1.116.2.18
diff -u -r1.116.2.18 dmllib.php
--- lib/dmllib.php	5 Feb 2008 10:39:29 -0000	1.116.2.18
+++ lib/dmllib.php	15 Feb 2008 22:51:17 -0000
@@ -55,7 +55,7 @@
  * @uses $db
  * @param string $command The sql string you wish to be executed.
  * @param bool $feedback Set this argument to true if the results generated should be printed. Default is true.
- * @return string
+ * @return bool success
  */
 function execute_sql($command, $feedback=true) {
 /// Completely general function - it just runs some SQL and reports success.
Index: lib/moodlelib.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/moodlelib.php,v
retrieving revision 1.960.2.41
diff -u -r1.960.2.41 moodlelib.php
--- lib/moodlelib.php	15 Feb 2008 11:31:38 -0000	1.960.2.41
+++ lib/moodlelib.php	15 Feb 2008 22:51:24 -0000
@@ -1341,6 +1341,28 @@
 }
 
 /**
+ * Returns an int which represents the systems's timezone difference from GMT in seconds
+ * @param mixed $tz timezone
+ * @return int if found, false is timezone 99 or error
+ */
+function get_timezone_offset($tz) {
+    global $CFG;
+
+    if ($tz == 99) {
+        return false;
+    }
+
+    if (is_numeric($tz)) {
+        return intval($tz * 60*60);
+    }
+
+    if (!$tzrecord = get_timezone_record($tz)) {
+        return false;
+    }
+    return intval($tzrecord->gmtoff * 60);
+}
+
+/**
  * Returns a float or a string which denotes the user's timezone
  * A float value means that a simple offset from GMT is used, while a string (it will be the name of a timezone in the database)
  * means that for this timezone there are also DST rules to be taken into account
@@ -3945,6 +3967,7 @@
 
     global $CFG, $FULLME;
 
+return true; // remove before commit!!!!
 
     if (empty($user)) {
         return false;
Index: version.php
===================================================================
RCS file: /cvsroot/moodle/moodle/version.php,v
retrieving revision 1.563.2.16
diff -u -r1.563.2.16 version.php
--- version.php	29 Jan 2008 05:47:01 -0000	1.563.2.16
+++ version.php	15 Feb 2008 22:51:07 -0000
@@ -6,7 +6,7 @@
 // This is compared against the values stored in the database to determine
 // whether upgrades should be performed (see lib/db/*.php)
 
-    $version = 2007101508;  // YYYYMMDD   = date of the 1.9 branch (don't change)
+    $version = 2007101508.6;  // YYYYMMDD   = date of the 1.9 branch (don't change)
                             //         X  = release number 1.9.[0,1,2,3...]
                             //          Y = micro-increments between releases
 
Index: admin/report/courseoverview/index.php
===================================================================
RCS file: /cvsroot/moodle/moodle/admin/report/courseoverview/index.php,v
retrieving revision 1.16
diff -u -r1.16 index.php
--- admin/report/courseoverview/index.php	30 Apr 2007 17:08:51 -0000	1.16
+++ admin/report/courseoverview/index.php	15 Feb 2008 22:51:07 -0000
@@ -18,7 +18,6 @@
         redirect("$CFG->wwwroot/$CFG->admin/settings.php?section=stats", get_string('mustenablestats', 'admin'), 3);
     }
 
-
     $course = get_site();
     stats_check_uptodate($course->id);
 
@@ -48,7 +47,7 @@
     }
 
     echo '<form action="index.php" method="post">'."\n";
-    echo '<fieldset class="invisiblefieldset">';
+    echo '<div>';
 
     $table->width = '*';
     $table->align = array('left','left','left','left','left','left');
@@ -58,17 +57,19 @@
                            '<input type="submit" value="'.get_string('view').'" />') ;
 
     print_table($table);
-    echo '</fieldset>';
+    echo '</div>';
     echo '</form>';
 
+    print_heading($reportoptions[$report]);
+
+
     if (!empty($report) && !empty($time)) {
         $param = stats_get_parameters($time,$report,SITEID,STATS_MODE_RANKED);
-
         if (!empty($param->sql)) {
             $sql = $param->sql;
         } else {
             $sql = "SELECT courseid,".$param->fields." FROM ".$CFG->prefix.'stats_'.$param->table
-                ." WHERE timeend >= ".$param->timeafter.' AND stattype = \'activity\''
+                ." WHERE timeend >= $param->timeafter AND stattype = 'activity' AND roleid = 0"
                 ." GROUP BY courseid "
                 .$param->extras
                 ." ORDER BY ".$param->orderby;
@@ -82,9 +83,9 @@
 
         } else {
             if (empty($CFG->gdversion)) {
-                echo '<div class="boxaligncenter">(' . get_string("gdneed") .')</div>';
+                echo '<div class="graph">(' . get_string("gdneed") .')</div>';
             } else {
-                echo '<div class="boxaligncenter"><img alt="'.get_string('courseoverviewgraph').'" src="'.$CFG->wwwroot.'/'.$CFG->admin.'/report/courseoverview/reportsgraph.php?time='.$time.'&report='.$report.'&numcourses='.$numcourses.'" /></div>';
+                echo '<div class="graph"><img alt="'.get_string('courseoverviewgraph').'" src="'.$CFG->wwwroot.'/'.$CFG->admin.'/report/courseoverview/reportsgraph.php?time='.$time.'&report='.$report.'&numcourses='.$numcourses.'" /></div>';
             }
 
             $table = new StdClass;
Index: admin/report/courseoverview/reportsgraph.php
===================================================================
RCS file: /cvsroot/moodle/moodle/admin/report/courseoverview/reportsgraph.php,v
retrieving revision 1.8
diff -u -r1.8 reportsgraph.php
--- admin/report/courseoverview/reportsgraph.php	3 Aug 2007 03:30:23 -0000	1.8
+++ admin/report/courseoverview/reportsgraph.php	15 Feb 2008 22:51:07 -0000
@@ -20,7 +20,7 @@
         $sql = $param->sql;
     } else {
         $sql = "SELECT courseid,".$param->fields." FROM ".$CFG->prefix.'stats_'.$param->table
-            ." WHERE timeend >= ".$param->timeafter.' AND stattype = \'activity\''
+            ." WHERE timeend >= $param->timeafter AND stattype = 'activity' AND roleid = 0"
             ." GROUP BY courseid "
             .$param->extras
             ." ORDER BY ".$param->orderby;
Index: course/report/stats/report.php
===================================================================
RCS file: /cvsroot/moodle/moodle/course/report/stats/report.php,v
retrieving revision 1.17
diff -u -r1.17 report.php
--- course/report/stats/report.php	30 Apr 2007 17:08:49 -0000	1.17
+++ course/report/stats/report.php	15 Feb 2008 22:51:09 -0000
@@ -17,26 +17,25 @@
         error(get_string('nostatstodisplay'), $CFG->wwwroot.'/course/view.php?id='.$course->id);
     }
 
-    $table->width = '*';
+    $table->width = 'auto';
 
     if ($mode == STATS_MODE_DETAILED) {
         $param = stats_get_parameters($time,null,$course->id,$mode); // we only care about the table and the time string (if we have time)
 
-        $sql = 'SELECT DISTINCT s.userid,s.roleid,r.name AS rolename,r.sortorder,u.firstname,u.lastname,u.idnumber 
+        $sql = 'SELECT DISTINCT s.userid, u.firstname, u.lastname, u.idnumber 
                      FROM '.$CFG->prefix.'stats_user_'.$param->table.' s 
                      JOIN '.$CFG->prefix.'user u ON u.id = s.userid 
-                     JoIN '.$CFG->prefix.'role r ON s.roleid = r.id
                      WHERE courseid = '.$course->id
             . ((!empty($param->stattype)) ? ' AND stattype = \''.$param->stattype.'\'' : '')
             . ((!empty($time)) ? ' AND timeend >= '.$param->timeafter : '')
-            .' ORDER BY r.sortorder';
+            .' ORDER BY u.lastname, u.firstname ASC';
         
         if (!$us = get_records_sql($sql)) {
             error('Cannot enter detailed view: No users found for this course.');
         }
 
         foreach ($us as $u) {
-            $users[$u->userid] = $u->rolename.' - '.fullname($u,true);
+            $users[$u->userid] = fullname($u, true);
         }
         
         $table->align = array('left','left','left','left','left','left','left','left');
@@ -59,24 +58,25 @@
     }
 
     echo '<form action="index.php" method="post">'."\n"
-        .'<fieldset class="invisiblefieldset">'."\n"
+        .'<div>'."\n"
         .'<input type="hidden" name="mode" value="'.$mode.'" />'."\n";
 
     print_table($table);
 
-    echo '</fieldset>';
+    echo '</div>';
     echo '</form>';
 
     if (!empty($report) && !empty($time)) {
         if ($report == STATS_REPORT_LOGINS && $course->id != SITEID) {
             error('This type of report is only available for the site course');
         }
-        $timesql = 
+
         $param = stats_get_parameters($time,$report,$course->id,$mode);
 
         if ($mode == STATS_MODE_DETAILED) {
             $param->table = 'user_'.$param->table;
         }
+
         if (!empty($param->sql)) {
             $sql = $param->sql;
         } else {
@@ -109,16 +109,22 @@
                 echo "(".get_string("gdneed").")";
             } else {
                 if ($mode == STATS_MODE_DETAILED) {
-                    echo '<center><img src="'.$CFG->wwwroot.'/course/report/stats/graph.php?mode='.$mode.'&course='.$course->id.'&time='.$time.'&report='.$report.'&userid='.$userid.'" alt="'.get_string('statisticsgraph').'" /></center>';
+                    echo '<div class="graph"><img src="'.$CFG->wwwroot.'/course/report/stats/graph.php?mode='.$mode.'&amp;course='.$course->id.'&amp;time='.$time.'&amp;report='.$report.'&amp;userid='.$userid.'" alt="'.get_string('statisticsgraph').'" /></div';
                 } else {
-                    echo '<center><img src="'.$CFG->wwwroot.'/course/report/stats/graph.php?mode='.$mode.'&course='.$course->id.'&time='.$time.'&report='.$report.'&roleid='.$roleid.'" alt="'.get_string('statisticsgraph').'" /></center>';
+                    echo '<div class="graph"><img src="'.$CFG->wwwroot.'/course/report/stats/graph.php?mode='.$mode.'&amp;course='.$course->id.'&amp;time='.$time.'&amp;report='.$report.'&amp;roleid='.$roleid.'" alt="'.get_string('statisticsgraph').'" /></div>';
                 }
             }
 
             $table = new StdClass;
             $table->align = array('left','center','center','center');
             $param->table = str_replace('user_','',$param->table);
-            $table->head = array(get_string('periodending','moodle',$param->table));
+            switch ($param->table) {
+                case 'daily'  : $period = get_string('day'); break;
+                case 'weekly' : $period = get_string('week'); break;
+                case 'monthly': $period = get_string('month', 'form'); break;
+                default : $period = '';
+            }
+            $table->head = array(get_string('periodending','moodle',$period));
             if (empty($param->crosstab)) {
                 $table->head[] = $param->line1;
                 if (!empty($param->line2)) {
@@ -133,8 +139,8 @@
                     }
                     if (empty($CFG->loglifetime) || ($stat->timeend-(60*60*24)) >= (time()-60*60*24*$CFG->loglifetime)) {
                         $a[] = '<a href="'.$CFG->wwwroot.'/course/report/log/index.php?id='.
-                            $course->id.'&chooselog=1&showusers=1&showcourses=1&user='
-                            .$userid.'&date='.usergetmidnight($stat->timeend-(60*60*24)).'">'
+                            $course->id.'&amp;chooselog=1&amp;showusers=1&amp;showcourses=1&amp;user='
+                            .$userid.'&amp;date='.usergetmidnight($stat->timeend-(60*60*24)).'">'
                             .get_string('course').' ' .get_string('logs').'</a>&nbsp;';
                     }
                     $table->data[] = $a;
@@ -153,18 +159,20 @@
                         if (!array_key_exists($stat->roleid,$roles)) {
                             $roles[$stat->roleid] = get_field('role','name','id',$stat->roleid);
                         }
+                    } else {
+                        if (!array_key_exists($stat->roleid,$roles)) {
+                            $roles[$stat->roleid] = get_string('all');
+                        }
                     }
                     if (!array_key_exists($stat->timeend,$times)) {
                         $times[$stat->timeend] = userdate($stat->timeend,get_string('strftimedate'),$CFG->timezone);
                     }
                 }
+
                 foreach ($data as $time => $rolesdata) {
                     if (in_array($time,$missedlines)) {
                         $rolesdata = array();
                         foreach ($roles as $roleid => $guff) {
-                            if ($roleid == 0 ) {
-                                continue;
-                            }
                             $rolesdata[$roleid] = 0;
                         }
                     }
@@ -179,8 +187,8 @@
                     $row = array_merge(array($times[$time]),$rolesdata);
                     if (empty($CFG->loglifetime) || ($stat->timeend-(60*60*24)) >= (time()-60*60*24*$CFG->loglifetime)) {
                         $row[] = '<a href="'.$CFG->wwwroot.'/course/report/log/index.php?id='
-                            .$course->id.'&chooselog=1&showusers=1&showcourses=1&user='.$userid
-                            .'&date='.usergetmidnight($time-(60*60*24)).'">'
+                            .$course->id.'&amp;chooselog=1&amp;showusers=1&amp;showcourses=1&amp;user='.$userid
+                            .'&amp;date='.usergetmidnight($time-(60*60*24)).'">'
                             .get_string('course').' ' .get_string('logs').'</a>&nbsp;';
                     }
                     $table->data[] = $row;
Index: course/report/stats/graph.php
===================================================================
RCS file: /cvsroot/moodle/moodle/course/report/stats/graph.php,v
retrieving revision 1.11
diff -u -r1.11 graph.php
--- course/report/stats/graph.php	11 Apr 2007 23:53:15 -0000	1.11
+++ course/report/stats/graph.php	15 Feb 2008 22:51:09 -0000
@@ -60,8 +60,6 @@
     $graph->parameter['title'] = false; // moodle will do a nicer job.
     $graph->y_tick_labels = null;
 
-    $c = array_keys($graph->colour);
-
     if (empty($param->crosstab)) {
         foreach ($stats as $stat) {
             $graph->x_data[] = userdate($stat->timeend,get_string('strftimedate'),$CFG->timezone);
@@ -74,16 +72,17 @@
             }
         }
         $graph->y_order = array('line1');
-        $graph->y_format['line1'] = array('colour' => $c[1],'line' => 'line','legend' => $param->line1);
+        $graph->y_format['line1'] = array('colour' => 'blue','line' => 'line','legend' => $param->line1);
         if (!empty($param->line2)) {
             $graph->y_order[] = 'line2';
-            $graph->y_format['line2'] = array('colour' => $c[2],'line' => 'line','legend' => $param->line2); 
+            $graph->y_format['line2'] = array('colour' => 'green','line' => 'line','legend' => $param->line2); 
         }
         if (!empty($param->line3)) {
             $graph->y_order[] = 'line3';
-            $graph->y_format['line3'] = array('colour' => $c[3],'line' => 'line','legend' => $param->line3); 
+            $graph->y_format['line3'] = array('colour' => 'red','line' => 'line','legend' => $param->line3); 
         }
         $graph->y_tick_labels = false;
+
     } else {
         $data = array();
         $times = array();
@@ -98,6 +97,10 @@
                 if (!array_key_exists($stat->roleid,$roles)) {
                     $roles[$stat->roleid] = get_field('role','name','id',$stat->roleid);
                 }
+            } else {
+                if (!array_key_exists($stat->roleid,$roles)) {
+                    $roles[$stat->roleid] = get_string('all');
+                }
             }
             if (!array_key_exists($stat->timeend,$times)) {
                 $times[$stat->timeend] = userdate($stat->timeend,get_string('strftimedate'),$CFG->timezone);
@@ -110,27 +113,30 @@
                 }
             }
         }
-        foreach ($data as $role => $stuff) {
-            ksort($data[$role]);
-        }
-        $nonzeroroleid = 0;
-        foreach (array_keys($data) as $roleid) {
-            if ($roleid == 0) {
-                continue;
-            }
-            $graph->y_order[] = $roleid;
-            $graph->y_format[$roleid] = array('colour' => $c[$roleid], 'line' => 'line','legend' => $roles[$roleid]);
-            $nonzeroroleid = $roleid;
+
+        $roleid = 0;
+        krsort($roles); // the same sorting as in table bellow graph
+
+        $colors = array('green', 'blue', 'red', 'purple', 'yellow', 'olive', 'navy', 'maroon', 'gray', 'ltred', 'ltltred', 'ltgreen', 'ltltgreen', 'orange', 'ltorange', 'ltltorange', 'lime', 'ltblue', 'ltltblue', 'fuchsia', 'aqua', 'grayF0', 'grayEE', 'grayDD', 'grayCC', 'gray33', 'gray66', 'gray99');
+        $colorindex = 0;
+
+        foreach ($roles as $roleid=>$rname) {
+            ksort($data[$roleid]);
+            $graph->y_order[] = $roleid+1;
+            if ($roleid) {
+                $color = $colors[$colorindex++];
+                $colorindex = $colorindex % count($colors);
+            } else {
+                $color = 'black';
+            }
+            $graph->y_format[$roleid+1] = array('colour' => $color, 'line' => 'line','legend' => $rname);
         }
-        foreach (array_keys($data[$nonzeroroleid]) as $time) {
+        foreach (array_keys($data[$roleid]) as $time) {
             $graph->x_data[] = $times[$time];
         }
         foreach ($data as $roleid => $t) {
-            if ($roleid == 0) {
-                continue;
-            }
             foreach ($t as $time => $data) {
-                $graph->y_data[$roleid][] = $data;
+                $graph->y_data[$roleid+1][] = $data;
             }
         }
     }
Index: theme/standard/styles_layout.css
===================================================================
RCS file: /cvsroot/moodle/moodle/theme/standard/styles_layout.css,v
retrieving revision 1.516.2.39
diff -u -r1.516.2.39 styles_layout.css
--- theme/standard/styles_layout.css	5 Feb 2008 11:42:51 -0000	1.516.2.39
+++ theme/standard/styles_layout.css	15 Feb 2008 22:51:34 -0000
@@ -1154,12 +1154,11 @@
   margin-right: auto;
 }
 
-#admin-report-stats-index .invisiblefieldset {
-  display: block;
-}
-
-#admin-report-courseoverview-index .invisiblefieldset {
-  display: block;
+#admin-report-courseoverview-index .graph,
+#course-report-stats-index .graph,
+#admin-report-stats-index .graph {
+  text-align: center;
+  margin-bottom: 1em;
 }
 
 #admin-uploaduser table#uuresults {
Index: admin/cron.php
===================================================================
RCS file: /cvsroot/moodle/moodle/admin/cron.php,v
retrieving revision 1.126.2.7
diff -u -r1.126.2.7 cron.php
--- admin/cron.php	3 Feb 2008 17:27:32 -0000	1.126.2.7
+++ admin/cron.php	15 Feb 2008 22:51:07 -0000
@@ -430,47 +430,20 @@
     }
 
     if (!empty($CFG->enablestats) and empty($CFG->disablestatsprocessing)) {
-
+        require_once($CFG->dirroot.'/lib/statslib.php');
         // check we're not before our runtime
-        $timetocheck = strtotime("today $CFG->statsruntimestarthour:$CFG->statsruntimestartminute");
+        $timetocheck = stats_get_base_daily() + $CFG->statsruntimestarthour*60*60 + $CFG->statsruntimestartminute*60;
 
         if (time() > $timetocheck) {
-            $time = 60*60*20; // set it to 20 here for first run... (overridden by $CFG)
-            $clobber = true;
-            if (!empty($CFG->statsmaxruntime)) {
-                $time = $CFG->statsmaxruntime+(60*30); // add on half an hour just to make sure (it could take that long to break out of the loop)
-            }
-            if (!get_field_sql('SELECT id FROM '.$CFG->prefix.'stats_daily')) {
-                // first run, set another lock. we'll check for this in subsequent runs to set the timeout to later for the normal lock.
-                set_cron_lock('statsfirstrunlock',true,$time,true);
-                $firsttime = true;
-            }
-            $time = 60*60*2; // this time set to 2.. (overridden by $CFG)
-            if (!empty($CFG->statsmaxruntime)) {
-                $time = $CFG->statsmaxruntime+(60*30); // add on half an hour to make sure (it could take that long to break out of the loop)
-            }
-            if ($config = get_record('config','name','statsfirstrunlock')) {
-                if (!empty($config->value)) {
-                    $clobber = false; // if we're on the first run, just don't clobber it.
-                }
-            }
-            if (set_cron_lock('statsrunning',true,$time, $clobber)) {
-                require_once($CFG->dirroot.'/lib/statslib.php');
-                $return = stats_cron_daily();
-                if (stats_check_runtime() && $return == STATS_RUN_COMPLETE) {
-                    stats_cron_weekly();
-                }
-                if (stats_check_runtime() && $return == STATS_RUN_COMPLETE) {
-                    $return = $return && stats_cron_monthly();
-                }
-                if (stats_check_runtime() && $return == STATS_RUN_COMPLETE) {
-                    stats_clean_old();
-                }
-                set_cron_lock('statsrunning',false);
-                if (!empty($firsttime)) {
-                    set_cron_lock('statsfirstrunlock',false);
+            // process max 3 days per cron execution
+            if (stats_cron_daily(3)) {
+                if (stats_cron_weekly()) {
+                    if (stats_cron_monthly()) {
+                        stats_clean_old();
+                    }
                 }
             }
+            @set_time_limit(0);
         }
     }
 
Index: course/user.php
===================================================================
RCS file: /cvsroot/moodle/moodle/course/user.php,v
retrieving revision 1.75.2.6
diff -u -r1.75.2.6 user.php
--- course/user.php	10 Jan 2008 10:58:09 -0000	1.75.2.6
+++ course/user.php	15 Feb 2008 22:51:09 -0000
@@ -161,7 +161,13 @@
             $table = new object();
             $table->align = array('left','center','center','center');
             $param->table = str_replace('user_','',$param->table);
-            $table->head = array(get_string('periodending','moodle',$param->table),$param->line1,$param->line2,$param->line3);
+            switch ($param->table) {
+                case 'daily'  : $period = get_string('day'); break;
+                case 'weekly' : $period = get_string('week'); break;
+                case 'monthly': $period = get_string('month', 'form'); break;
+                default : $period = '';
+            }
+            $table->head = array(get_string('periodending','moodle',$period),$param->line1,$param->line2,$param->line3);
             foreach ($stats as $stat) {
                 if (!empty($stat->zerofixed)) {  // Don't know why this is necessary, see stats_fix_zeros above - MD
                     continue;
Index: lang/en_utf8/admin.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lang/en_utf8/admin.php,v
retrieving revision 1.154.2.29
diff -u -r1.154.2.29 admin.php
--- lang/en_utf8/admin.php	2 Feb 2008 16:22:16 -0000	1.154.2.29
+++ lang/en_utf8/admin.php	15 Feb 2008 22:51:11 -0000
@@ -217,9 +217,11 @@
 $string['configsmtphosts'] = 'Give the full name of one or more local SMTP servers that Moodle should use to send mail (eg \'mail.a.com\' or \'mail.a.com;mail.b.com\'). If you leave it blank, Moodle will use the PHP default method of sending mail.';
 $string['configsmtpuser'] = 'If you have specified an SMTP server above, and the server requires authentication, then enter the username and password here.';
 $string['configstartwday'] = 'Start of Week';
+$string['configstatscatdepth'] = 'Statistics code uses simplified course enrolment logic, overrides are ignored and there is a maximum number of verified parent course categories. Number 0 means detect only direct role assignments on site and course level, 1 means detect also role assignments in parent category of course, etc. Higher numbers result in much higher database server load during stats processing.';
 $string['configstatsfirstrun'] = 'This specifies how far back the logs should be processed <b>the first time</b> the cronjob wants to process statistics. If you have a lot of traffic and are on shared hosting, it\'s probably not a good idea to go too far back, as it could take a long time to run and be quite resource intensive. (Note that for this setting, 1 month = 28 days. In the graphs and reports generated, 1 month = 1 calendar month.)';
 $string['configstatsmaxruntime'] = 'Stats processing can be quite intensive, so use a combination of this field and the next one to specify when it will run and how long for.';
-$string['configstatsruntimestart'] = 'What time should the cronjob that does the stats processing <b>start</b>?';
+$string['configstatsmaxruntime2'] = 'Stats processing can be quite intensive, specify maximum time allowed for gathering of one day of statistics. Maximum number of days processed in one cron execution is 3.';
+$string['configstatsruntimestart'] = 'What time should the cronjob that does the stats processing <b>start</b>? Please specify different times if there are multiple Moodles on one physical server.';
 $string['configstatsuserthreshold'] = 'If you enter a non-zero,  non numeric value here, for ranking courses, courses with less than this number of enrolled users (all roles) will be ignored';
 $string['configsupportemail'] = 'This email address will be published to users of this site as the one to email when they need general help (for example, when new users create their own accounts).  If this email is left blank then no such helpful email address is supplied.';
 $string['configsupportname'] = 'This is the name of a person or other entity offering general help via the support email or web address.';
@@ -631,6 +633,7 @@
 $string['smtppass'] = 'SMTP password';
 $string['smtpuser'] = 'SMTP username';
 $string['stats'] = 'Statistics';
+$string['statscatdepth'] = 'Maximum parent categories';
 $string['statsfirstrun'] = 'Maximum processing interval';
 $string['statsmaxruntime'] = 'Maximum runtime';
 $string['statsruntimestart'] = 'Run at';
Index: lib/db/install.xml
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/db/install.xml,v
retrieving revision 1.135.2.2
diff -u -r1.135.2.2 install.xml
--- lib/db/install.xml	15 Dec 2007 00:30:41 -0000	1.135.2.2
+++ lib/db/install.xml	15 Feb 2008 22:51:29 -0000
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<XMLDB PATH="lib/db" VERSION="20071215" COMMENT="XMLDB file for core Moodle tables"
+<XMLDB PATH="lib/db" VERSION="20080202" COMMENT="XMLDB file for core Moodle tables"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
 >
@@ -285,11 +285,10 @@
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
       </KEYS>
       <INDEXES>
-        <INDEX NAME="time-course-module-action" UNIQUE="false" FIELDS="time, course, module, action" NEXT="course-module-action"/>
-        <INDEX NAME="course-module-action" UNIQUE="false" FIELDS="course, module, action" PREVIOUS="time-course-module-action" NEXT="course-userid"/>
-        <INDEX NAME="course-userid" UNIQUE="false" FIELDS="course, userid" PREVIOUS="course-module-action" NEXT="userid"/>
-        <INDEX NAME="userid" UNIQUE="false" FIELDS="userid" PREVIOUS="course-userid" NEXT="info"/>
-        <INDEX NAME="info" UNIQUE="false" FIELDS="info" PREVIOUS="userid"/>
+        <INDEX NAME="course-module-action" UNIQUE="false" FIELDS="course, module, action" NEXT="course-userid"/>
+        <INDEX NAME="course-userid" UNIQUE="false" FIELDS="course, userid" PREVIOUS="course-module-action" NEXT="time"/>
+        <INDEX NAME="time" UNIQUE="false" FIELDS="time" PREVIOUS="course-userid" NEXT="action"/>
+        <INDEX NAME="action" UNIQUE="false" FIELDS="action" PREVIOUS="time"/>
       </INDEXES>
     </TABLE>
     <TABLE NAME="log_display" COMMENT="For a particular module/action, specifies a moodle table/field" PREVIOUS="log" NEXT="message">
@@ -1681,4 +1680,4 @@
       </SENTENCES>
     </STATEMENT>
   </STATEMENTS>
-</XMLDB>
+</XMLDB>
\ No newline at end of file
Index: lib/db/upgrade.php
===================================================================
RCS file: /cvsroot/moodle/moodle/lib/db/upgrade.php,v
retrieving revision 1.154.2.17
diff -u -r1.154.2.17 upgrade.php
--- lib/db/upgrade.php	29 Jan 2008 05:47:02 -0000	1.154.2.17
+++ lib/db/upgrade.php	15 Feb 2008 22:51:33 -0000
@@ -1342,6 +1342,35 @@
     /// Launch create table for grade_grades_history
         $result = $result && create_table($table);
 
+
+    /// Define table scale_history to be created
+        $table = new XMLDBTable('scale_history');
+
+    /// Adding fields to table scale_history
+        $table->addFieldInfo('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null);
+        $table->addFieldInfo('action', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('oldid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('source', XMLDB_TYPE_CHAR, '255', null, null, null, null, null, null);
+        $table->addFieldInfo('timemodified', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, null, null, null, null, null);
+        $table->addFieldInfo('loggeduser', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, null, null, null, null, null);
+        $table->addFieldInfo('courseid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('userid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('name', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('scale', XMLDB_TYPE_TEXT, 'small', null, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('description', XMLDB_TYPE_TEXT, 'small', null, XMLDB_NOTNULL, null, null, null, null);
+
+    /// Adding keys to table scale_history
+        $table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id'));
+        $table->addKeyInfo('oldid', XMLDB_KEY_FOREIGN, array('oldid'), 'scale', array('id'));
+        $table->addKeyInfo('courseid', XMLDB_KEY_FOREIGN, array('courseid'), 'course', array('id'));
+        $table->addKeyInfo('loggeduser', XMLDB_KEY_FOREIGN, array('loggeduser'), 'user', array('id'));
+
+    /// Adding indexes to table scale_history
+        $table->addIndexInfo('action', XMLDB_INDEX_NOTUNIQUE, array('action'));
+
+    /// Launch create table for scale_history
+        $result = $result && create_table($table);
+
     /// upgrade the old 1.8 gradebook - migrade data into new grade tables
         if ($result) {
             if ($rs = get_recordset('course')) {
@@ -2689,6 +2718,103 @@
         upgrade_main_savepoint($result, 2007101508);
     }
 
+    if ($result && $oldversion < 2007101508.6) { // TODO: change to 2007101509 before commit!
+
+        // upgade totals, no big deal if it fails
+        require_once($CFG->libdir.'/statslib.php');
+        stats_upgrade_totals();
+
+        if (isset($CFG->loglifetime) and $CFG->loglifetime == 30) {
+            set_config('loglifetime', 35); // we need more than 31 days for monthly stats!
+        }
+
+        notify('Upgrading log table indexes, this may take a long time, please be patient.', 'notifysuccess');
+
+    /// Define index time-course-module-action (not unique) to be dropped form log
+        $table = new XMLDBTable('log');
+        $index = new XMLDBIndex('time-course-module-action');
+        $index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('time', 'course', 'module', 'action'));
+
+    /// Launch drop index time-course-module-action
+        if (index_exists($table, $index)) {
+            $result = drop_index($table, $index) && $result;
+        }
+
+    /// Define index userid (not unique) to be dropped form log
+        $table = new XMLDBTable('log');
+        $index = new XMLDBIndex('userid');
+        $index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('userid'));
+
+    /// Launch drop index userid
+        if (index_exists($table, $index)) {
+            $result = drop_index($table, $index) && $result;
+        }
+
+    /// Define index info (not unique) to be dropped form log
+        $table = new XMLDBTable('log');
+        $index = new XMLDBIndex('info');
+        $index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('info'));
+
+    /// Launch drop index info
+        if (index_exists($table, $index)) {
+            $result = drop_index($table, $index) && $result;
+        }
+
+    /// Define index time (not unique) to be added to log
+        $table = new XMLDBTable('log');
+        $index = new XMLDBIndex('time');
+        $index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('time'));
+
+    /// Launch add index time
+        if (!index_exists($table, $index)) {
+            $result = add_index($table, $index) && $result;
+        }
+
+    /// Define index action (not unique) to be added to log
+        $table = new XMLDBTable('log');
+        $index = new XMLDBIndex('action');
+        $index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('action'));
+
+    /// Launch add index action
+        if (!index_exists($table, $index)) {
+            $result = add_index($table, $index) && $result;
+        }
+
+
+// add forgotten table
+    /// Define table scale_history to be created
+        $table = new XMLDBTable('scale_history');
+
+    /// Adding fields to table scale_history
+        $table->addFieldInfo('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null);
+        $table->addFieldInfo('action', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('oldid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('source', XMLDB_TYPE_CHAR, '255', null, null, null, null, null, null);
+        $table->addFieldInfo('timemodified', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, null, null, null, null, null);
+        $table->addFieldInfo('loggeduser', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, null, null, null, null, null);
+        $table->addFieldInfo('courseid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('userid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, '0');
+        $table->addFieldInfo('name', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('scale', XMLDB_TYPE_TEXT, 'small', null, XMLDB_NOTNULL, null, null, null, null);
+        $table->addFieldInfo('description', XMLDB_TYPE_TEXT, 'small', null, XMLDB_NOTNULL, null, null, null, null);
+
+    /// Adding keys to table scale_history
+        $table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id'));
+        $table->addKeyInfo('oldid', XMLDB_KEY_FOREIGN, array('oldid'), 'scale', array('id'));
+        $table->addKeyInfo('courseid', XMLDB_KEY_FOREIGN, array('courseid'), 'course', array('id'));
+        $table->addKeyInfo('loggeduser', XMLDB_KEY_FOREIGN, array('loggeduser'), 'user', array('id'));
+
+    /// Adding indexes to table scale_history
+        $table->addIndexInfo('action', XMLDB_INDEX_NOTUNIQUE, array('action'));
+
+        if ($result and !table_exists($table)) {
+        /// Launch create table for scale_history
+            $result = $result && create_table($table);
+        }
+
+    /// Main savepoint reached
+        upgrade_main_savepoint($result, 2007101508.6); // TODO: change to 2007101509 before commit!
+    }
 
     return $result;
 }
Index: admin/settings/server.php
===================================================================
RCS file: /cvsroot/moodle/moodle/admin/settings/server.php,v
retrieving revision 1.26.2.10
diff -u -r1.26.2.10 server.php
--- admin/settings/server.php	2 Feb 2008 16:22:15 -0000	1.26.2.10
+++ admin/settings/server.php	15 Feb 2008 22:51:08 -0000
@@ -123,7 +123,9 @@
                                                                                                                                                            60*60*24*140 => get_string('nummonths','moodle',5),
                                                                                                                                                            60*60*24*168 => get_string('nummonths','moodle',6),
                                                                                                                                                            'all' => get_string('all') )));
-$temp->add(new admin_setting_configselect('statsmaxruntime', get_string('statsmaxruntime', 'admin'), get_string('configstatsmaxruntime', 'admin'), 0, array(0 => get_string('untilcomplete'),
+$temp->add(new admin_setting_configselect('statsmaxruntime', get_string('statsmaxruntime', 'admin'), get_string('configstatsmaxruntime2', 'admin'), 0, array(0 => get_string('untilcomplete'),
+                                                                                                                                                            60*30 => '10 '.get_string('minutes'),
+                                                                                                                                                            60*30 => '30 '.get_string('minutes'),
                                                                                                                                                             60*60 => '1 '.get_string('hour'),
                                                                                                                                                             60*60*2 => '2 '.get_string('hours'),
                                                                                                                                                             60*60*3 => '3 '.get_string('hours'),
@@ -134,6 +136,9 @@
                                                                                                                                                             60*60*8 => '8 '.get_string('hours') )));
 $temp->add(new admin_setting_configtime('statsruntimestarthour', 'statsruntimestartminute', get_string('statsruntimestart', 'admin'), get_string('configstatsruntimestart', 'admin'), array('h' => 0, 'm' => 0)));
 $temp->add(new admin_setting_configtext('statsuserthreshold', get_string('statsuserthreshold', 'admin'), get_string('configstatsuserthreshold', 'admin'), 0, PARAM_INT));
+
+$options = array(0=>0, 1=>1, 2=>2, 3=>3, 4=>4, 5=>5, 6=>6);
+$temp->add(new admin_setting_configselect('statscatdepth', get_string('statscatdepth', 'admin'), get_string('configstatscatdepth', 'admin'), 1, $options));
 $ADMIN->add('server', $temp);