### Eclipse Workspace Patch 1.0 #P moodle Index: grade/import/grade_import_form.php =================================================================== RCS file: grade/import/grade_import_form.php diff -N grade/import/grade_import_form.php --- grade/import/grade_import_form.php 24 May 2007 08:57:36 -0000 1.3 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,99 +0,0 @@ -libdir.'/formslib.php'; - -class grade_import_form extends moodleform { - function definition (){ - $mform =& $this->_form; - - // course id needs to be passed for auth purposes - $mform->addElement('hidden', 'id', optional_param('id')); - - // file upload - $mform->addElement('file', 'userfile', get_string('file')); - $mform->addRule('userfile', null, 'required'); - - $this->add_action_buttons(false, get_string('uploadgrades')); - } - - function get_userfile_name(){ - if ($this->is_submitted() and $this->is_validated()) { - // return the temporary filename to process - return $this->_upload_manager->files['userfile']['tmp_name']; - }else{ - return NULL; - } - } -} - -class grade_import_mapping_form extends moodleform { - - function definition () { - global $CFG; - - $mform =& $this->_form; - - // this is an array of headers - $header = $this->_customdata['header']; - // temporary filename - $filename = $this->_customdata['filename']; - // course id - $id = $this->_customdata['id']; - - $mform->addElement('header', 'general', get_string('identifier')); - $mapfromoptions = array(); - - if ($header) { - foreach ($header as $h) { - $mapfromoptions[$h] = $h; - } - } - $mform->addElement('select', 'mapfrom', get_string('mapfrom'), $mapfromoptions); - //choose_from_menu($mapfromoptions, 'mapfrom'); - - $maptooptions = array('userid'=>'userid', 'username'=>'username', 'useridnumber'=>'useridnumber', 'useremail'=>'useremail', '0'=>'ignore'); - //choose_from_menu($maptooptions, 'mapto'); - $mform->addElement('select', 'mapto', get_string('mapto'), $maptooptions); - - $mform->addElement('header', 'general', get_string('mappings')); - - $gradeitems = array(); - - include_once($CFG->libdir.'/gradelib.php'); - - if ($id) { - if ($grade_items = grade_get_items($id)) { - foreach ($grade_items as $grade_item) { - $gradeitems[$grade_item->idnumber] = $grade_item->itemname; - } - } - } - - if ($header) { - foreach ($header as $h) { - - $h = trim($h); - // this is the order of the headers - $mform->addElement('hidden', 'maps[]', $h); - //echo ''; - // this is what they map to - - $mapfromoptions = array_merge(array('0'=>'ignore'), $gradeitems); - $mform->addElement('select', 'mapping[]', $h, $mapfromoptions); - //choose_from_menu($mapfromoptions, 'mapping[]', $h); - - } - } - $newfilename = 'cvstemp_'.time(); - move_uploaded_file($filename, $CFG->dataroot.'/temp/'.$newfilename); - - // course id needs to be passed for auth purposes - $mform->addElement('hidden', 'map', 1); - $mform->addElement('hidden', 'id', optional_param('id')); - //echo ''; - $mform->addElement('hidden', 'filename', $newfilename); - - $this->add_action_buttons(false, get_string('uploadgrades')); - - } -} -?> Index: lib/textlib.class.php =================================================================== RCS file: /cvsroot/moodle/moodle/lib/textlib.class.php,v retrieving revision 1.17 diff -u -r1.17 textlib.class.php --- lib/textlib.class.php 10 Apr 2007 15:32:37 -0000 1.17 +++ lib/textlib.class.php 1 Jun 2007 09:18:31 -0000 @@ -332,5 +332,26 @@ } return $str; } + + /** + * Returns encoding options for select boxes, utf-8 and platform encoding first + * @return array encodings + */ + function get_encodings() { + $encodings = array(); + $encodings['UTF-8'] = 'UTF-8'; + $winenc = strtoupper(get_string('localewincharset')); + if ($winenc != '') { + $encodings[$winenc] = $winenc; + } + $nixenc = strtoupper(get_string('oldcharset')); + $encodings[$nixenc] = $nixenc; + + foreach ($this->typo3cs->synonyms as $enc) { + $enc = strtoupper($enc); + $encodings[$enc] = $enc; + } + return $encodings; + } } ?> Index: lib/db/upgrade.php =================================================================== RCS file: /cvsroot/moodle/moodle/lib/db/upgrade.php,v retrieving revision 1.62 diff -u -r1.62 upgrade.php --- lib/db/upgrade.php 1 Jun 2007 04:46:24 -0000 1.62 +++ lib/db/upgrade.php 1 Jun 2007 09:18:37 -0000 @@ -1326,6 +1326,43 @@ } return $result; + + + if ($result && $oldversion < 2007060101) { + + /// Define table import_buffer to be created + $table = new XMLDBTable('import_buffer'); + + /// Adding fields to table import_buffer + $table->addFieldInfo('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null); + $table->addFieldInfo('userid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, null); + $table->addFieldInfo('headers', XMLDB_TYPE_TEXT, 'medium', null, XMLDB_NOTNULL, null, null, null, null); + $table->addFieldInfo('timecreated', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, null); + + /// Adding keys to table import_buffer + $table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id')); + + /// Launch create table for import_buffer + $result = $result && create_table($table); + + + /// Define table import_buffer_data to be created + $table = new XMLDBTable('import_buffer_data'); + + /// Adding fields to table import_buffer_data + $table->addFieldInfo('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null); + $table->addFieldInfo('bufferid', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, null); + $table->addFieldInfo('data', XMLDB_TYPE_TEXT, 'big', null, XMLDB_NOTNULL, null, null, null, null); + + /// Adding keys to table import_buffer_data + $table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id')); + + /// Launch create table for import_buffer_data + $result = $result && create_table($table); + + } + + return $result; } ?> Index: lib/db/install.xml =================================================================== RCS file: /cvsroot/moodle/moodle/lib/db/install.xml,v retrieving revision 1.67 diff -u -r1.67 install.xml --- lib/db/install.xml 1 Jun 2007 04:46:24 -0000 1.67 +++ lib/db/install.xml 1 Jun 2007 09:18:36 -0000 @@ -1384,7 +1384,7 @@ - +
@@ -1403,6 +1403,27 @@
+ + + + + + + + + + +
+ + + + + + + + + +
Index: version.php =================================================================== RCS file: /cvsroot/moodle/moodle/version.php,v retrieving revision 1.460 diff -u -r1.460 version.php --- version.php 1 Jun 2007 04:46:24 -0000 1.460 +++ version.php 1 Jun 2007 09:18:31 -0000 @@ -6,7 +6,7 @@ // This is compared against the values stored in the database to determine // whether upgrades should be performed (see lib/db/*.php) - $version = 2007060100; // YYYYMMDD = date + $version = 2007060101; // YYYYMMDD = date // XY = increments within a single day $release = '1.9 dev'; // Human-friendly version name Index: grade/import/csv/index.php =================================================================== RCS file: /cvsroot/moodle/moodle/grade/import/csv/index.php,v retrieving revision 1.7 diff -u -r1.7 index.php --- grade/import/csv/index.php 29 May 2007 00:56:46 -0000 1.7 +++ grade/import/csv/index.php 1 Jun 2007 09:18:31 -0000 @@ -1,174 +1,59 @@ -libdir.'/importlib.php'); +require_once($CFG->libdir.'/gradelib.php'); +require_once($CFG->dirroot.'/grade/lib.php'); +require_once($CFG->dirroot.'/grade/import/lib.php'); +require_once('forms.php'); -// capability check -$id = required_param('id', PARAM_INT); // course id -// require_capability('moodle/site:uploadusers', get_context_instance(CONTEXT_SYSTEM)); - -$csv_encode = '/\&\#44/'; -if (isset($CFG->CSV_DELIMITER)) { - $csv_delimiter = '\\' . $CFG->CSV_DELIMITER; - $csv_delimiter2 = $CFG->CSV_DELIMITER; +$id = required_param('id', PARAM_INT); // course id +$action = optional_param('action', 'selectfile', PARAM_ACTION); - if (isset($CFG->CSV_ENCODE)) { - $csv_encode = '/\&\#' . $CFG->CSV_ENCODE . '/'; - } -} else { - $csv_delimiter = "\,"; - $csv_delimiter2 = ","; +if(!$course = get_record('course', 'id', $id)) { + error('Incorrect course id'); } -require_once('../grade_import_form.php'); -require_once($CFG->dirroot.'/grade/lib.php'); - -$course = get_record('course', 'id', $id); -$action = 'importcsv'; -print_header($course->shortname.': '.get_string('grades'), $course->fullname, grade_nav($course, $action)); - -$mform = new grade_import_form(); - -//if ($formdata = $mform2->get_data() ) { - -if (($formdata = data_submitted()) && !empty($formdata->map)) { -// i am not able to get the mapping[] and map[] array using the following line -// they are somehow not returned with get_data() -// if ($formdata = $mform2->get_data()) { - - foreach ($formdata->maps as $i=>$header) { - $map[$header] = $formdata->mapping[$i]; - } - - $map[$formdata->mapfrom] = $formdata->mapto; - - // temporary file name supplied by form - $filename = $CFG->dataroot.'/temp/'.$formdata->filename; - - // Large files are likely to take their time and memory. Let PHP know - // that we'll take longer, and that the process should be recycled soon - // to free up memory. - @set_time_limit(0); - @raise_memory_limit("192M"); - if (function_exists('apache_child_terminate')) { - @apache_child_terminate(); - } - - $text = my_file_get_contents($filename); - - // we only operate if file is readable - if ($fp = fopen($filename, "r")) { - - // --- get header (field names) --- - $header = split($csv_delimiter, fgets($fp,1024)); - - foreach ($header as $i => $h) { - $h = trim($h); $header[$i] = $h; // remove whitespace - } - - while (!feof ($fp)) { - // add something - $line = split($csv_delimiter, fgets($fp,1024)); - - // each line is a student record - unset ($studentid); - unset ($studentgrades); - - foreach ($line as $key => $value) { - - //decode encoded commas - $value = preg_replace($csv_encode,$csv_delimiter2,trim($value)); - switch ($map[$header[$key]]) { - case 'userid': // - $studentid = $value; - break; - case 'useridnumber': - $user = get_record('user', 'idnumber', $value); - $studentid = $user->id; - break; - case 'useremail': - $user = get_record('user', 'email', $value); - $studentid = $user->id; - break; - case 'username': - $user = get_record('user', 'username', $value); - $studentid = $user->id; - break; - // might need to add columns for comments - default: - if (!empty($map[$header[$key]])) { - // case of an idnumber, only maps idnumber of a grade_item - $studentgrades[$map[$header[$key]]] = $value; - } // otherwise, we ignore this column altogether (e.g. institution, address etc) - break; - } - } - if (!empty($studentgrades)) { - foreach ($studentgrades as $idnumber => $studentgrade) { - - unset($eventdata); - $eventdata->idnumber = $idnumber; - $eventdata->userid = $studentid; - $eventdata->gradevalue = $studentgrade; - events_trigger('grade_updated_external', $eventdata); - - debugging("triggering event for $idnumber... student id is $studentid and grade is $studentgrade"); - } - } - } - - // temporary file can go now - unlink($filename); - } else { - error ('import file '.$filename.' not readable'); - } +require_login($course); +require_capability('moodle/course:managegrades', get_context_instance(CONTEXT_COURSE, $course->id)); -} else if ($formdata = $mform->get_data() ) { +switch ($action) { + case 'mapping': + do_mapping('importcvs'); die; // TODO: localize nav bar + + case 'selectfile': + default: + select_file(); die; +} - $filename = $mform->get_userfile_name(); - - // Large files are likely to take their time and memory. Let PHP know - // that we'll take longer, and that the process should be recycled soon - // to free up memory. - @set_time_limit(0); - @raise_memory_limit("192M"); - if (function_exists('apache_child_terminate')) { - @apache_child_terminate(); - } +function select_file() { + global $CFG, $COURSE, $USER; - $text = my_file_get_contents($filename); - // trim utf-8 bom - $textlib = new textlib(); - $text = $textlib->trim_utf8_bom($text); - // Fix mac/dos newlines - $text = preg_replace('!\r\n?!',"\n",$text); - $fp = fopen($filename, "w"); - fwrite($fp,$text); - fclose($fp); - - $fp = fopen($filename, "r"); - - // --- get header (field names) --- - $header = split($csv_delimiter, fgets($fp,1024)); - - // print mapping form - $mform2 = new grade_import_mapping_form(qualified_me(), array('id'=>$id, 'header'=>$header, 'filename'=>$filename)); - $mform2->display(); -} else { - $mform->display(); -} -print_footer(); + $previewrows = optional_param('previewrows', 10, PARAM_INT); + $mform = new import_select_csv_form(); -function my_file_get_contents($filename, $use_include_path = 0) { - /// Returns the file as one big long string + if ($mform->is_cancelled()) { + redirect($CFG->wwwroot.'/grade/index.php?id='.$COURSE->id); - $data = ""; - $file = @fopen($filename, "rb", $use_include_path); - if ($file) { - while (!feof($file)) { - $data .= fread($file, 1024); + } else if ($data = $mform->get_data()) { + $dir = $CFG->dataroot.'/temp/grade_import_csv/'.$USER->id; + $filename = $mform->get_new_filename(); + + $mform->save_files($dir); + + $icr = new import_csv_reader(array('encoding'=>$data->encoding)); + if (!$bufferid = $icr->process_file("$dir/$filename")) { + @unlink("$dir/$filename"); + // TODO: add better error explanation + error("Error csv file processing - ".$icr->get_error()); } - fclose($file); + redirect($CFG->wwwroot.'/grade/import/csv/index.php?id='.$COURSE->id.'&action=mapping&bufferid='.$bufferid.'&previewrows='.$previewrows); } - return $data; + + // TODO: localize importcsv + print_header($COURSE->shortname.': '.get_string('grades'), $COURSE->fullname, grade_nav($COURSE, 'importcsv')); + $mform->display(); + print_footer($COURSE); } + ?> \ No newline at end of file Index: grade/import/lib.php =================================================================== RCS file: grade/import/lib.php diff -N grade/import/lib.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ grade/import/lib.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,317 @@ +dirroot.'/grade/import/forms.php'; + +function grade_import_buffer($bufferid, $mapping, $forceitemid) { + global $CFG; + + $buffer = new import_buffer($bufferid); + $errors = 0; + + $grade_item = false; + if ($forceitemid) { + if (!$grade_item = grade_item::fetch('id', $forceitemid)) { + $buffer->delete(); + error('Incorrect grade item submitted.'); + } + } + + $buffer->open(); + $r = 0; + while ($data = $buffer->next($mapping)) { + $r++; + + if (!$user = grade_import_get_user($data, $r, false)) { + $errors++; + continue; + } + + if ($grade_item) { + // it was already slected + } else if (!$grade_item = grade_import_get_grade_item($data, $r, false)) { + $errors++; + continue; + } + + if (!array_key_exists('gradevalue', $data) + and !array_key_exists('feedback', $data) + and !array_key_exists('feedbackformat', $data)) { + $errors++; + continue; + } + + $eventdata = new object(); + $eventdata->itemid = $grade_item->id; + $eventdata->userid = $user->id; + $eventdata->itemtype = $grade_item->itemtype; // speedup of external grade events + $eventdata->itemmodule = $grade_item->itemmodule; // speedup of external grade events + if (array_key_exists('gradevalue', $data)) { + $eventdata->gradevalue = $data['gradevalue']; + } + if (array_key_exists('feedback', $data)) { + $eventdata->feedback = $data['feedback']; + } + if (array_key_exists('feedbackformat', $data)) { + $eventdata->feedbackformat = $data['feedbackformat']; + } + @set_time_limit(60); + + events_trigger('grade_updated_external', $eventdata); + } + $buffer->close(); + return $errors; +} + + +function grade_verify_buffer($bufferid, $mapping, $forceitemid) { + $buffer = new import_buffer($bufferid); + $errors = 0; + + $grade_item = false; + if ($forceitemid) { + if (!$grade_item = grade_item::fetch('id', $forceitemid)) { + $buffer->delete(); + error('Incorrect grade item submitted.'); + } + } + + $buffer->open(); + $r = 0; + while ($data = $buffer->next($mapping)) { + $r++; + + if (!$user = grade_import_get_user($data, $r, true)) { + $errors++; + continue; + } + + if ($grade_item) { + // it was already slected + } else if (!$grade_item = grade_import_get_grade_item($data, $r, true)) { + $errors++; + continue; + } + + if (!array_key_exists('gradevalue', $data) + and !array_key_exists('feedback', $data) + and !array_key_exists('feedbackformat', $data)) { + notify($r.': No grade given'); // TODO: localize + $errors++; + continue; + } + } + $buffer->close(); + return $errors; +} + +/** + * Verify grade_item exists and return it + */ +function grade_import_get_grade_item($data, $r, $verbose=false) { + global $COURSE; + + if (!empty($data['itemid'])) { + if (!$grade_item = grade_item::fetch('id', $data['itemid'])) { + $verbose AND notify($r.': Incorrect grade item specified - item id: '.$data['itemid']); // TODO: localize + return false; + } else { + return $grade_item; + } + } + + $grade_items = grade_get_items($COURSE->id, + empty($data['itemtype']) ? null : $data['itemtype'], + empty($data['itemmodule']) ? null : $data['itemmodule'], + empty($data['iteminstance']) ? null : $data['iteminstance'], + null, + empty($data['itemnumber']) ? null : $data['itemnumber'], + empty($data['itemidnumber']) ? null : $data['itemidnumber'] + ); + + if (!$grade_items or count($grade_items != 1)) { + $verbose AND notify($r.': Grade item not found'); // TODO: localize + return false; + } + $grade_item = reset($grade_items); + return $grade_item; +} + +/** + * Verify user exists and return it + */ +function grade_import_get_user($data, $r, $verbose=false) { + global $CFG; + + $user = false; + if (!empty($data['userid'])) { + if (!$user = get_record('user', 'id', $data['userid'])) { + $verbose AND notify($r.': Incorrect user specified - id: '.$data['userid']); // TODO: localize + return false; + } else { + return $user; + } + } + + if (!empty($data['username'])) { + if (!$user = get_record('user', 'username', $data['username'], 'mnethostid', $CFG->mnet_localhost_id)) { + $verbose AND notify($r.': Incorrect user specified - username: '.$data['username']); // TODO: localize + return false; + } + } + + if (!empty($data['useridnumber'])) { + if ($user and $user->idnumber != $data['useridnumber']) { + $verbose AND notify($r.': Incorrect user specified - idnumber: '.$data['useridnumber']); // TODO: localize + return false; + } else { + $users = get_records('user', 'idnumber', $data['useridnumber']); + if (!$users or count($users) != 1) { + $verbose AND notify($r.': Incorrect user specified - idnumber: '.$data['useridnumber']); // TODO: localize + return false; + } + $user = reset($user); + } + } + + if (!empty($data['useremail'])) { + if ($user and $user->idnumber != $data['useremail']) { + $verbose AND notify($r.': Incorrect user specified - email: '.$data['useremail']); // TODO: localize + return false; + } else { + $users = get_records('user', 'email', $data['useremail']); + if (!$users or count($users) != 1) { + $verbose AND notify($r.': Incorrect user specified - email: '.$data['useremail']); // TODO: localize + return false; + } + $user = reset($user); + } + } + + if (!$user) { + $verbose AND notify($r.': No user specified'); // TODO: localize + return false; + } + + return $user; +} + +function grade_import_sections() { + $sections = array(); + $sections['user'] = new object(); + $sections['user']->title = 'User mapping'; // TODO: localize + $sections['user']->items = array(); + + $userfields = array('userid', 'username', 'useridnumber', 'useremail'); + foreach($userfields as $field) { + $item = new object(); + $item->name = $field; + $item->title = get_string('fieldtitle_'.$field, 'grades'); // TODO: localize + $sections['user']->items[] = $item; + } + + $sections['grade'] = new object(); + $sections['grade']->title = 'Grade mapping'; // TODO: localize + $sections['grade']->items = array(); + $gafields = array('gradevalue', 'feedback', 'feedbackformat'); + foreach($gafields as $field) { + $item = new object(); + $item->name = $field; + $item->title = get_string('fieldtitle_'.$field, 'grades'); // TODO: localize + $sections['grade']->items[] = $item; + } + + $sections['grade_item'] = new object(); + $sections['grade_item']->title = 'Grade item mapping'; // TODO: localize + $sections['grade_item']->items = array(); + $gafields = array('itemid', 'itemtype', 'itemmodule', 'iteminstance', 'itemidnumber', 'itemnumber'); + foreach($gafields as $field) { + $item = new object(); + $item->name = $field; + $item->title = get_string('fieldtitle_'.$field, 'grades'); // TODO: localize + $sections['grade_item']->items[] = $item; + } + + $userfields = array('userid', 'username', 'useridnumber', 'useremail'); + foreach($userfields as $field) { + $item = new object(); + $item->name = $field; + $item->title = get_string('fieldtitle_'.$field, 'grades'); // TODO: localize + } + + return $sections; +} + +function do_mapping($type) { + global $CFG, $COURSE; + + $headerprinted = false; + $bufferid = required_param('bufferid', PARAM_INT); + + $sections = grade_import_sections(); + $mform = new grade_import_mapping_form(null, array('sections'=>$sections, 'bufferid'=>$bufferid)); + + if ($mform->is_cancelled()) { + $buffer = new import_buffer($bufferid); + $buffer->delete(); + redirect($CFG->wwwroot.'/grade/index.php?id='.$COURSE->id); + + } else if ($data = $mform->get_data()) { + $mapping = $mform->get_mapping(); + if (empty($mapping)) { + error('Unknown mapping error'); + } + + if ($data->verify) { + print_header($COURSE->shortname.': '.get_string('grades'), $COURSE->fullname, grade_nav($COURSE, $type)); + print_heading('Verifying grading data...'); // TODO: localize + $errors = grade_verify_buffer($bufferid, $mapping, $data->itemid); + if ($errors == 0) { + notify('Everything OK', 'notifysuccess'); // TODO: localize + print_heading('Importing grades...'); // TODO: localize + $errors = grade_import_buffer($bufferid, $mapping, $data->itemid); + $buffer->delete(); + if ($errors == 0) { + notify('Import finished', 'notifysuccess'); // TODO: localize + } else { + notify("Skipped grades: $errors"); // TODO: localize + } + print_continue($CFG->wwwroot.'/grade/index.php?id='.$COURSE->id); + print_footer($COURSE); + die; + } else { + $headerprinted = true; + notify("Found errors: $errors"); // TODO: localize + } + } else { + print_header($COURSE->shortname.': '.get_string('grades'), $COURSE->fullname, grade_nav($COURSE, $type)); + print_heading('Importing grades...'); // TODO: localize + $errors = grade_import_buffer($bufferid, $mapping, $data->itemid); + $buffer->delete(); + if ($errors == 0) { + notify('Import finished', 'notifysuccess'); // TODO: localize + } else { + notify("Skipped grades: $errors"); // TODO: localize + } + print_continue($CFG->wwwroot.'/grade/index.php?id='.$COURSE->id); + print_footer($COURSE); + die; + } + } + + if (!$headerprinted) { + print_header($COURSE->shortname.': '.get_string('grades'), $COURSE->fullname, grade_nav($COURSE, $type)); + } + + $previewrows = optional_param('previewrows', 10, PARAM_INT); + + print_heading('Import preview'); // TODO: localize + $buffer = new import_buffer($bufferid); + print_box($buffer->preview($previewrows)); + print_heading('Choose field mapping'); // TODO: localize + $mform->display(); + print_footer($COURSE); + die; +} + + +?> Index: grade/import/csv/forms.php =================================================================== RCS file: grade/import/csv/forms.php diff -N grade/import/csv/forms.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ grade/import/csv/forms.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,32 @@ +libdir.'/formslib.php'; + +class import_select_csv_form extends moodleform { + function definition() { + + global $CFG, $COURSE; + $mform =& $this->_form; + + // course id needs to be passed for auth purposes + $mform->addElement('hidden', 'id', $COURSE->id); + + $mform->addElement('header', 'general', 'Csv file import'); // TODO: localize + + $this->set_upload_manager(new upload_manager('csvfile', true, false, $COURSE, false, 0, true, true, false)); + $mform->addElement('file', 'csvfile', get_string('file')); + $mform->addRule('csvfile', get_string('required'), 'required', null, 'client'); + + $textlib = new textlib(); + $encodings = $textlib->get_encodings(); + $mform->addElement('select', 'encoding', get_string('encoding'), $encodings); + + $options = array('10'=>10, '20'=>20, '100'=>100, '1000'=>1000, '100000'=>100000); + $mform->addElement('select', 'previewrows', 'Preview rows', $options); // TODO: localize + + $this->add_action_buttons(); + } +} + + +?> Index: lib/simpletest/testimportlib.php =================================================================== RCS file: lib/simpletest/testimportlib.php diff -N lib/simpletest/testimportlib.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ lib/simpletest/testimportlib.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,79 @@ +libdir.'/simpletestlib.php'); +require_once($CFG->libdir.'/importlib.php'); + +class importlib_test extends UnitTestCase { + + var $csvfile; + + /** + * Create temporary entries in the database for these tests. + * These tests have to work no matter the data currently in the database + * (meaning they should run on a brand new site). This means several items of + * data have to be artificially inseminated (:-) in the DB. + */ + function setUp() { + global $CFG; + + make_upload_directory('temp/simpletest'); + $this->csvfile = $CFG->dataroot.'/temp/simpletest/csvfile.csv'; + } + + /** + * Delete temporary entries from the database + */ + function tearDown() { + @unlink($this->csvfile); + } + + /** + * Tests the installation of event handlers from file + */ + function test__import_csv_reader() { + $fp = fopen($this->csvfile, "w"); + $sample = "username, firstname, lastname\n"; + $sample .= "doe, John, Doe\n"; + $sample .= "novak, Josef, Novak\n"; + $sample .= "smith, John, Smith\n"; + fwrite($fp,$sample); + fclose($fp); + + $icr = new import_csv_reader(); + $bufferid = $icr->process_file($this->csvfile); + + $b = new import_buffer($bufferid); + $this->assertEqual(3, count($b->headers)); + + echo $b->preview(); + + $section = new object(); + $section->headerstr = 'lala'; + $section->items = array(); + $section->items[0] = new object(); + $section->items[0]->name = 'pokus'; + $section->items[0]->title = 'pokus title'; + $section->items[0]->required = true; + $section->items[1] = new object(); + $section->items[1]->name = 'ohlala'; + $section->items[1]->title = 'hahahhahahah'; + $section->items[2] = new object(); + $section->items[2]->name = 'username'; + $section->items[2]->title = 'Uživatelské jméno'; + $s = array($section); + $mform = new import_mapping_form(null, array('sections'=>$s, 'bufferid'=>$bufferid)); + $mform->display(); + + $b->delete(); + } + +} + +?> Index: lib/importlib.php =================================================================== RCS file: lib/importlib.php diff -N lib/importlib.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ lib/importlib.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,529 @@ +libdir.'/formslib.php'; + +/** + * Utility class for reading of csv of files into import buffer. + */ +class import_csv_reader { + var $params; + var $error = false; + + /** + * Constructor. + * @param array @params configuration options TODO: add more and document them + */ + function import_csv_reader($params=null) { + global $CFG; + + @raise_memory_limit("256M"); + if (function_exists('apache_child_terminate')) { + @apache_child_terminate(); + } + + if ($params === null) { + $this->params = array(); + } else { + $this->params = $params; + } + + if (empty($this->params['encoding'])) { + $this->params['encoding'] = 'utf-8'; + } + + if (empty($this->params['csv_encode'])) { + if (isset($CFG->CSV_ENCODE)) { + $this->params['csv_encode'] = '/\&\#' . $CFG->CSV_ENCODE . '/'; + } else { + $this->params['csv_encode'] = '/\&\#44/'; + } + } + + if (empty($this->params['csv_delimiter'])) { + if (isset($CFG->CSV_DELIMITER)) { + $this->params['csv_delimiter'] = '\\' . $CFG->CSV_DELIMITER; + } else { + $this->params['csv_delimiter'] = "\,"; + } + } + + if (empty($this->params['csv_delimiter2'])) { + if (isset($CFG->CSV_DELIMITER)) { + $this->params['csv_delimiter2'] = $CFG->CSV_DELIMITER; + } else { + $this->params['csv_delimiter2'] = ","; + } + } + } + + /** + * Feed csv file into input buffer and delete it. + * + * @param string $filename full file name inluding path + * @return mixed input buffer id or false if error + */ + function process_file($filename) { + global $CFG; + + $textlib = new textlib(); + + /// normalize line endings and do the encoding conversion + $text = file_get_contents($filename); + $text = $textlib->convert($text, $this->params['encoding']); + $text = preg_replace('!\r\n?!',"\n",$text); // Fix mac/dos newlines + if (!$fp = fopen($filename, "w")) { + $this->error = "Can not read file csv file!"; // TODO: localize + return false; + } + fwrite($fp,$text); + unset($text); // release memory + fclose($fp); + + $fp = fopen($filename, "r"); + + // --- get header (field names) --- + $firstline = $textlib->trim_utf8_bom(fgets($fp)); // trim utf-8 bom + $rawheaders = split($this->params['csv_delimiter'], $firstline); + $headers = array(); + foreach ($rawheaders as $h) { + $h = $textlib->strtolower(trim($h)); // remove whitespace, normalize + if ($h == '') { + $this->error = "Error reading headers"; //TODO: localize + return false; + } + if (array_key_exists($h, $headers)) { + $this->error = "Duplicate header"; //TODO: localize + return false; + } + $headers[] = $h; + } + if (empty($headers)) { + $this->error = "No headers found!"; // TODO: localize + return false; + } + + $ibf = new import_buffer_feeder($headers); + + /// read all lines + $linenum = 1; // count header line too + while (!feof ($fp)) { + $linenum++; + $line = trim(fgets($fp)); + if ($line == '') { + continue; // skip empty lines + } + + $i = 0; + $data = array(); + $headercout = count($headers); + + $items = split($this->params['csv_delimiter'], $line); + foreach ($items as $value) { + if ($i == $headercout) { + $ibf->delete(); + $this->error = "Incorrect data on line: $linenum, too many columns or delimiter problem"; // TODO: localize + return false; + } + $value = preg_replace($this->params['csv_encode'], $this->params['csv_delimiter2'], trim($value)); + $data[$headers[$i]] = $value; + $i++; + } + if ($i != $headercout) { + $ibf->delete(); + $this->error = "Incorrect data on line: $linenum, not enough columns"; // TODO: localize + return false; + } + $ibf->store($data); + } + + // temporary file can go now + unlink($filename); + + return $ibf->get_id(); // check using is_numeric(); + } + + /** + * Returns reason why process_file() failed + * @return string error message + */ + function get_error() { + return $this->error; + } +} + +/** + * Class that creates and feeds data into input buffer. + */ +class import_buffer_feeder { + var $headers; + var $bufferid; + + /** + * Contructor + * @param array $headers array of header names + */ + function import_buffer_feeder($headers) { + global $USER; + + if (!is_array($headers) or empty($headers)) { + error("Incorrect headers for import buffer!"); + } + + $buffer = new object(); + $buffer->userid = $USER->id; + $buffer->headers = addslashes(serialize($headers)); + $buffer->timecreated = time(); + + + if (!$bufferid = insert_record('import_buffer', $buffer)) { + error("Con not insert buffer"); + } + + $this->headers = $headers; + $this->bufferid = $bufferid; + } + + /** + * Store data row in input buffer. + * + * @param array $row array of arrays + */ + function store($row) { + if (!$this->bufferid) { + error("Buffer not initialized, already destroyed?"); + } + + if (!is_array($row) or empty($row)) { + error("Incorrect buffer row data!"); + } + + $data = new object(); + $data->bufferid = $this->bufferid; + $data->data = addslashes(serialize($row)); + if (!insert_record('import_buffer_data', $data)) { + error("Con not insert data into buffer"); + } + } + + /** + * Returns id of created buffer. + * @return int buffer id + */ + function get_id() { + return $this->bufferid; + } + + /** + * Delete associated buffer and all its data. + */ + function delete() { + $ib = new import_buffer($this->bufferid); + $ib->delete(); + $this->bufferid = false; + } +} + +/** + * Import buffer class. Used for for temporary storage of data imported from files. + * It allows mapping of columns and data preview. + */ +class import_buffer { + var $rs = false; // result set of data rows + var $freshrs = false; // track rs state to eliminate reopening of freshly created rs + var $bufferid; // id of import buffer + var $headers; // array of header strings + var $defaults; // empty strings used as default values + + /** + * Constructor + * @param int $bufferid id + */ + function import_buffer($bufferid) { + global $USER; + + if (!$buffer = get_record('import_buffer', 'id', $bufferid)) { + error("Incorrect buffer id"); + } + + if ($buffer->userid != $USER->id) { + error("This is not your import buffer!!"); + } + + $this->bufferid = $buffer->id; + $this->headers = unserialize($buffer->headers); + $this->defaults = array(); + foreach($this->headers as $h) { + $this->defaults[$h] = ''; + } + } + + /** + * Open buffer, prepare for sequential reading with next() + */ + function open() { + if ($this->freshrs) { + return; + } + $this->close(); + if ($this->rs = get_recordset('import_buffer_data', 'bufferid', $this->bufferid, 'id ASC')) { + $this->freshrs = true; + } + } + + /** + * Return all headers as array + * + * @param array $mapping array used for remapping of headers + * @return array + */ + function get_headers($mapping=false) { + if (!$mapping) { + return $this->headers; + } + $mdata = array(); + foreach ($mapping as $m=>$h) { + if (in_array($h, $this->headers)) { + $mdata[] = $h; + } + } + return $mdata; + } + + /** + * Return buffer id + * @return array + */ + function get_id() { + return $this->bufferid; + } + + /** + * Get next data row as associative array. + * + * @param array $mapping array used for remapping of headers + * @return array or false if error + */ + function next($mapping=false) { + $this->freshrs = false; + if (!$this->rs) { + return false; + } + + if ($record = rs_fetch_next_record($this->rs)) { + $data = array_merge($this->defaults, unserialize($record->data)); + if (!$mapping) { + return $data; + } + $mdata = array(); + foreach ($mapping as $m=>$h) { + if (array_key_exists($h, $data)) { + $mdata[$m] = $data[$h]; + } else { + $mdata[$m] = null; // this should not happen + } + } + return $mdata; + } + + $this->close(); + return false; + } + + /** + * Close buffer - release resultset, keep all data. + */ + function close() { + $this->freshrs = false; + if ($this->rs) { + rs_close($this->rs); + $this->rs = false; + } + } + + /** + * Delete buffer and its data. + */ + function delete() { + $this->close(); + delete_records('import_buffer_data', 'bufferid', $this->bufferid); + delete_records('import_buffer', 'id', $this->bufferid); + } + + /** + * Static method for general cleanup if db tables, to be used from cron script. + */ + function cleanupdb() { + $oldtime = round(time(), -3) - 3600*24; // delete all entries older than one day + if ($oldbufs = get_records_select('import_buffer', "timecreated < $oldtime")) { + foreach ($oldbufs as $buffer) { + delete_records('import_buffer_data', 'bufferid', $buffer->id); + delete_records('import_buffer', 'id', $buffer->id); + } + } + } + + /** + * Return preview of data formatted as html table. + * + * @param int $rows max number of rows + * @param int $cols max number of columns, 0 means all + */ + function preview($rows=10, $cols=0) { + $this->open(); + if ($cols == 0 or $cols > count($this->headers)) { + $cols = count($this->headers); + } + + $res = '
'; + $res .= ''; + $c = 0; + foreach ($this->headers as $h) { + $res .= ''; + $c++; + if ($c >= $cols) { + break; + } + } + $res .= ''; + + $r = 0; + while ($data = $this->next()) { + $r++; + if ($r > $rows) { + continue; + } + $res .= ''; + $res .= ''; + $c = 0; + foreach ($this->headers as $h) { + $res .= ''; + $c++; + if ($c >= $cols) { + break; + } + } + $res .= ''; + } + $res .= '
#'.s($h).'
'.$r.''.s($data[$h]).'
'; + $res .= "Number of data rows: $r, number of columns: ".count($this->headers); // TODO: localize + $res .= '
'; + + return $res; + } +} + +/** + * Basic input mapping form. + */ +class import_mapping_form extends moodleform { + var $buffer; + var $sections; + + /** + * Form definitions - override if needed. + */ + function definition() { + $this->custom_init(); + $this->add_mapping_sections(); + $this->add_action_buttons(); + } + + /** + * Custom validation - override if needed. + * + * @param array $data array of ("fieldname"=>value) of submitted data + * @return bool array of errors or true if ok + */ + function validation($data) { + $errors = array(); + + $used = array(); + foreach ($data as $field=>$value) { + if ($value == '' or strpos($field, 'mapping_') !==0) { + continue; + } + if (in_array($value, $used)) { + $errors[$field] = 'Field already used!'; //TODO: localize + } else { + $used[] = $value; + } + } + if (count($errors) == 0){ + return true; + } else { + return $errors; + } + } + + /** + * Init form + */ + function custom_init() { + global $COURSE; + $mform =& $this->_form; + + $this->buffer = new import_buffer($this->_customdata['bufferid']); + $this->sections = $this->_customdata['sections']; + + // course id needs to be passed for auth purposes + $mform->addElement('hidden', 'id', $COURSE->id); + // selected buffer id + $mform->addElement('hidden', 'bufferid', $this->buffer->get_id()); + } + + /** + * Adds mapping sections + */ + function add_mapping_sections() { + $mform =& $this->_form; + + $headers = $this->buffer->get_headers(); + + $options = array(''=>' '); //nothing selected == ignore + foreach($headers as $h) { + $options[$h] = $h; + } + + foreach ($this->sections as $section) { + $mform->addElement('header', 'general', $section->title); + foreach ($section->items as $item) {//TODO: add CSS + $mform->addElement('select', 'mapping_'.$item->name, ''.$item->name.''.$item->title, $options); + if (in_array($item->name, $headers)) { + $mform->setDefault('mapping_'.$item->name, $item->name); + } + if (!empty($item->required)) { + $mform->addRule('mapping_'.$item->name, get_string('required'), 'required', null, 'client'); + } + } + } + } + + /** + * Returns selected mapping. + * @return associative array, without magicquotes; false if error. + */ + function get_mapping() { + if (!$data = $this->get_data(false)) { + return false; + } + + $data = (array)$data; + $mapping = array(); + + foreach ($this->sections as $section) { + foreach ($section->items as $item) { + if (!array_key_exists('mapping_'.$item->name, $data)) { + continue; + } + $value = $data['mapping_'.$item->name]; + if ($value != '') { + $mapping[$item->name] = $value; + } + } + } + + return $mapping; + } +} + +?> Index: grade/import/forms.php =================================================================== RCS file: grade/import/forms.php diff -N grade/import/forms.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ grade/import/forms.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,83 @@ +_form; + + $this->custom_init(); + $this->add_mapping_sections(); + + $mform->addElement('hidden', 'action', 'mapping'); + + $mform->addElement('header', 'general', 'One grade item only'); // TODO: localize + $grading_items = grade_get_items($COURSE->id); + $options = array('0'=>' '); //nothing selected == ignore + foreach ($grading_items as $item) { + $options[$item->id] = format_string($item->itemname); + } + $mform->addElement('select', 'itemid', get_string('gradeitem', 'grades'), $options); + + $mform->addElement('header', 'general', get_string('miscellaneous')); + $options = array('10'=>10, '20'=>20, '100'=>100, '1000'=>1000, '100000'=>100000); + $mform->addElement('select', 'previewrows', 'Preview rows', $options); // TODO: localize + $mform->setDefault('previewrows', optional_param('previewrows', 10, PARAM_INT)); // passed from file select page + + $mform->addElement('header', 'general', 'Verification'); // TODO: localize + $options = array('1'=>'Verify all data first', '0'=>'No verification, skip bad data'); + $mform->addElement('select', 'verify', 'Error handling', $options); // TODO: localize + + + $usersection = $this->sections['user']; + foreach($usersection->items as $i) { + if ($i->name == 'userid') { + continue; + } + $mform->disabledIf('mapping_'.$i->name, 'mapping_userid', 'noteq', ''); + } + + $gradeitemsection = $this->sections['grade_item']; + foreach($gradeitemsection->items as $i) { + $mform->disabledIf('mapping_'.$i->name, 'itemid', 'noteq', '0'); + if ($i->name != 'itemid') { + $mform->disabledIf('mapping_'.$i->name, 'mapping_itemid', 'noteq', ''); + } + } + + $this->add_action_buttons(); + } + + function validation($data) { + $errors = parent::validation($data); + if ($errors === true) { + $errors = array(); + } + + $userfields = array('userid', 'username', 'useridnumber', 'useremail'); + $selected = 0; + foreach ($userfields as $f) { + if (!empty($data['mapping_'.$f])) { + $selected++; + } + } + if ($selected != 1) { + foreach ($userfields as $f) { + $errors['mapping_'.$f] = 'Select exactly one field describing the user.'; // TODO: localize + } + } + + if (empty($data['mapping_gradevalue']) and empty($data['mapping_feedback'])) { + $errors['mapping_gradevalue'] = 'Select at least one grade mapping.'; // TODO: localize + $errors['mapping_feedback'] = 'Select at least one grade mapping.'; // TODO: localize + } + + //TODO: add some grade item mapping validation + + if (count($errors) == 0){ + return true; + } else { + return $errors; + } + } +} +?>