<?php
/**
 * @file
 * Public API of the Node import module.
 */

/**
 * @mainpage Public API of Node import
 *
 * Global variable: $node_import_can_continue
 *
 *   Boolean. Some imports may not be able to do more then one row at a
 *   time because of all kinds of (static) caching in Drupal. For example
 *   if you import a node with taxonomy terms and select to create the
 *   non-existing terms on import, taxonomy_get_tree() will not have an
 *   up-to-date internal list of terms and so the submission of the form
 *   will fail due to "An illegal choice has been detected. Please contact
 *   the site administrator.".
 *
 * TODO
 */

/**
 * @defgroup node_import_constants Node import constants
 * @{
 * The constants are used in {node_import_tasks} and {node_import_status}
 * database tables to specify the status of either the complete task or
 * one individual row.
 */

/**
 * Status pending.
 *
 * - In {node_import_tasks} : the task is not yet finished and there are
 *    more pending rows to import.
 *
 * - In {node_import_status} : the import of the row has started, but has
 *   not yet finished. This means the row may have to be reimported. Note
 *   that we don't really know what part of it didn't finish :-(
 */
define('NODE_IMPORT_STATUS_PENDING', 0);

/**
 * Status error.
 *
 * - In {node_import_tasks} : the task has finished with some errors and
 *   the user has chosen to retry importing the rows that had errors.
 *   Possibly after editing the task. This has not yet been implemented.
 *
 * - In {node_import_status} : the row did not import and there were
 *   errors.
 */
define('NODE_IMPORT_STATUS_ERROR', 1);

/**
 * Status finished.
 *
 * - In {node_import_tasks} : the task is fully finished and file and task
 *   can be deleted.
 *
 * - In {node_import_status} : the row has been imported succesfully.
 */
define('NODE_IMPORT_STATUS_DONE', 2);

/**
 * @}
 */

// Load all support files.
foreach ((array)drupal_system_listing('.*\.inc', drupal_get_path('module', 'node_import'), 'name') as $name => $file) {
  if (module_exists($name)) {
    require_once('./'. $file->filename);
  }
}

// Make sure the import directory is created.
node_import_directory();

/**
 * Returns the API version of node_import.
 *
 * Currently versions are/were:
 * - 1.x : 4.7.x version (unsupported),
 * - 2.x : 5.x version (unsupported),
 * - 3.x : 6.x-1.x version (current).
 *
 * @return
 *   A string similar to PHP or Drupal version. Major versions
 *   will not change function prototypes or return values. Minor
 *   versions may add additional keys to returned arrays and
 *   add non-required parameters to functions.
 *
 *   Note that the version does not reflect the version of the
 *   support files only of the API.
 */
function node_import_version() {
  return '3.0';
}

/**
 * @defgroup node_import_hooks Node import hooks
 * @{
 */

/**
 * Returns a list of available content types.
 *
 * @param $check_access
 *   Boolean. If TRUE, only the types the user can create are
 *   returned. If FALSE, all types are returned.
 *
 * @param $reset
 *   Boolean. If TRUE, the internal cache is rebuilt.
 *
 * @return
 *   Array of types. See hook_node_import_types().
 */
function node_import_types($check_access = TRUE, $reset = FALSE) {
  static $types;
  static $allowed_types;

  if (!isset($types) || $reset) {
    $defaults = array(
      'title' => '',
      'can_create' => FALSE,
      'create' => '',
    );

    $utypes = (array)module_invoke_all('node_import_types');
    foreach ($utypes as $type => $typeinfo) {
      $utypes[$type] = array_merge($defaults, $typeinfo);
    }
    drupal_alter('node_import_types', $utypes);

    $types = array_map('strip_tags', node_import_extract_property($utypes, 'title'));
    asort($types);
    foreach ($types as $type => $title) {
      $types[$type] = $utypes[$type];
    }

    $allowed_types = array();
    foreach ($types as $type => $typeinfo) {
      if ($typeinfo['can_create'] === TRUE || (function_exists($function = $typeinfo['can_create']) && $function($type) == TRUE)) {
        $allowed_types[$type] = $typeinfo;
      }
    }
  }

  return $check_access ? $allowed_types : $types;
}

/**
 * Returns a list of available content fields for given
 * node_import type.
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $reset
 *   Boolean. If TRUE, the internal cache is rebuilt.
 *
 * @return
 *   Array of fields. See hook_node_import_fields().
 */
function node_import_fields($type, $reset = FALSE) {
  static $fields;

  if (!isset($fields[$type]) || $reset) {
    $defaults = array(
      'title' => '',
      'tips' => array(),
      'group' => '',
      'module' => '',
      'weight' => 0,

      'is_mappable' => TRUE,
      'map_required' => FALSE,

      'has_multiple' => FALSE,
      'multiple_separator' => variable_get('node_import:multiple_separator', '||'),

      'has_hierarchy' => FALSE,
      'hierarchy_separator' => variable_get('node_import:hierarchy_separator', '>>'),
      'hierarchy_reverse' => FALSE,

      'input_format' => '',
      'preprocess' => array(),
      'allowed_values' => array(),

      'default_value' => NULL,
      'allow_empty' => FALSE,
      'map_required' => FALSE,
      'is_checkboxes' => FALSE,
    );

    $fields[$type] = (array)module_invoke_all('node_import_fields', $type);

    foreach ($fields[$type] as $fieldname => $fieldinfo) {
      // Merge sane defaults.
      $fields[$type][$fieldname] = $fieldinfo = array_merge($defaults, $fieldinfo);

      // Add preprocessors for builtin input_formats.
      if (!empty($fieldinfo['allowed_values'])) {
        $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_values';
        $s = '';
        foreach ($fieldinfo['allowed_values'] as $key => $value) {
          if (drupal_strlen($s) > 60) {
            $s .= ', &#8230;';
            break;
          }
          $s .= ($s == '' ? '' : ', ') . check_plain(drupal_strtolower($key)) . ': ' . check_plain(drupal_strtolower($value));
        }
        $fields[$type][$fieldname]['tips'][] = t('Allowed values (!values).', array('!values' => $s));
      }

      switch ($fieldinfo['input_format']) {
        case 'boolean':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_boolean';
          $fields[$type][$fieldname]['tips'][] = t('Boolean (0/1, off/on, no/yes, false/true).');
          break;

        case 'date':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_date';
          $fields[$type][$fieldname]['tips'][] = t('Date ("YYYY-MM-DD HH:MM" or specified custom format).');
          break;

        case 'email':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_email';
          break;

        case 'filepath':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_filepath';
          break;

        case 'node_reference':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_node_reference';
          $fields[$type][$fieldname]['tips'][] = t('Node reference (by nid or title).');
          break;

        case 'user_reference':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_user_reference';
          $fields[$type][$fieldname]['tips'][] = t('User reference (by uid, name or mail).');
          break;

        case 'weight':
          $fields[$type][$fieldname]['preprocess'][] = 'node_import_check_weight';
          break;
      }
    }

    drupal_alter('node_import_fields', $fields[$type], $type);

    // Sort by weight.
    $count = 0;
    $max_count = count($fields[$type]) * 10;
    foreach ($fields[$type] as $fieldname => $fieldinfo) {
      $fields[$type][$fieldname]['weight'] += $count / $max_count;
      $count++;
    }
    uasort($fields[$type], 'node_import_sort');
  }

  return $fields[$type];
}

/**
 * Returns a list of default (form elements).
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $defaults
 *   Array of currently filled in values.
 *
 * @param $fields
 *   Array of available fields.
 *
 * @param $map
 *   Array of how fields are mapped.
 *
 * @return
 *   FAPI array. See hook_node_import_defaults().
 */
function node_import_defaults($type, $defaults, $fields, $map) {
  if (!is_array($defaults)) {
    $defaults = array();
  }

  $form = (array)module_invoke_all('node_import_defaults', $type, $defaults, $fields, $map);

  foreach ($fields as $fieldname => $fieldinfo) {
    // Set #node_import-group if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#node_import-group'])) {
      $form[$fieldname]['#node_import-group'] = $fieldinfo['group'];
    }

    // Set #weight if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#weight'])) {
      $form[$fieldname]['#weight'] = $fieldinfo['weight'];
    }

    // Set #description if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#description'])) {
      if (count($fieldinfo['tips']) > 1) {
        $form[$fieldname]['#description'] = '<ul class="tips">';
        $form[$fieldname]['#description'] .= '<li>' . implode('</li><li>', $fieldinfo['tips']) . '</li>';
        $form[$fieldname]['#description'] .= '</ul>';
      }
      else if (!empty($fieldinfo['tips'])) {
        $form[$fieldname]['#description'] = implode('', $fieldinfo['tips']);
      }
    }

    // Set default_value as value.
    if (!empty($fieldinfo['default_value']) && !isset($form[$fieldname])) {
      $form[$fieldname] = array(
        '#type' => 'value',
        '#value' => $fieldinfo['default_value'],
      );
    }
  }

  drupal_alter('node_import_defaults', $form, $type, $defaults, $fields, $map);

  return $form;
}

/**
 * Returns a list of options (form elements).
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $options
 *   Array of currently filled in values.
 *
 * @param $fields
 *   Array of available fields.
 *
 * @param $map
 *   Array of how fields are mapped.
 *
 * @return
 *   FAPI array. See hook_node_import_options().
 */
function node_import_options($type, $options, $fields, $map) {
  if (!is_array($options)) {
    $options = array();
  }

  $form = (array)module_invoke_all('node_import_options', $type, $options, $fields, $map);

  // Copy from modules/system/system.admin.inc
  $date_short = array('Y-m-d H:i', 'm/d/Y - H:i', 'd/m/Y - H:i', 'Y/m/d - H:i',
     'd.m.Y - H:i', 'm/d/Y - g:ia', 'd/m/Y - g:ia', 'Y/m/d - g:ia',
     'M j Y - H:i', 'j M Y - H:i', 'Y M j - H:i',
     'M j Y - g:ia', 'j M Y - g:ia', 'Y M j - g:ia');
  $date_short_choices = array();
  foreach ($date_short as $f) {
    $date_short_choices[$f] = format_date(time(), 'custom', $f);
  }
  $date_short_choices['custom'] = t('Custom format');
  $timezones = date_timezone_names(TRUE);

  foreach ((array)$fields as $fieldname => $fieldinfo) {
    // Set #node_import-group if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#node_import-group'])) {
      $form[$fieldname]['#node_import-group'] = $fieldinfo['group'];
    }

    // Set #description if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#description'])) {
      if (count($fieldinfo['tips']) > 1) {
        $form[$fieldname]['#description'] = '<ul class="tips">';
        $form[$fieldname]['#description'] .= '<li>' . implode('</li><li>', $fieldinfo['tips']) . '</li>';
        $form[$fieldname]['#description'] .= '</ul>';
      }
      else if (!empty($fieldinfo['tips'])) {
        $form[$fieldname]['#description'] = implode('', $fieldinfo['tips']);
      }
    }

    // Set #weight if not set.
    if (isset($form[$fieldname]) && !isset($form[$fieldname]['#weight'])) {
      $form[$fieldname]['#weight'] = $fieldinfo['weight'];
    }

    $map_count = node_import_field_map_count($fieldname, $map);

    // Add multiple_separator option for fields that can have multiple
    // values AND that is mapped to exactly one file column.
    if ($fieldinfo['has_multiple'] &&
        $map_count == 1 &&
        (!isset($form[$fieldname]) || !isset($form[$fieldname]['multiple_separator']))) {

      if (!isset($form[$fieldname])) {
        $form[$fieldname] = array(
          '#title' => $fieldinfo['title'],
          '#node_import-group' => $fieldinfo['group'],
        );
      }

      $form[$fieldname]['multiple_separator'] = array(
        '#type' => 'textfield',
        '#title' => t('Multiple values are separated by'),
        '#size' => 6,
        '#default_value' => isset($options[$fieldname]['multiple_separator']) ? $options[$fieldname]['multiple_separator'] : $fieldinfo['multiple_separator'],
      );
    }

    // Add hierarchy_separator option for fields that can have
    // hierarchical values AND (that can have multiple values OR
    // that can not have multiple values but is mapped to exactly
    // one file column).
    if ($fieldinfo['has_hierarchy'] &&
        ($fieldinfo['has_multiple'] || $map_count == 1) &&
        $map_count > 0 &&
        (!isset($form[$fieldname]) || !isset($form[$fieldname]['hierarchy_separator']))) {

      if (!isset($form[$fieldname])) {
        $form[$fieldname] = array(
          '#title' => $fieldinfo['title'],
          '#node_import-group' => $fieldinfo['group'],
        );
      }

      $form[$fieldname]['hierarchy_separator'] = array(
        '#type' => 'textfield',
        '#title' => t('Hierarchy is specified by'),
        '#size' => 6,
        '#default_value' => isset($options[$fieldname]['hierarchy_separator']) ? $options[$fieldname]['hierarchy_separator'] : $fieldinfo['hierarchy_separator'],
      );
    }

    // Add hierarchy_reverse option for fields that can have
    // hierarchical values AND that can not have multiple values
    // AND that is mapped to more than one file column.
    if ($fieldinfo['has_hierarchy'] &&
        $map_count > 1 && !$fieldinfo['has_multiple'] &&
        (!isset($form[$fieldname]) || !isset($form[$fieldname]['hierarchy_reverse']))) {

      if (!isset($form[$fieldname])) {
        $form[$fieldname] = array(
          '#title' => $fieldinfo['title'],
          '#node_import-group' => $fieldinfo['group'],
        );
      }

      if ($map_count > 1 && !$fieldinfo['has_multiple']) {
        $form[$fieldname]['hierarchy_reverse'] = array(
          '#type' => 'checkbox',
          '#title' => t('Reverse file columns for hierarchy'),
          '#default_value' => isset($options[$fieldname]['hierarchy_reverse']) ? $options[$fieldname]['hierarchy_reverse'] : $fieldinfo['hierarchy_reverse'],
        );
      }
    }

    // Add a custom date format option for fields that are dates.
    if ($fieldinfo['input_format'] == 'date' && $map_count > 0) {
      if (!isset($form[$fieldname])) {
        $form[$fieldname] = array(
          '#title' => $fieldinfo['title'],
          '#node_import-group' => $fieldinfo['group'],
        );
      }

      $form[$fieldname]['timezone'] = array(
        '#type' => 'select',
        '#title' => t('Timezone'),
        '#default_value' => isset($options[$fieldname]['timezone']) ? $options[$fieldname]['timezone'] : date_default_timezone_name(),
        '#options' => $timezones,
        '#description' => t('Select the default time zone. If in doubt, choose the timezone that is closest to your location which has the same rules for daylight saving time.'),
      );

      $form[$fieldname]['date_format'] = array(
        '#type' => 'select',
        '#title' => t('Date format'),
        '#default_value' => isset($options[$fieldname]['date_format']) ? $options[$fieldname]['date_format'] : variable_get('date_format_short', 'm/d/Y - H::i'),
        '#options' => $date_short_choices,
        '#description' => t('Select the date format for import. If you choose <em>Custom format</em> enter the custom format below.'),
      );
      $form[$fieldname]['date_custom'] = array(
        '#type' => 'textfield',
        '#title' => t('Custom date format'),
        '#attributes' => array('class' => 'custom-format'),
        '#default_value' => isset($options[$fieldname]['date_custom']) ? $options[$fieldname]['date_custom'] : variable_get('date_format_short', 'm/d/Y - H::i'),
        '#description' => t('See the <a href="@url" target="_blank">PHP manual</a> for available options.', array('@url' => 'http://php.net/manual/function.date.php')),
      );
    }

    // Add directory selection for fields that are filepaths.
    if ($fieldinfo['input_format'] == 'filepath' && $map_count > 0) {
      if (!isset($form[$fieldname])) {
        $form[$fieldname] = array(
          '#title' => $fieldinfo['title'],
          '#node_import-group' => $fieldinfo['group'],
        );
      }

      $form[$fieldname][] = array(
        '#value' => t('You need to FTP the files you reference in this field manually to the correct location (%path) before doing the import.', array('%path' => file_create_path(isset($fieldinfo['to_directory']) ? $fieldinfo['to_directory'] : ''))),
      );
      $form[$fieldname]['manually_moved'] = array(
        '#type' => 'hidden',
        '#value' => TRUE,
      );

/*
TODO: we disable this until I get the time to figure out why files are not
moved to the correct location if you do not move them manually.

//TODO: as the directory must be relative... we could let the user choose from a select box
//TODO: we might use the sample data to find the correct directory
      $form[$fieldname]['from_directory'] = array(
        '#type' => 'textfield',
        '#title' => t('Copy from'),
        '#field_prefix' => node_import_directory() . '/',
        '#field_suffix' => '/',
        '#default_value' => isset($options[$fieldname]['from_directory']) ? $options[$fieldname]['from_directory'] : '',
        '#description' => t('Fill in the directory which contains the files.'),
      );

      $form[$fieldname]['manually_moved'] = array(
        '#type' => 'checkbox',
        '#title' => t('Files have been manually moved'),
        '#default_value' => isset($options[$fieldname]['manually_moved']) ? $options[$fieldname]['manually_moved'] : 0,
        '#description' => t('Check this box if you have already moved the files to the correct location on your server (%path).', array('%path' => file_create_path(isset($fieldinfo['to_directory']) ? $fieldinfo['to_directory'] : ''))),
      );
*/

/* TODO
      $form[$fieldname]['delete_on_success'] = array(
        '#type' => 'checkbox',
        '#title' => t('Delete files after import'),
        '#default_value' => isset($options[$fieldname]['delete_on_success']) ? $options[$fieldname]['delete_on_success'] : 1,
        '#description' => t('Check this box if you want to delete the files from the server after a succesful import.'),
      );
*/
    }
  }

  drupal_alter('node_import_options', $form, $type, $options, $fields, $map);

  return $form;
}

/**
 * Create an array of values to submit to the form.
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $data
 *   Array of data from the file as ($col_index => $value).
 *
 * @param $map
 *   Array of how the data maps to fields.
 *
 * @param $defaults
 *   Array of default values.
 *
 * @param $options
 *   Array of options.
 *
 * @param $fields
 *   Array of available fields.
 *
 * @param $preview
 *   Boolean. If TRUE a preview will be created. If FALSE construct the
 *   values for real.
 *
 * @return
 *   Array of values to submit to a form. See hook_node_import_values().
 */
function node_import_values($type, $data, $map, $defaults, $options, $fields, $preview) {
  $values = module_invoke_all('node_import_values', $type, $defaults, $options, $fields, $preview);

  foreach ($fields as $fieldname => $fieldinfo) {
    $map[$fieldname] = isset($map[$fieldname]) ? $map[$fieldname] : '';
    $map[$fieldname] = is_array($map[$fieldname]) ? $map[$fieldname] : array($map[$fieldname]);
    $map_count = node_import_field_map_count($fieldname, $map);

    $mseparator = isset($options[$fieldname]['multiple_separator']) ? $options[$fieldname]['multiple_separator'] : $fieldinfo['multiple_separator'];
    $hseparator = isset($options[$fieldname]['hierarchy_separator']) ? $options[$fieldname]['hierarchy_separator'] : $fieldinfo['hierarchy_separator'];
    $hreverse = isset($options[$fieldname]['hierarchy_reverse']) ? $options[$fieldname]['hierarchy_reverse'] : $fieldinfo['hierarchy_reverse'];

    // Merge default value for each field.
    if (isset($defaults[$fieldname])) {
      if ($fieldinfo['is_checkboxes']) {
        $values[$fieldname] = array_keys(array_filter($defaults[$fieldname]));
      }
      else {
        $values[$fieldname] = $defaults[$fieldname];
      }
    }
    else if (isset($fieldinfo['default_value'])) {
      $values[$fieldname] = $fieldinfo['default_value'];
    }

    // Map the data ONLY IF the data to map is not empty.
    if ($fieldinfo['has_multiple']) {
      if ($map_count > 0) {
        $fieldvalues = array();

        foreach ($map[$fieldname] as $col) {
          $value = isset($data[$col]) ? (string)$data[$col] : '';
          if ($map_count == 1 && strlen($mseparator) > 0) {
            $fieldvalues = strlen($value) > 0 ? array_map('trim', explode($mseparator, $value)) : array();
            break;
          }
          $fieldvalues[] = $value;
        }

        if (!$fieldinfo['allow_empty']) {
          $fieldvalues = array_filter($fieldvalues, 'drupal_strlen');
        }

        if ($fieldinfo['has_hierarchy'] && strlen($hseparator) > 0) {
          foreach ($fieldvalues as $i => $value) {
            $fieldvalues[$i] = strlen($value) > 0 ? array_map('trim', explode($hseparator, $value)) : array($value);
          }
        }

        if (empty($fieldvalues) && isset($values[$fieldname])) {
          $values[$fieldname] = $values[$fieldname];
        }
        else {
          $values[$fieldname] = $fieldvalues;
        }
      }
    }
    else {
      if ($map_count > 0 && $fieldinfo['has_hierarchy']) {
        $fieldvalues = array();

        foreach ($map[$fieldname] as $col) {
          $value = isset($data[$col]) ? (string)$data[$col] : '';
          if ($map_count == 1 && strlen($hseparator) > 0) {
            $fieldvalues = drupal_strlen($value) > 0 ? array_map('trim', explode($hseparator, $value)) : array();
            break;
          }
          $fieldvalues[] = $value;
        }

        if (!$fieldinfo['allow_empty']) {
          $fieldvalues = array_filter($fieldvalues, 'drupal_strlen');
        }

        if ($hreverse) {
          $fieldvalues = array_reverse($fieldvalues);
        }

        $values[$fieldname] = empty($fieldvalues) ? $values[$fieldname] : $fieldvalues;
      }
      else if ($map_count == 1) {
        foreach ($map[$fieldname] as $col) {
          $value = isset($data[$col]) ? (string)$data[$col] : '';
        }

        $values[$fieldname] = drupal_strlen($value) > 0 ? $value : (isset($values[$fieldname]) ? $values[$fieldname] : '');
      }

      $values[$fieldname] = array(isset($values[$fieldname]) ? $values[$fieldname] : '');
    }

    // Preprocess the data as long as the value is not empty and it
    // validates.
    $values[$fieldname] = isset($values[$fieldname]) ? $values[$fieldname] : array('');
    foreach ((array)$values[$fieldname] as $i => $value) {
      foreach ($fieldinfo['preprocess'] as $function) {
        if ((is_array($values[$fieldname][$i]) && !empty($values[$fieldname][$i])) // has_multiple values are arrays
            || (is_string($values[$fieldname][$i]) && drupal_strlen($values[$fieldname][$i]) > 0)) {
          $options[$fieldname] = isset($options[$fieldname]) ? $options[$fieldname] : array();

          $value = $values[$fieldname][$i];
          $return = $function($value, $fieldinfo, $options[$fieldname], $preview);
          $values[$fieldname][$i] = $value;

          if ($return === FALSE) {
            // No longer validates - stop.
            $values[$fieldname][$i] = '';
            continue 2;
          }
          else if ($return === TRUE) {
            // Final preprocess reported - stop.
            continue 2;
          }
        }
      }
    }

    // If empty values are not allowed, filter them out.
    if (!$fieldinfo['allow_empty']) {
      $values[$fieldname] = array_filter((array)$values[$fieldname], 'drupal_strlen');
    }

    // Handle files specially. The preprocess function only returns
    // the path - we need to make sure we save the file now into the
    // db and set the value to the fid. We need to do this here instead
    // of in the preprocess function because we need $values['uid'].
    if ($fieldinfo['input_format'] == 'filepath') {
      foreach ($values[$fieldname] as $i => $value) {
        if (drupal_strlen($value) > 0) {
          $result = db_result(db_query("SELECT fid FROM {files} WHERE filepath = '%s'", $value));
          if ($result) {
            $values[$fieldname][$i] = $result;
          }
          else {
            // TODO: don't we need more stuff - eg run the validators?
            global $user;
            $file = new stdClass();
            $file->uid = isset($values['uid']) ? $values['uid'] : $user->uid;
            $file->filename = basename($value);
            $file->filepath = $value;
            $file->filesize = filesize($value);
            $file->filemime = file_get_mimetype($file->filename);
            $file->status = FILE_STATUS_TEMPORARY;
            $file->timestamp = time();
            drupal_write_record('files', $file);
            $values[$fieldname][$i] = $file->fid;
          }
        }
      }
    }

    // If only a single value is allowed, get the first one.
    if (!$fieldinfo['has_multiple']) {
      $values[$fieldname] = array_shift($values[$fieldname]);
    }

    // Convert checkboxes fields to a format FAPI understands.
    if ($fieldinfo['is_checkboxes'] && !empty($values[$fieldname])) {
      // Only in PHP > 5.2: $values[$fieldname] = array_fill_keys($values[$fieldname], 1);
      $values[$fieldname] = array_combine($values[$fieldname], array_fill(0, count($values[$fieldname]), 1));
    }
  }

  drupal_alter('node_import_values', $values, $type, $defaults, $options, $fields, $preview);

  return $values;
}

/**
 * Get a list of options for different stuff presented to the
 * user in the wizard form such as 'record separators', ...
 *
 * @param $op
 *   String. See hook_node_import_format_options().
 *
 * @param $reset
 *   Boolean. If TRUE, the internal cache is reset.
 *
 * @return
 *   Array.
 */
function node_import_format_options($op, $reset = FALSE) {
  static $options;

  if (!isset($options) || ($reset && !isset($op))) {
    $options = array();
  }

  if (isset($op) && (!isset($options[$op]) || $reset)) {
    $options[$op] = module_invoke_all('node_import_format_options', $op);
    drupal_alter('node_import_format_options', $options[$op], $op);
  }

  return isset($op) ? $options[$op] : array();
}

/**
 * @}
 */

/**
 * Import a number of rows from all available tasks. Should only be called
 * from within hook_cron() or from a JS callback as this function may take
 * a long time.
 *
 * The function ends when $count $units have been finished. For example
 * @code
 * node_import_do_all_tasks('all');
 * node_import_do_all_tasks('rows', 10);
 * node_import_do_all_tasks('bytes', 4096);
 * node_import_do_all_tasks('ms', 1000);
 * @endcode
 *
 * @param $unit
 *   String. Either 'rows', 'bytes', 'ms' (milliseconds) or 'all'.
 *   Defaults to 'all'.
 *
 * @param $count
 *   Integer. Number of $units to do. Defaults to 0 (in which case
 *   exactly one row will be imported if $unit != 'all').
 *
 * @return
 *   Nothing.
 */
function node_import_do_all_tasks($unit = 'all', $count = 0) {
  global $node_import_can_continue;

  $byte_count = 0;
  $row_count = 0;
  timer_start('node_import:do_all_tasks');

  foreach (node_import_list_tasks(TRUE) as $taskid => $task) {
    $bytes = $task['file_offset'];
    $rows = $task['row_done'] + $task['row_error'];

    node_import_do_task($task, $unit, $count);

    $byte_count += $task['file_offset'] - $bytes;
    $row_count += $task['row_done'] + $task['row_error'] - $rows;

    if ($node_import_can_continue && ($unit == 'all'
        || ($unit == 'bytes' && $byte_count < $count)
        || ($unit == 'rows' && $row_count < $count)
        || ($unit == 'ms' && timer_read('node_import:do_all_tasks') < $count))) {
      continue;
    }

    break;
  }

  timer_stop('node_import:do_all_tasks');
}

/**
 * Import a number of rows from the specified task. Should only be called
 * from within hook_cron() or from a JS callback as this function may take
 * a long time.
 *
 * The function ends when $count $units have been finished. For example
 * @code
 * node_import_do_task($task, 'all');
 * node_import_do_task($task, 'rows', 10);
 * node_import_do_task($task, 'bytes', 4096);
 * node_import_do_task($task, 'ms', 1000);
 * @endcode
 *
 * @param $task
 *   Array. The task to continue. Note that this is passed by reference!
 *   So you can check the status of the task after running this function
 *   without having to query the database.
 *
 * @param $unit
 *   String. Either 'rows', 'bytes', 'ms' (milliseconds) or 'all'.
 *   Defaults to 'all'.
 *
 * @param $count
 *   Integer. Number of $units to do. Defaults to 0 (in which case
 *   exactly one row will be imported if $unit != 'all').
 *
 * @return
 *   The status of each imported row (error or not) is stored in the
 *   database. @see node_import_constants.
 */
function node_import_do_task(&$task, $unit = 'all', $count = 0) {
  global $node_import_can_continue;
  $node_import_can_continue = TRUE;

  if ($task['status'] != NODE_IMPORT_STATUS_DONE && node_import_lock_acquire()) {
    global $user;
    $backup_user = $user;
    if ($task['uid'] != $user->uid) {
      session_save_session(FALSE);
      $user = user_load(array('uid' => $task['uid']));
    }

    $taskid = $task['taskid'];

    $file_offset = 0;
    $byte_count = 0;
    $row_count = 0;
    timer_start('node_import:do_task:'. $taskid);

    $data = array();
    switch ($task['status']) {
      case NODE_IMPORT_STATUS_PENDING:
        if ($task['file_offset'] == 0 && $task['has_headers']) {
          list($file_offset, $data) = node_import_read_from_file($task['file']->filepath, $file_offset, $task['file_options']);
        }
        else {
          $file_offset = $task['file_offset'];
        }
        break;

      case NODE_IMPORT_STATUS_ERROR:
        $task['status'] = NODE_IMPORT_STATUS_DONE;
        $file_offset = $task['file']->filesize; //TODO
        break;
    }

    module_invoke_all('node_import_task', $task, 'continue');

    while ($task['status'] != NODE_IMPORT_STATUS_DONE) {
      list($new_offset, $data) = node_import_read_from_file($task['file']->filepath, $file_offset, $task['file_options']);

      if (is_array($data)) {
        switch ($task['status']) {
          case NODE_IMPORT_STATUS_PENDING:
            db_query("DELETE FROM {node_import_status} WHERE taskid = %d AND file_offset = %d", $taskid, $file_offset);
            db_query("INSERT INTO {node_import_status} (taskid, file_offset, errors) VALUES (%d, %d, '%s')", $taskid, $file_offset, serialize(array()));
            break;

          case NODE_IMPORT_STATUS_ERROR:
            db_query("UPDATE {node_import_status} SET errors = '%s', status = %d WHERE taskid = %d AND file_offset = %d", serialize(array()), NODE_IMPORT_STATUS_PENDING, $taskid, $file_offset);
            break;
        }

        db_query("UPDATE {node_import_tasks} SET file_offset = %d, changed = %d WHERE taskid = %d", $new_offset, time(), $taskid);
        $task['file_offset'] = $new_offset;

        $errors = node_import_create($task['type'], $data, $task['map'], $task['defaults'], $task['options'], FALSE);

        if (is_array($errors)) {
          db_query("UPDATE {node_import_status} SET status = %d, errors = '%s' WHERE taskid = %d AND file_offset = %d", NODE_IMPORT_STATUS_ERROR, serialize($errors), $taskid, $file_offset);
          db_query("UPDATE {node_import_tasks} SET row_error = row_error + 1 WHERE taskid = %d", $taskid);
          $task['row_error']++;
        }
        else {
          db_query("UPDATE {node_import_status} SET status = %d, objid = %d WHERE taskid = %d AND file_offset = %d", NODE_IMPORT_STATUS_DONE, $errors, $taskid, $file_offset);
          db_query("UPDATE {node_import_tasks} SET row_done = row_done + 1 WHERE taskid = %d", $taskid);
          $task['row_done']++;
        }

        $byte_count += $new_offset - $file_offset;
        $row_count++;
      }
      else {
        db_query("UPDATE {node_import_tasks} SET status = %d, file_offset = %d WHERE taskid = %d", NODE_IMPORT_STATUS_DONE, $task['file']->filesize, $taskid);
        $task['status'] = NODE_IMPORT_STATUS_DONE;
        $task['file_offset'] = $task['file']->filesize;
      }

      switch ($task['status']) {
        case NODE_IMPORT_STATUS_PENDING:
          $file_offset = $new_offset;
          break;

        case NODE_IMPORT_STATUS_ERROR:
          $file_offset = $task['file']->filesize; //TODO
          break;
      }

      if ($node_import_can_continue && ($unit == 'all'
          || ($unit == 'bytes' && $byte_count < $count)
          || ($unit == 'rows' && $row_count < $count)
          || ($unit == 'ms' && timer_read('node_import:do_task:'. $taskid) < $count))) {
        continue;
      }

      break;
    }

    module_invoke_all('node_import_task', $task, 'pause');

    // Cleanup before exit.
    $user = $backup_user;
    session_save_session(TRUE);
    timer_stop('node_import:do_task:'. $taskid);
    node_import_lock_release();
  }
}

/**
 * JS callback to continue the specified task and returns the status
 * of it. This function will take at most one second.
 *
 * @param $task
 *   Full loaded $task.
 *
 * @return
 *   JSON.
 */
function node_import_js($task) {
  node_import_do_task($task, 'ms', 1000);
  echo drupal_json(array(
    'status' => 1,
    'message' => format_plural($task['row_done'], t('1 row imported'), t('@count rows imported')) .'<br />'.
                 format_plural($task['row_error'], t('1 row with errors'), t('@count rows with errors')),
    'percentage' => $task['status'] == NODE_IMPORT_STATUS_DONE ? 100 : round(floor(100.0 * $task['file_offset'] / $task['file']->filesize), 0),
  ));
  exit();
}

/**
 * Create a new object of specified $type.
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $data
 *   Array of data from the file as ($col_index => $value).
 *
 * @param $map
 *   Array of how the data maps to fields.
 *
 * @param $defaults
 *   Array of default values.
 *
 * @param $options
 *   Array of options.
 *
 * @param $preview
 *   Boolean.
 *
 * @return
 *   The return value is: if $preview is TRUE, a string with the preview
 *   of the object is returned. If $preview is FALSE, an unique identifier
 *   is returned or an array with errors if failed.
 */
function node_import_create($type, $data, $map, $defaults, $options, $preview) {
  $output = $preview ? '' : array();

  // Reset execution time. Note that this only works when SAFE_MODE is OFF but
  // it has no side-effects if SAFE_MODE is ON. See
  // http://php.net/manual/en/function.set-time-limit.php
  set_time_limit(variable_get('node_import:set_time_limit', 60));

  $types = node_import_types();
  $fields = node_import_fields($type);

  // We need to clean out the form errors before submitting.
  form_set_error(NULL, '', TRUE);

  // Create a list of values to submit.
  $values = node_import_values($type, $data, $map, $defaults, $options, $fields, $preview);

  // Submit it for preview or creation.
  if (function_exists($function = $types[$type]['create'])) {
    $output = $function($type, $values, $preview);
  }
  module_invoke_all('node_import_postprocess', $type, $values, $options, $preview);

  // Check for errors and clear them again.
  if ($preview) {
    $output = theme('status_messages') . $output;
  }
  if (($errors = form_get_errors())) {
    if ($preview) {
      $output .= '<pre>values = '. print_r($values, TRUE) .'</pre>'; //TODO: show data instead?
    }
    else {
      $output = $errors;
    }
  }
  form_set_error(NULL, '', TRUE);
  drupal_get_messages(NULL, TRUE); // Otherwise they are still showed.

  return $output;
}

/**
 * @defgroup node_import_tasks Node import tasks
 * @{
 */

/**
 * Create a new import task.
 *
 * @param $values
 *   Array of filled in values.
 *
 * @return
 *   Integer. Unique identifier for the task or FALSE if the task could
 *   not be saved to the database.
 */
function node_import_save_task($values) {
  global $user;

  if (!isset($values['uid'])) {
    $values['uid'] = $user->uid;
  }
  if (!isset($values['created'])) {
    $values['created'] = time();
  }
  if (!isset($values['changed'])) {
    $values['changed'] = time();
  }

  if (drupal_write_record('node_import_tasks', $values) === SAVED_NEW) {
    module_invoke_all('node_import_task', $values, 'insert');
    return $values['taskid'];
  }
  return FALSE;
}

/**
 * Get a list of available tasks.
 *
 * @param $all
 *   Boolean. If TRUE, all tasks are returned. If FALSE, only the tasks
 *   the current user has access to.
 *
 * @return
 *   Array of tasks.
 */
function node_import_list_tasks($all = FALSE) {
  global $user;
  $tasks = array();

  if ($all || user_access('administer imports')) {
    $result = db_query("SELECT * FROM {node_import_tasks} ORDER BY created ASC");
  }
  else {
    $result = db_query("SELECT * FROM {node_import_tasks} WHERE uid = %d ORDER BY created ASC", $user->uid);
  }

  while (($task = db_fetch_array($result))) {
    foreach (array('file_options', 'headers', 'map', 'defaults', 'options') as $key) {
      $task[$key] = isset($task[$key]) ? unserialize($task[$key]) : array();
    }

    $task['file'] = db_fetch_object(db_query("SELECT * FROM {files} WHERE fid = %d", $task['fid']));
    $tasks[$task['taskid']] = $task;
  }

  return $tasks;
}

/**
 * Delete an import task.
 *
 * @param $taskid
 *   Unique identifier.
 *
 * @return
 *   Nothing.
 */
function node_import_delete_task($taskid) {
  module_invoke_all('node_import_task', $taskid, 'delete');
  db_query("DELETE FROM {node_import_tasks} WHERE taskid = %d", $taskid);
  db_query("DELETE FROM {node_import_status} WHERE taskid = %d", $taskid);
}

/**
 * @}
 */

/**
 * @defgroup node_import_preprocess Node import preprocess functions
 * @{
 */

/**
 * Check if the value is a valid boolean (1, 0, true, false, yes, no, on, off).
 *
 * Uses: nothing.
 */
function node_import_check_boolean(&$value, $field, $options, $preview) {
  static $trues;
  static $falses;

  if (!isset($trues)) {
    $trues = array(
      '1',
      'on', drupal_strtolower(t('On')),
      'yes', drupal_strtolower(t('Yes')),
      'true', drupal_strtolower(t('True')),
    );
    $falses = array(
      '0',
      'off', drupal_strtolower(t('Off')),
      'no', drupal_strtolower(t('No')),
      'false', drupal_strtolower(t('False')),
    );
  }

  if (in_array(drupal_strtolower($value), $trues, TRUE)) {
    $value = '1';
    return TRUE;
  }
  else if (in_array(drupal_strtolower($value), $falses, TRUE)) {
    $value = '0';
    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not a boolean).', array('%value' => $value, '%name' => $field['title'])));
  return FALSE;
}

/**
 * Check if the value is a valid date.
 *
 * Uses: $field['output_format'] (output format type - defaults to DATE_UNIX).
 * Uses: $options['date_format'], $options['date_custom'] and $options['timezone'] (default to date_default_timezone_name()).
 */
function node_import_check_date(&$value, $field, $options, $preview) {
  $timezone = isset($options['timezone']) ? $options['timezone'] : date_default_timezone_name();
  $input_format = $options['date_format'] == 'custom' ? $options['date_custom'] : $options['date_format'];
  $output_format = isset($field['output_format']) ? $field['output_format'] : DATE_UNIX;

  if (date_is_valid($value, DATE_ISO)) {
    $value = date_convert($value, DATE_ISO, $output_format, $timezone);
    return TRUE;
  }

  module_load_include('inc', 'date_api', 'date_api_elements');

  if (($date = date_convert_from_custom($value, $input_format))) {
    // It is useless to check for date_is_valid() as it is a DATE_DATETIME already.
    $value = date_convert($date, DATE_DATETIME, $output_format, $timezone);
    return TRUE;
  }

  if (date_is_valid($value, DATE_UNIX)) {
    $value = date_convert($value, DATE_UNIX, $output_format, $timezone);
    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not a date in %date format).', array('%value' => $value, '%name' => $field['title'], '%date' => format_date(time(), 'custom', $input_format))));
  return FALSE;
}

/**
 * Check if the value is a valid email address.
 *
 * Uses: nothing.
 */
function node_import_check_email(&$value, $field, $options, $preview) {
  if (!valid_email_address($value)) {
    node_import_input_error(t('Input error: %value is not a valid e-mail address.', array('%value' => $value)));
    return FALSE;
  }
  return TRUE;
}

/**
 * Check if the value points to a valid filepath.
 *
 * Uses: $field['to_directory'], $options['from_directory'], $options['manually_moved'].
 */
function node_import_check_filepath(&$value, $field, $options, $preview) {
  // No need to check empty values.
  if (drupal_strlen($value) == 0) {
    return TRUE;
  }

  // Where should be file be located?
  $find_in = isset($options['from_directory']) ? $options['from_directory'] : '';
  $find_in = node_import_directory() . (strlen($find_in) > 0 ? '/'. $find_in : '');
  if (isset($options['manually_moved']) && $options['manually_moved']) {
    $find_in = isset($field['to_directory']) ? $field['to_directory'] : '';
  }
  $find_in = file_create_path($find_in);

  // Check if the file exists.
  $filepath = $find_in .'/'. $value;
  if (file_check_location($filepath, $find_in) && file_exists($filepath)) {
    $value = $filepath;
    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not a file in %path).', array('%value' => $value, '%name' => $field['title'], '%path' => $find_in)));
  $value = '';
  return FALSE;
}

/**
 * Check if the value is a valid node reference (by nid or title).
 *
 * Uses: $field['output_format']. Either 'nid' (default) or 'title'.
 */
function node_import_check_node_reference(&$value, $field, $options, $preview) {
  if (($nid = node_import_get_object('node', $value)) !== NULL ||
      ($nid = db_result(db_query("SELECT nid FROM {node} WHERE nid = %d OR LOWER(title) = '%s' LIMIT 1", is_numeric($value) && intval($value) > 0 ? $value : -1, drupal_strtolower($value))))) {

    node_import_set_object('node', $value, $nid);
    $value = $nid;

    $field['output_format'] = isset($field['output_format']) ? $field['output_format'] : 'nid';
    switch ($field['output_format']) {
      case 'title':
        if (($title = node_import_get_object('node:title', $nid)) ||
            ($title = db_result(db_query("SELECT title FROM {node} WHERE nid = %d LIMIT 1", $nid)))) {
          $value = $title;
          node_import_set_object('node:title', $nid, $title);
        }
        break;

      case 'nid':
      default:
        break;
    }

    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not a node reference).', array('%value' => $value, '%name' => $field['title'])));
  return FALSE;
}

/**
 * Check if the value is a valid user (by uid, name or email).
 *
 * Uses: $field['output_format']. Either 'uid' (default), 'name' or 'email'.
 */
function node_import_check_user_reference(&$value, $field, $options, $preview) {
  if (($uid = node_import_get_object('user', $value)) !== NULL ||
      ($uid = db_result(db_query("SELECT uid FROM {users} WHERE uid = %d OR LOWER(name) = '%s' OR LOWER(mail) = '%s' LIMIT 1", is_numeric($value) && intval($value) > 0 ? $value : -1, drupal_strtolower($value), drupal_strtolower($value))))) {

    node_import_set_object('user', $value, $uid);
    $value = $uid;

    $field['output_format'] = isset($field['output_format']) ? $field['output_format'] : 'uid';
    switch ($field['output_format']) {
      case 'name':
        if (($name = node_import_get_object('user:name', $uid)) ||
            ($name = db_result(db_query("SELECT name FROM {users} WHERE uid = %d LIMIT 1", $uid)))) {
          $value = $name;
          node_import_set_object('user:name', $uid, $name);
        }
        break;

      case 'email':
        if (($email = node_import_get_object('user:email', $uid)) ||
            ($email = db_result(db_query("SELECT mail FROM {users} WHERE uid = %d LIMIT 1", $uid)))) {
          $value = $email;
          node_import_set_object('user:email', $uid, $email);
        }
        break;

      case 'uid':
      default:
        break;
    }

    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not an user).', array('%value' => $value, '%name' => $field['title'])));
  return FALSE;
}

/**
 * Check if the value is in the list of allowed values (by key or value).
 *
 * Uses: $field['allowed_values'].
 */
function node_import_check_values(&$value, $field, $options, $preview) {
  foreach ($field['allowed_values'] as $key => $title) {
    $tmp = drupal_strtolower($value);
    if ($tmp === drupal_strtolower($key) || $tmp === drupal_strtolower($title)) {
      $value = (string)$key;
      return TRUE;
    }
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not in allowed values list).', array('%value' => $value, '%name' => $field['title'])));
  return FALSE;
}

/**
 * Check if the value is a valid weight (integer between -X and X).
 *
 * Uses: $field['delta'].
 */
function node_import_check_weight(&$value, $field, $options, $preview) {
  $weight = isset($field['delta']) ? $field['delta'] : 10;

  if (is_numeric($value) && intval($value) <= $weight && intval($value) >= -$weight) {
    $value = intval($value);
    return TRUE;
  }

  node_import_input_error(t('Input error: %value is not allowed for %name (not a weight).', array('%value' => $value, '%name' => $field['title'])));
  return FALSE;
}

/**
 * @}
 */

/**
 * @defgroup node_import_util Various node import utility functions.
 * @{
 */

/**
 * Store an object-id in the node_import cache.
 *
 * As some string->object-id lookups can be expensive (in db queries) and
 * most of the time the same strings are looked up (eg users), we have a
 * cache of object-ids we already have looked up.
 *
 * This function is used to store an object-id in the cache.
 *
 * @param $type
 *   String. The type of object (eg 'user').
 *
 * @param $value
 *   String or array. The value we haved looked up.
 *
 * @param $oid
 *   Integer. The looked-up object-id. If NULL, the currently stored
 *   object-id is returned without setting the $type/$value to NULL.
 *   If you want to reset (eg make it NULL) a value, use
 *   @code
 *   node_import_set_object($type, $value, NULL, TRUE);
 *   @endcode
 *
 * @param $reset
 *   Boolean. Whether to reset the cache. If $type is NULL, the whole
 *   cache is reset. If $value is not NULL only the cache for that
 *   specific $type/$value is reset.
 *
 * @return
 *   Integer. The looked-up object-id. NULL if not found.
 *
 * @see node_import_get_object().
 */
function node_import_set_object($type, $value, $oid = NULL, $reset = FALSE) {
  static $cache;

  if (!isset($cache)) {
    $cache = array();
  }
  if (isset($type) && !isset($cache[$type])) {
    $cache[$type] = array();
  }

  $stored_value = NULL;
  if (isset($value)) {
    $stored_value = is_array($value) ? implode("\n", array_map('drupal_strtolower', $value)) : drupal_strtolower($value);
  }

  if ($reset) {
    if (isset($type)) {
      if (isset($value)) {
        unset($cache[$type][$stored_value]);
      }
      else {
        $cache[$type] = array();
      }
    }
    else {
      $cache = array();
    }
    return;
  }

  if (isset($oid)) {
    $cache[$type][$stored_value] = $oid;
  }

  return isset($cache[$type][$stored_value]) ? $cache[$type][$stored_value] : NULL;
}

/**
 * Get an object-id from the node_import cache.
 *
 * @param $type
 *   String. The type of object (eg 'user').
 *
 * @param $value
 *   String or array. The value to get the object-id of.
 *
 * @return
 *   NULL if not yet in cache. Otherwise an integer.
 *
 * @see node_import_set_object().
 */
function node_import_get_object($type, $value) {
  return node_import_set_object($type, $value);
}

/**
 * Get a property from each element of an array.
 *
 * @param $array
 *   Array of ($key => $info).
 *
 * @param $property
 *   String.
 *
 * @return
 *   Array of ($key => $info->$property).
 */
function node_import_extract_property($array, $property = 'title') {
  $result = array();
  foreach ((array)$array as $key => $info) {
    if (is_array($info) || is_object($info)) {
      $info = (array)$info;
      $result[$key] = isset($info[$property]) ? $info[$property] : '';
    }
    else {
      $result[$key] = $info;
    }
  }
  return $result;
}

/**
 * Function used by uasort to sort structured arrays by weight.
 */
function node_import_sort($a, $b) {
  $a_group = (is_array($a) && isset($a['group'])) ? $a['group'] : '';
  $b_group = (is_array($b) && isset($b['group'])) ? $b['group'] : '';
  if ($a_group !== $b_group && ($a_group === '' || $b_group === '')) {
    return ($a_group === '') ? -1 : 1;
  }

  $a_weight = (is_array($a) && isset($a['weight'])) ? $a['weight'] : 0;
  $b_weight = (is_array($b) && isset($b['weight'])) ? $b['weight'] : 0;
  if ($a_weight == $b_weight) {
    return 0;
  }

  return ($a_weight < $b_weight) ? -1 : 1;
}

/**
 * Returns the number of columns given $fieldname is mapped
 * to.
 *
 * @param $fieldname
 *   String.
 *
 * @param $map
 *   Array of file column mapping.
 *
 * @return
 *   Integer. Number of file columns the field is mapped to.
 */
function node_import_field_map_count($fieldname, $map) {
  if (!isset($map[$fieldname])) {
    return 0;
  }
  if (!is_array($map[$fieldname])) {
    return strlen($map[$fieldname]) > 0;
  }
  $count = 0;
  foreach ($map[$fieldname] as $col) {
    if ($col !== '') {
      $count++;
    }
  }
  return $count;
}

/**
 * Set an error on a random form element.
 */
function node_import_input_error($message, $args = array()) {
  static $count = 0;
  form_set_error('node_import-'. $count, $message, $args);
  $count++;
}

/**
 * @}
 */

/**
 * @defgroup node_import_files Node import file functions
 * @{
 */

/**
 * Return the full path where files to import are stored.
 *
 * @param $reset
 *   Boolean. Reset internal cache.
 *
 * @return
 *   String. The directory is created if needed.
 */
function node_import_directory($reset = FALSE) {
  static $path;

  if (!isset($path) || $reset) {
    $path = variable_get('node_import:directory', 'imports');

    if (function_exists('token_replace')) {
      global $user;
      $path = token_replace($path, 'user', user_load($user->uid));
    }

    if (strlen($path) > 0) {
      $parts = array_filter(explode('/', $path));
      $path = '';
      while (!empty($parts)) {
        $path .= array_shift($parts) . '/';
        file_check_directory(file_create_path($path), FILE_CREATE_DIRECTORY, 'node_import:directory');
      }
    }

    $path = rtrim($path, '/');
    $path = file_create_path($path);
  }

  return $path;
}

/**
 * Returns a list of available files.
 */
function node_import_list_files($reset = FALSE) {
  global $user;
  static $files;

  if (!isset($files) || $reset) {
    $files = array();
    $path = node_import_directory();

    // If FTP uploads of files is allowed, rescan the directory.
    if (variable_get('node_import:ftp:enabled', 0)) {
      $ftp_user = user_load(array('name' => variable_get('node_import:ftp:user', '')));
      //TODO: use the $validators functionality of file_save_upload() ?
      $extensions = array_map('drupal_strtolower', array_filter(explode(' ', variable_get('node_import:ftp:extensions', 'csv tsv txt'))));
      foreach ($extensions as $extension) {
        $extensions[] = drupal_strtoupper($extension);
      }

      if (!empty($extensions)) {
        $existing_files = array();
        $result = db_query("SELECT filepath FROM {files} WHERE filepath LIKE '%s%%' AND status = %d", $path, FILE_STATUS_PERMANENT);
        while (($file = db_fetch_object($result))) {
          $existing_files[$file->filepath] = TRUE;
        }
        foreach (file_scan_directory($path, '.*\.(('. implode(')|(', $extensions) .'))') as $filename => $file) {
          if (!isset($existing_files[$file->filename])) {
            $record = (object)array(
              'uid' => $ftp_user->uid,
              'filename' => $file->basename,
              'filepath' => $file->filename,
              'filemime' => 'text/plain', //TODO: how to get real MIME?
              'filesize' => filesize($file->filename),
              'status' => FILE_STATUS_PERMANENT,
              'timestamp' => time(),
            );
            drupal_write_record('files', $record);
            drupal_set_message(t('A new file %name was detected in %path.', array('%name' => $record->filename, '%path' => $path)));
          }
        }
      }
    }

    // Users with 'administer imports' permission can see all files.
    //TODO: we should also filter out files that are already in use by another task.
    if (user_access('administer imports')) {
      $result = db_query("SELECT * FROM {files} WHERE filepath LIKE '%s%%' AND status = %d ORDER BY filename, timestamp", $path, FILE_STATUS_PERMANENT);
    }
    else {
      $result = db_query("SELECT * FROM {files} WHERE filepath LIKE '%s%%' AND (uid = %d OR uid = 0) AND status = %d ORDER BY filename, timestamp", $path, $user->uid, FILE_STATUS_PERMANENT);
    }

    // Create the list, keep only the still existing files.
    while (($file = db_fetch_object($result))) {
      if (!file_exists(file_create_path($file->filepath))) {
        drupal_set_message(t('The previously uploaded file %name is no longer present. Removing it from the list of uploaded files.', array('%name' => $file->filepath)));
        file_set_status($file, FILE_STATUS_TEMPORARY);
      }
      else {
        $files[$file->fid] = $file;
      }
    }
  }

  return $files;
}

/**
 * Return an autodetected mapping for given headers and content type.
 *
 * The automapping is done by checking the column titles in the file,
 * whether they match with the field name or field title.
 *
 * @param $type
 *   String. The node_import type.
 *
 * @param $headers
 *   Array of column titles in the file.
 *
 * @return
 *   Array of mapping.
 */
function node_import_automap($type, $headers) {
  global $user;

  if (user_access('administer imports')) {
    $result = db_query("SELECT map FROM {node_import_tasks} WHERE type = '%s' AND LOWER(headers) = '%s' ORDER BY created DESC LIMIT 1", $type, strtolower(serialize($headers)));
  }
  else {
    $result = db_query("SELECT map FROM {node_import_tasks} WHERE type = '%s' AND LOWER(headers) = '%s' AND uid = %d ORDER BY created DESC LIMIT 1", $type, strtolower(serialize($headers)), $user->uid);
  }
  if (($map = db_result($result))) {
    return unserialize($map);
  }

  $map = array();
  $headers = array_map('drupal_strtolower', $headers);

  foreach (node_import_fields($type) as $fieldname => $fieldinfo) {
    if ($fieldinfo['is_mappable']) {
      $map[$fieldname] = '';
      if (($col = array_search(drupal_strtolower($fieldname), $headers)) !== FALSE
          || ($col = array_search(drupal_strtolower($fieldinfo['title']), $headers)) !== FALSE) {
        $map[$fieldname] = $col;
      }
    }
  }

  return $map;
}

/**
 * Return an autodetected file format and file options for given
 * file.
 *
 * This checks which file format options cause the file to contain
 * the most fields. The file options tested are the ones used
 * by the user (last 5) and the builtin ones.
 *
 * @param $filepath
 *   String. Path to file.
 *
 * @return
 *   Array of file options.
 */
function node_import_autodetect($filepath) {
  global $user;

  if (user_access('administer imports')) {
    $result = db_query("SELECT DISTINCT file_options FROM {node_import_tasks} ORDER BY created DESC LIMIT 5");
  }
  else {
    $result = db_query("SELECT DISTINCT file_options FROM {node_import_tasks} WHERE uid = %d ORDER BY created DESC LIMIT 5");
  }

  $file_formats = node_import_format_options('file formats');
  while (($used_format = db_fetch_array($result))) {
    $file_formats[] = unserialize($used_format['file_options']);
  }

  $format = 'csv';
  $max_cols = 0;

  foreach ($file_formats as $file_format => $file_options) {
    $file_offset = 0;

    $num_rows = variable_get('node_import:detect:row_count', 5);
    $num_cols = 0;
    $fields = array();

    while ($num_rows > 0 && is_array($fields)) {
      list($new_offset, $fields) = node_import_read_from_file($filepath, $file_offset, $file_options);
      $num_rows--;

      if (is_array($fields) && count($fields) > $num_cols) {
        $num_cols = count($fields);
      }
    }

    if ($num_cols > $max_cols) {
      $format = $file_format;
      $max_cols = $num_cols;
    }
  }

  $file_options = $file_formats[$format];
  $file_options['file_format'] = is_int($format) ? '' : $format;

  return $file_options;
}

/**
 * Returns one record from the file starting at file offset using
 * the supplied file options.
 *
 * @param $filepath
 *   String. Path to file.
 *
 * @param $file_offset
 *   Integer. Starting point of record.
 *
 * @param $file_options
 *   Array with 'record separator', 'field separator', 'text delimiter'
 *   and 'escape character'. If not set, the options default to the
 *   CSV options ("\n", ',', '"', '"').
 *
 * @return
 *   Array ($file_offset, $record). The $file_offset is the start offset
 *   of the next record. The $record is an array of fields (strings).
 *
 *   On error or when the end of the file has been reached we return
 *   FALSE.
 */
function node_import_read_from_file($filepath, $file_offset, $file_options) {
  // Open file and set to file offset.
  if (($fp = fopen($filepath, 'r')) === FALSE) {
    return FALSE;
  }
  if (fseek($fp, $file_offset)) {
    return FALSE;
  }

  // File options.
  _node_import_sanitize_file_options($file_options);

  $rs = $file_options['record separator'];
  $fs = $file_options['field separator'];
  $td = $file_options['text delimiter'];
  $ec = $file_options['escape character'];

  // The current record is stored in the $fields array. The $new_offset
  // contains the file position of the end of the returned record. Note
  // that if $new_offset == $file_offset we have reached the end of the
  // file.
  $fields = array();
  $new_offset = $file_offset;
  $start = 0;

  // We read $length bytes at a time in the $buffer.
  $length = variable_get('node_import:fgets:length', 1024);
  $buffer = fgets($fp, $length);

  // A field can be enclosed in text delimiters or not. If this variable is
  // TRUE, we need to parse until we find the next unescaped text delimiter.
  // If FALSE, the field value was not enclosed.
  $enclosed = FALSE;

  // Read until the EOF or until end of record.
  while (!feof($fp) || $start < strlen($buffer)) {
    if (!$enclosed) {
      // Find the next record separator, field separator and text delimiter.
      if ($rs === "\n") {
        $pos_rs = strpos($buffer, "\n", $start);
        $pos_rs = $pos_rs !== FALSE ? $pos_rs : strpos($buffer, "\r", $start);
      }
      else {
        $pos_rs = strpos($buffer, $rs, $start);
      }
      $pos_fs = strpos($buffer, $fs, $start);
      $pos_td = strlen($td) > 0 ? strpos($buffer, $td, $start) : FALSE;

      // Check for begin of text delimited field.
      if ($pos_td !== FALSE && ($pos_rs === FALSE || $pos_td <= $pos_rs) && ($pos_fs === FALSE || $pos_td <= $pos_fs)) {
        $enclosed = TRUE;
        $buffer = substr($buffer, 0, $pos_td) . substr($buffer, $pos_td + strlen($td));
        $new_offset += strlen($td);
        $start = $pos_td;
        continue;
      }

      // Check for end of record.
      if ($pos_rs !== FALSE && ($pos_fs === FALSE || $pos_rs <= $pos_fs)) {
        if ($pos_rs >= 0) {
          $fields[] = substr($buffer, 0, $pos_rs);
          $buffer = '';
          $new_offset += $pos_rs;
          $start = 0;
        }
        else if (empty($fields)) {
          $buffer = substr($buffer, strlen($rs), strlen($buffer) - strlen($rs));
          $new_offset += strlen($rs);
          $start = 0;
          continue;
        }
        $new_offset += strlen($rs);
        break;
      }

      // Check for end of field.
      if ($pos_fs !== FALSE) {
        $fields[] = substr($buffer, 0, $pos_fs);
        $buffer = substr($buffer, $pos_fs + strlen($fs));
        $new_offset += $pos_fs + strlen($fs);
        $start = 0;
        continue;
      }
    }
    else {
      // Find the next text delimiter and escaped text delimiter.
      $pos_td = strpos($buffer, $td, $start);
      $pos_ec = strlen($ec) > 0 ? strpos($buffer, $ec . $td, $start) : FALSE;

      // Check for end of text delimited field.
      if ($pos_td !== FALSE && ($pos_ec === FALSE || $pos_td <= ($pos_ec - strlen($td)))) {
        $enclosed = FALSE;
        $buffer = substr($buffer, 0, $pos_td) . substr($buffer, $pos_td + strlen($td));
        $new_offset += strlen($td);
        $start = $pos_td;
        continue;
      }

      // Check for escaped text delimiter.
      if ($pos_ec !== FALSE) {
        $buffer = substr($buffer, 0, $pos_ec) . substr($buffer, $pos_ec + strlen($ec));
        $new_offset += strlen($ec);
        $start = $pos_ec + strlen($td);
        continue;
      }
    }

    // Nothing found... read more data.
    $start = strlen($buffer);
    $buffer .= fgets($fp, $length);
  }

  // Check if we need to add the last field.
  if (feof($fp) && strlen($buffer) > 0) {
    $fields[] = $buffer;
    $new_offset += strlen($buffer);
  }

  // Remove extra white space.
  $fields = array_map('trim', $fields);

  // Check whether the whole row is empty.
  $empty_row = TRUE;
  foreach ($fields as $field) {
    if (strlen($field) > 0) {
      $empty_row = FALSE;
      break;
    }
  }
  if ($empty_row && !feof($fp) && !empty($fields)) {
    return node_import_read_from_file($filepath, $new_offset, $file_options);
  }

  // Cleanup and return.
  $result = (!feof($fp) || !empty($fields)) ? array($new_offset, $fields) : FALSE;
  unset($buffer);
  fclose($fp);

  return $result;
}

function _node_import_sanitize_file_options(&$file_options) {
  // When file_format is not '', we need to load the options from the
  // predefined file_format.
  if (isset($file_options['file_format']) && $file_options['file_format'] !== '') {
    $file_formats = node_import_format_options('file formats');
    $file_options = array_merge($file_options, $file_formats[$file_options['file_format']]);
  }

  // The default options (CSV).
  $options = array(
    'record separator' => '<newline>',
    'field separator'  => ',',
    'text delimiter'   => '"',
    'escape character' => '"',
  );

  // Use 'other ...' when selected.
  foreach ($options as $key => $default) {
    if (isset($file_options[$key]) && drupal_strlen($file_options[$key]) > 0) {
      $options[$key] = $file_options[$key];
    }
    else if (isset($file_options['other '. $key]) && drupal_strlen($file_options['other '. $key]) > 0) {
      $options[$key] = $file_options['other '. $key];
    }
  }

  // Replace our special characters.
  $replaces = array(
    '<newline>' => "\n",
    '<none>'    => '',
    '<space>'   => ' ',
    '<tab>'     => "\t",
  );
  $file_options = str_replace(array_keys($replaces), array_values($replaces), $options);
}

/**
 * Returns one line in the specified file format of the array of
 * values.
 *
 * @param $values
 *   Array of strings.
 *
 * @param $file_options
 *   Array with 'record separator', 'field separator', 'text delimiter'
 *   and 'escape character'. If not set, the options default to the
 *   CSV options ("\n", ',', '"', '"').
 *
 * @return
 *   String.
 */
function node_import_write_to_string($values, $file_options) {
  // File options.
  _node_import_sanitize_file_options($file_options);

  $rs = $file_options['record separator'];
  $fs = $file_options['field separator'];
  $td = $file_options['text delimiter'];
  $ec = $file_options['escape character'];

  // Write data.
  $output = '';

  if (is_array($values) && !empty($values)) {
    // TODO: we could avoid writing $td if the $value does not contain $td, $fs or $rs.
    if (drupal_strlen($td) > 0) {
      foreach ($values as $i => $value) {
        $values[$i] = $td . str_replace($td, $ec . $td, $value) . $td;
      }
    }
    $output = implode($fs, $values);
  }

  return $output . $rs;
}

/**
 * @}
 */

/**
 * @defgroup node_import_form_hacks Hacks for includes/form.inc
 * @{
 * The problem node_import has by design - where the design is that we
 * want to use the normal form validation - is that when a form is
 * submitted more then once, the validation of the form is not done
 * correctly by the core includes/form.inc
 *
 * This file tries to lift this limitation. The solution is based (or
 * rather copied) from sites/all/modules/views/includes/form.inc of
 * the views module which needs to do the same crappy stuff (and even
 * more).
 *
 * Short explanation: instead of
 * @code
 *   drupal_execute($form_id, $form_state, ...);
 * @endcode
 * use
 * @code
 *   node_import_drupal_execute($form_id, $form_state, ...);
 * @endcode
 * whenever the form you want to execute can be executed more than
 * once in the same page request.
 *
 * For the core bug, see http://drupal.org/node/260934 : Static caching:
 * cannot call drupal_validate_form on the same form more than once.
 *
 * Note that another bug for multiple form validation and submission
 * (one that could not be lifted) was fixed in Drupal 6.5, see
 * http://drupal.org/node/180063 : No way to flush form errors during
 * iterative programatic form submission.
 *
 * Many, many thanks to merlinofchaos!!
 */

/**
 * The original version of drupal_execute() calls drupal_process_form().
 * The modified version sets $form_state['must_validate'] = TRUE and
 * calls node_import_drupal_process_form() instead.
 */
function node_import_drupal_execute($form_id, &$form_state) {
  $args = func_get_args();
  $args[1] = &$form_state;
  $form = call_user_func_array('drupal_retrieve_form', $args);
  $form['#post'] = $form_state['values'];
  $form_state['must_validate'] = TRUE;
  drupal_prepare_form($form_id, $form, $form_state);
  node_import_drupal_process_form($form_id, $form, $form_state);
}

/**
 * The original version of drupal_process_form() calls drupal_validate_form().
 * The modified version calls node_import_drupal_validate_form() instead.
 */
function node_import_drupal_process_form($form_id, &$form, &$form_state) {
  $form_state['values'] = array();

  $form = form_builder($form_id, $form, $form_state);
  // Only process the form if it is programmed or the form_id coming
  // from the POST data is set and matches the current form_id.
  if ((!empty($form['#programmed'])) || (!empty($form['#post']) && (isset($form['#post']['form_id']) && ($form['#post']['form_id'] == $form_id)))) {
    node_import_drupal_validate_form($form_id, $form, $form_state);

    // form_clean_id() maintains a cache of element IDs it has seen,
    // so it can prevent duplicates. We want to be sure we reset that
    // cache when a form is processed, so scenerios that result in
    // the form being built behind the scenes and again for the
    // browser don't increment all the element IDs needlessly.
    form_clean_id(NULL, TRUE);

    if ((!empty($form_state['submitted'])) && !form_get_errors() && empty($form_state['rebuild'])) {
      $form_state['redirect'] = NULL;
      form_execute_handlers('submit', $form, $form_state);

      // We'll clear out the cached copies of the form and its stored data
      // here, as we've finished with them. The in-memory copies are still
      // here, though.
      if (variable_get('cache', CACHE_DISABLED) == CACHE_DISABLED && !empty($form_state['values']['form_build_id'])) {
        cache_clear_all('form_'. $form_state['values']['form_build_id'], 'cache_form');
        cache_clear_all('storage_'. $form_state['values']['form_build_id'], 'cache_form');
      }

      // If batches were set in the submit handlers, we process them now,
      // possibly ending execution. We make sure we do not react to the batch
      // that is already being processed (if a batch operation performs a
      // drupal_execute).
      if ($batch =& batch_get() && !isset($batch['current_set'])) {
        // The batch uses its own copies of $form and $form_state for
        // late execution of submit handers and post-batch redirection.
        $batch['form'] = $form;
        $batch['form_state'] = $form_state;
        $batch['progressive'] = !$form['#programmed'];
        batch_process();
        // Execution continues only for programmatic forms.
        // For 'regular' forms, we get redirected to the batch processing
        // page. Form redirection will be handled in _batch_finished(),
        // after the batch is processed.
      }

      // If no submit handlers have populated the $form_state['storage']
      // bundle, and the $form_state['rebuild'] flag has not been set,
      // we're finished and should redirect to a new destination page
      // if one has been set (and a fresh, unpopulated copy of the form
      // if one hasn't). If the form was called by drupal_execute(),
      // however, we'll skip this and let the calling function examine
      // the resulting $form_state bundle itself.
      if (!$form['#programmed'] && empty($form_state['rebuild']) && empty($form_state['storage'])) {
        drupal_redirect_form($form, $form_state['redirect']);
      }
    }
  }
}

/**
 * The original version of drupal_validate_form() keeps a static array
 * of validated forms. The modified version checks $form_state['must_validate']
 * to see if the form needs validation. If set and TRUE, validation is
 * forced even if it was already done.
 */
function node_import_drupal_validate_form($form_id, $form, &$form_state) {
  static $validated_forms = array();

  if (isset($validated_forms[$form_id]) &&
      (!isset($form_state['must_validate']) || $form_state['must_validate'] !== TRUE)) { //Changed!
    return;
  }

  // If the session token was set by drupal_prepare_form(), ensure that it
  // matches the current user's session.
  if (isset($form['#token'])) {
    if (!drupal_valid_token($form_state['values']['form_token'], $form['#token'])) {
      // Setting this error will cause the form to fail validation.
      form_set_error('form_token', t('Validation error, please try again. If this error persists, please contact the site administrator.'));
    }
  }

  _form_validate($form, $form_state, $form_id);
  $validated_forms[$form_id] = TRUE;
}

/**
 * @}
 */

/**
 * @defgroup node_import_locking Node import locking functions
 * @{
 * Locking functions for node_import. This code is based on #251792
 * (Implement a locking framework for long operations). We need
 * locking to avoid both hook_cron() and node_import_view_form() to
 * process the same task at the same time (which would result in
 * rows being processed twice).
 *
 * The code below uses the database-based locking mechanism. Note
 * that is does not use the full implementation as in the patch
 * because we can have only one global lock. One lock for each task
 * does not work because a task can spawn the creation of something
 * else (such as a taxonomy term).
 *
 * We only need one _acquire() and _release() function.
 *
 * When a more general locking framework is committed to Drupal,
 * we can easily replace this.
 */

/**
 * Acquire or release our node_import lock.
 *
 * @param $release
 *   Boolean. If TRUE, release the lock. If FALSE, acquire the
 *   lock.
 *
 * @return
 *   Boolean. Whether the lock was acquired.
 */
function node_import_lock_acquire($release = FALSE) {
  static $lock_id, $locked;

  if (!isset($lock_id)) {
    $lock_id = md5(uniqid());
    $locked = FALSE;
    register_shutdown_function('node_import_lock_release');
  }

  if ($release) {
    db_query("DELETE FROM {variable} WHERE name = '%s'", 'node_import:lock');
    $locked = FALSE;
  }
  else if (!$locked) {
    if (@db_query("INSERT INTO {variable} (name, value) VALUES ('%s', '%s')", 'node_import:lock', $lock_id)) {
      $locked = TRUE;
    }
  }

  return $locked;
}

/**
 * Release our node_import lock.
 *
 * @return
 *   Nothing.
 */
function node_import_lock_release() {
  node_import_lock_acquire(TRUE);
}

/**
 * @}
 */

