/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 *    C45Loader.java
 *    Copyright (C) 2000-2012 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.core.converters;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.util.ArrayList;

import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.RevisionUtils;
import weka.core.Utils;

/**
 <!-- globalinfo-start --> 
 * Reads a file that is C45 format. Can take a
 * filestem or filestem with .names or .data appended. Assumes that
 * path/&lt;filestem&gt;.names and path/&lt;filestem&gt;.data exist and contain
 * the names and data respectively.
 * <p/>
 <!-- globalinfo-end -->
 * 
 * @author Mark Hall (mhall@cs.waikato.ac.nz)
 * @version $Revision: 9290 $
 * @see Loader
 */
public class C45Loader extends AbstractFileLoader implements BatchConverter,
    IncrementalConverter {

  /** for serialization */
  static final long serialVersionUID = 5454329403218219L;

  /** the file extension */
  public static String FILE_EXTENSION = ".names";

  /**
   * Describe variable <code>m_sourceFileData</code> here.
   */
  private File m_sourceFileData = null;

  /**
   * Reader for names file
   */
  private transient Reader m_namesReader = null;

  /**
   * Reader for data file
   */
  private transient Reader m_dataReader = null;

  /**
   * Holds the filestem.
   */
  private String m_fileStem;

  /**
   * Number of attributes in the data (including ignore and label attributes).
   */
  private int m_numAttribs;

  /**
   * Which attributes are ignore or label. These are *not* included in the arff
   * representation.
   */
  private boolean[] m_ignore;

  /**
   * Returns a string describing this attribute evaluator
   * 
   * @return a description of the evaluator suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String globalInfo() {
    return "Reads a file that is C45 format. Can take a filestem or filestem "
        + "with .names or .data appended. Assumes that path/<filestem>.names and "
        + "path/<filestem>.data exist and contain the names and data "
        + "respectively.";
  }

  /**
   * Resets the Loader ready to read a new data set or the same data set again.
   * 
   * @throws IOException if something goes wrong
   */
  @Override
  public void reset() throws IOException {
    m_structure = null;
    setRetrieval(NONE);

    if (m_File != null) {
      setFile(new File(m_File));
    }
  }

  /**
   * Get the file extension used for arff files
   * 
   * @return the file extension
   */
  @Override
  public String getFileExtension() {
    return FILE_EXTENSION;
  }

  /**
   * Gets all the file extensions used for this type of file
   * 
   * @return the file extensions
   */
  @Override
  public String[] getFileExtensions() {
    return new String[] { ".names", ".data" };
  }

  /**
   * Returns a description of the file type.
   * 
   * @return a short file description
   */
  @Override
  public String getFileDescription() {
    return "C4.5 data files";
  }

  /**
   * Resets the Loader object and sets the source of the data set to be the
   * supplied File object.
   * 
   * @param file the source file.
   * @exception IOException if an error occurs
   */
  @Override
  public void setSource(File file) throws IOException {
    m_structure = null;
    setRetrieval(NONE);

    if (file == null) {
      throw new IOException("Source file object is null!");
    }

    String fname = file.getName();
    String fileStem;
    String path = file.getParent();
    if (path != null) {
      path += File.separator;
    } else {
      path = "";
    }
    if (fname.indexOf('.') < 0) {
      fileStem = fname;
      fname += ".names";
    } else {
      fileStem = fname.substring(0, fname.lastIndexOf('.'));
      fname = fileStem + ".names";
    }
    m_fileStem = fileStem;
    file = new File(path + fname);

    m_sourceFile = file;
    try {
      BufferedReader br = new BufferedReader(new FileReader(file));
      m_namesReader = br;
    } catch (FileNotFoundException ex) {
      throw new IOException("File not found : " + (path + fname));
    }

    m_sourceFileData = new File(path + fileStem + ".data");
    try {
      BufferedReader br = new BufferedReader(new FileReader(m_sourceFileData));
      m_dataReader = br;
    } catch (FileNotFoundException ex) {
      throw new IOException("File not found : " + (path + fname));
    }
    m_File = file.getAbsolutePath();
  }

  /**
   * Determines and returns (if possible) the structure (internally the header)
   * of the data set as an empty set of instances.
   * 
   * @return the structure of the data set as an empty set of Instances
   * @exception IOException if an error occurs
   */
  @Override
  public Instances getStructure() throws IOException {
    if (m_sourceFile == null) {
      throw new IOException("No source has beenspecified");
    }

    if (m_structure == null) {
      setSource(m_sourceFile);
      StreamTokenizer st = new StreamTokenizer(m_namesReader);
      initTokenizer(st);
      readHeader(st);
    }

    return m_structure;
  }

  /**
   * Return the full data set. If the structure hasn't yet been determined by a
   * call to getStructure then method should do so before processing the rest of
   * the data set.
   * 
   * @return the structure of the data set as an empty set of Instances
   * @exception IOException if there is no source or parsing fails
   */
  @Override
  public Instances getDataSet() throws IOException {
    if (m_sourceFile == null) {
      throw new IOException("No source has been specified");
    }
    if (getRetrieval() == INCREMENTAL) {
      throw new IOException(
          "Cannot mix getting Instances in both incremental and batch modes");
    }
    setRetrieval(BATCH);
    if (m_structure == null) {
      getStructure();
    }
    StreamTokenizer st = new StreamTokenizer(m_dataReader);
    initTokenizer(st);
    // st.ordinaryChar('.');
    Instances result = new Instances(m_structure);
    Instance current = getInstance(st);

    while (current != null) {
      result.add(current);
      current = getInstance(st);
    }
    try {
      // close the stream
      m_dataReader.close();
      // reset();
    } catch (Exception ex) {
      ex.printStackTrace();
    }
    return result;
  }

  /**
   * Read the data set incrementally---get the next instance in the data set or
   * returns null if there are no more instances to get. If the structure hasn't
   * yet been determined by a call to getStructure then method should do so
   * before returning the next instance in the data set.
   * 
   * If it is not possible to read the data set incrementally (ie. in cases
   * where the data set structure cannot be fully established before all
   * instances have been seen) then an exception should be thrown.
   * 
   * @param structure the dataset header information, will get updated in case
   *          of string or relational attributes
   * @return the next instance in the data set as an Instance object or null if
   *         there are no more instances to be read
   * @exception IOException if there is an error during parsing
   */
  @Override
  public Instance getNextInstance(Instances structure) throws IOException {
    if (m_sourceFile == null) {
      throw new IOException("No source has been specified");
    }

    if (getRetrieval() == BATCH) {
      throw new IOException(
          "Cannot mix getting Instances in both incremental and batch modes");
    }
    setRetrieval(INCREMENTAL);

    if (m_structure == null) {
      getStructure();
    }

    StreamTokenizer st = new StreamTokenizer(m_dataReader);
    initTokenizer(st);
    // st.ordinaryChar('.');
    Instance nextI = getInstance(st);
    if (nextI != null) {
      nextI.setDataset(m_structure);
    } else {
      try {
        // close the stream
        m_dataReader.close();
        // reset();
      } catch (Exception ex) {
        ex.printStackTrace();
      }
    }
    return nextI;
  }

  /**
   * Reads an instance using the supplied tokenizer.
   * 
   * @param tokenizer the tokenizer to use
   * @return an Instance or null if there are no more instances to read
   * @exception IOException if an error occurs
   */
  private Instance getInstance(StreamTokenizer tokenizer) throws IOException {
    double[] instance = new double[m_structure.numAttributes()];

    StreamTokenizerUtils.getFirstToken(tokenizer);
    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
      return null;
    }

    int counter = 0;
    for (int i = 0; i < m_numAttribs; i++) {
      if (i > 0) {
        StreamTokenizerUtils.getToken(tokenizer);
      }

      if (!m_ignore[i]) {
        // Check if value is missing.
        if (tokenizer.ttype == '?') {
          instance[counter++] = Utils.missingValue();
        } else {
          String val = tokenizer.sval;

          if (i == m_numAttribs - 1) {
            // remove trailing period
            if (val.charAt(val.length() - 1) == '.') {
              val = val.substring(0, val.length() - 1);
            }
          }
          if (m_structure.attribute(counter).isNominal()) {
            int index = m_structure.attribute(counter).indexOfValue(val);
            if (index == -1) {
              StreamTokenizerUtils.errms(tokenizer,
                  "nominal value not declared in " + "header :" + val
                      + " column " + i);
            }
            instance[counter++] = index;
          } else if (m_structure.attribute(counter).isNumeric()) {
            try {
              instance[counter++] = Double.valueOf(val).doubleValue();
            } catch (NumberFormatException e) {
              StreamTokenizerUtils.errms(tokenizer, "number expected");
            }
          } else {
            System.err.println("Shouldn't get here");
            System.exit(1);
          }
        }
      }
    }

    return new DenseInstance(1.0, instance);
  }

  /**
   * removes the trailing period
   * 
   * @param val the string to work on
   * @return the processed string
   */
  private String removeTrailingPeriod(String val) {
    // remove trailing period
    if (val.charAt(val.length() - 1) == '.') {
      val = val.substring(0, val.length() - 1);
    }
    return val;
  }

  /**
   * Reads header (from the names file) using the supplied tokenizer
   * 
   * @param tokenizer the tokenizer to use
   * @exception IOException if an error occurs
   */
  private void readHeader(StreamTokenizer tokenizer) throws IOException {

    ArrayList<Attribute> attribDefs = new ArrayList<Attribute>();
    ArrayList<Integer> ignores = new ArrayList<Integer>();
    StreamTokenizerUtils.getFirstToken(tokenizer);
    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
      StreamTokenizerUtils.errms(tokenizer, "premature end of file");
    }

    m_numAttribs = 1;
    // Read the class values
    ArrayList<String> classVals = new ArrayList<String>();
    while (tokenizer.ttype != StreamTokenizer.TT_EOL) {
      String val = tokenizer.sval.trim();

      if (val.length() > 0) {
        val = removeTrailingPeriod(val);
        classVals.add(val);
      }
      StreamTokenizerUtils.getToken(tokenizer);
    }

    // read the attribute names and types
    int counter = 0;
    while (tokenizer.ttype != StreamTokenizer.TT_EOF) {
      StreamTokenizerUtils.getFirstToken(tokenizer);
      if (tokenizer.ttype != StreamTokenizer.TT_EOF) {

        String attribName = tokenizer.sval;

        StreamTokenizerUtils.getToken(tokenizer);
        if (tokenizer.ttype == StreamTokenizer.TT_EOL) {
          StreamTokenizerUtils.errms(tokenizer,
              "premature end of line. Expected " + "attribute type.");
        }
        String temp = tokenizer.sval.toLowerCase().trim();
        if (temp.startsWith("ignore") || temp.startsWith("label")) {
          ignores.add(new Integer(counter));
          counter++;
        } else if (temp.startsWith("continuous")) {
          attribDefs.add(new Attribute(attribName));
          counter++;
        } else {
          counter++;
          // read the values of the attribute
          ArrayList<String> attribVals = new ArrayList<String>();
          while (tokenizer.ttype != StreamTokenizer.TT_EOL
              && tokenizer.ttype != StreamTokenizer.TT_EOF) {
            String val = tokenizer.sval.trim();

            if (val.length() > 0) {
              val = removeTrailingPeriod(val);
              attribVals.add(val);
            }
            StreamTokenizerUtils.getToken(tokenizer);
          }
          attribDefs.add(new Attribute(attribName, attribVals));
        }
      }
    }

    boolean ok = true;
    int i = -1;
    if (classVals.size() == 1) {
      // look to see if this is an attribute name (ala c5 names file style)
      for (i = 0; i < attribDefs.size(); i++) {
        if (attribDefs.get(i).name().compareTo(classVals.get(0)) == 0) {
          ok = false;
          m_numAttribs--;
          break;
        }
      }
    }

    if (ok) {
      attribDefs.add(new Attribute("Class", classVals));
    }

    m_structure = new Instances(m_fileStem, attribDefs, 0);

    try {
      if (ok) {
        m_structure.setClassIndex(m_structure.numAttributes() - 1);
      } else {
        m_structure.setClassIndex(i);
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }

    m_numAttribs = m_structure.numAttributes() + ignores.size();
    m_ignore = new boolean[m_numAttribs];
    for (i = 0; i < ignores.size(); i++) {
      m_ignore[ignores.get(i).intValue()] = true;
    }
  }

  /**
   * Initializes the stream tokenizer
   * 
   * @param tokenizer the tokenizer to initialize
   */
  private void initTokenizer(StreamTokenizer tokenizer) {
    tokenizer.resetSyntax();
    tokenizer.whitespaceChars(0, (' ' - 1));
    tokenizer.wordChars(' ', '\u00FF');
    tokenizer.whitespaceChars(',', ',');
    tokenizer.whitespaceChars(':', ':');
    // tokenizer.whitespaceChars('.','.');
    tokenizer.commentChar('|');
    tokenizer.whitespaceChars('\t', '\t');
    tokenizer.quoteChar('"');
    tokenizer.quoteChar('\'');
    tokenizer.eolIsSignificant(true);
  }

  /**
   * Returns the revision string.
   * 
   * @return the revision
   */
  @Override
  public String getRevision() {
    return RevisionUtils.extract("$Revision: 9290 $");
  }

  /**
   * Main method for testing this class.
   * 
   * @param args should contain &lt;filestem&gt;[.names | data]
   */
  public static void main(String[] args) {
    runFileLoader(new C45Loader(), args);
  }
}
