/*
 * Copyright 2009, 2010 Lari Natri <lari.natri@gmail.com>
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 * 
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 * 
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package compid.importers;

import net.aavalla.utils.jobpool.parameters.FieldAnnotations;
import compid.data.tuples.PeptideHit;
import compid.data.tuples.Peptide;
import compid.data.tuples.ProteinHit;
import compid.data.tuples.DatasetInfo;
import compid.data.tuples.Query;
import compid.data.tuples.ProtHitPeptHitJoin;
import compid.data.tuples.Protein;
import java.io.File;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.HashMap;
import java.util.StringTokenizer;

import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.hssf.usermodel.*;

import compid.data.*;
import compid.data.tuples.ProteinHit.GroupMemberStatus;
import compid.importers.ImportException.ExceptionType;

import java.util.Date;

/**
 *
 * @author Lari Natri
 */
public class ParagonExcelImporter extends DatasetFileImporter<DatasetFileImporterParams> {

    public class ParagonExcelImporterParams extends DatasetFileImporterParams {

        @FieldAnnotations(
            name="Include Unassigned Peptides",
            description="Import also peptides which aren't assigned to any proteins. \n" +
                        "This should be chosen if one wishes to create peptide reports \n" +
                        "for all peptides. All queries are imported in any case.")
        private boolean includeUnassigned = true;

        public ParagonExcelImporterParams() {
            this.setDescription("Imports a dataset from Paragon in .xls format. This format <br/>" +
                                "only shows winner proteins. It is better to import ProteinPilot <br/>" +
                                "datasets using XML format, if possible, because XML has more <br/>" +
                                "information available.");
        }
    }
    
    private static final String DESCRIPTION = "Dataset importer for Paragon (Excel)";
    private static final String FILE_EXT = ".xls"; // NOI18N
    private static final int PR_MINHEADERS = 6;
    private static final int SP_MINHEADERS = 10;
    private static final int MAXHEADERROWDEPTH = 20;
    
    private static enum ProtCol
    {
        N          { String colHdr() { return "N"; } }, // NOI18N
        UNUSED     { String colHdr() { return "UNUSED"; } }, // NOI18N
        TOTAL      { String colHdr() { return "TOTAL"; } }, // NOI18N
        COV        { String colHdr() { return "COV"; } }, // NOI18N
        ACCESSION  { String colHdr() { return "ACCESSION"; } }, // NOI18N
        NAME       { String colHdr() { return "NAME"; } }, // NOI18N
        SPECIES    { String colHdr() { return "SPECIES"; } }, // NOI18N
        BIOLOGICAL { String colHdr() { return "BIOLOGICAL"; } }, // NOI18N
        MOLECULAR  { String colHdr() { return "MOLECULAR"; } }, // NOI18N
        PANTHERID  { String colHdr() { return "PANTHERID"; } }; // NOI18N
        
        abstract String colHdr();
    }

    private static enum SpectrCol
    {
        SPECTRUM { String colHdr() { return "SPECTRUM"; } },
        TIME     { String colHdr() { return "TIME"; } },
        PRECMW   { String colHdr() { return "PRECMW"; } },
        PRECMZ   { String colHdr() { return "PRECMZ"; } },
        PRECZ    { String colHdr() { return "PRECZ"; } },
        N        { String colHdr() { return "PROTN"; } },
        BESTSEQ  { String colHdr() { return "BESTSEQUENCE"; } },
        MODS     { String colHdr() { return "MODIFICATIONS"; } },
        CONF     { String colHdr() { return "CONF"; } },
        THEORMW  { String colHdr() { return "THEORMW"; } },
        THEORZ   { String colHdr() { return "Z"; } };
        
        abstract String colHdr();
    }    
    
    private HashMap<ProtCol,Integer> protCols = null;
    private HashMap<SpectrCol,Integer> spectrCols = null;
    private int protHeaderRow = -1;
    private int spectrHeaderRow = -1;
    private HSSFSheet protSheet = null;
    private HSSFSheet spectrSheet = null;
    private File[] fileNames = null;

    public ParagonExcelImporter() {
        super();
        this.setName(DESCRIPTION);
    }
    
    @Override
    public String getFileNameExtension() { return FILE_EXT; }
    
    @Override
    protected DatasetFileImporterParams getNonfinalEditableParameters() {
        return new ParagonExcelImporterParams();
    }
    
    @Override
    public FullDataset getAsDataSet(DatasetFileImporterParams params)
                                                   throws ImportException {

        ParagonExcelImporterParams p = (ParagonExcelImporterParams)params;

        fileNames = new File[1];
        fileNames[0] = p.getFile();
        
        // load files & check their validity
        try {
            HSSFWorkbook[] wbs = new HSSFWorkbook[fileNames.length];
            int i=0;
            for (File f : fileNames) {
                if (!f.exists())
                    throw new ImportException(ExceptionType.FILENOTFOUND,
                                              f.getAbsolutePath());
                if (!f.canRead())
                    throw new ImportException(ExceptionType.FILENOTREADABLE,
                                              f.getAbsolutePath());                
                FileInputStream fis = new FileInputStream(f);
                BufferedInputStream bis = new BufferedInputStream(fis);
                POIFSFileSystem fs = new POIFSFileSystem(bis);
                wbs[i] = new HSSFWorkbook(fs);
                bis.close();
                fis.close();
                ++i;
            }
            if (!this.findCoordinates(wbs))
                throw new ImportException(ExceptionType.PGON_SHEETSMISSING);

        } catch (ImportException ex) {
            throw ex;    
        } catch (java.io.IOException ex) {
            throw new ImportException("Error loading file. " + ex);
        }
        
        DatasetInfo cpDSInfo = new DatasetInfo(p.getName(),p.getDescr(),p.getDB(),
                this.fileNames[0].getName()+ (fileNames.length>1 ? " + "
                + (fileNames.length-1) + " files" : ""),
                DatasetInfo.AlgType.PARAGON, null, new Date());

        if (p.includeUnassigned) {
            cpDSInfo.setParams("Unassigned Peptides = "+p.includeUnassigned);
        }

        FullDataset cpFullDataSet = new FullDataset();
        cpFullDataSet.addDataSetInfo(cpDSInfo);
        
        
        int debug_prothitpepthitjoins = 0;
        int debug_spectrums = 0;
        
        try {

            Iterator rowIter = spectrSheet.rowIterator();
                
            // loop through all rows in spectrum sheet
            while (rowIter.hasNext()) {
                HSSFRow row = (HSSFRow)rowIter.next();
                if (row.getRowNum()<=spectrHeaderRow) continue; // skip header

                CellFormatter qSpectrum = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.SPECTRUM).intValue()));
                CellFormatter pepConf = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.CONF).intValue()));
                CellFormatter pepObsMass = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.PRECMZ).intValue()));
                //CellFormatter pepPredMass = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.PRECMW).intValue()));
                //CellFormatter pepCalcMass = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.THEORMW).intValue()));
                CellFormatter pepCharge = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.PRECZ).intValue())); // was PRECZ
                CellFormatter pepTheorCharge = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.THEORZ).intValue())); // was PRECZ
                CellFormatter pepSequence = new CellFormatter(row.getCell(spectrCols.get(SpectrCol.BESTSEQ).intValue()));
                if (pepCharge.asDouble==0 && pepTheorCharge.goodDouble) pepCharge.asDouble = pepTheorCharge.asDouble; // use Theoretical Charge, if instrument reports 0 - as per Paragon documentation
                //if (pepPredMass.asDouble==0) pepPredMass.asDouble=pepCalcMass.asDouble;

                Query cpQuery = new Query(cpDSInfo, qSpectrum.asString,
                                          pepObsMass.asDouble,
                                          pepCharge.asDouble);
                debug_spectrums++;
                if (pepSequence.goodString && !pepSequence.asString.trim().equals("")) { // peptide hit exists
                    Peptide cpPep = new Peptide(pepSequence.asString.trim());

                    PeptideHit cpPepHit = new PeptideHit(cpPep,cpQuery,pepConf.asDouble/100d); // XML reports 0..1, excel 0..99, so divide here

                    int protN = spectrCols.get(SpectrCol.N).intValue();
                    CellFormatter protList = new CellFormatter(row.getCell(protN));
                    if (!protList.asString.equals("")) { // protein hits exist
                        HSSFRow[] protRows = this.getProteinRowsFromProteinListString(protList.asString);
                        for (int i=0; i<protRows.length; ++i) {
                            if (protRows[i]==null) continue;
                            CellFormatter prAccession = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.ACCESSION).intValue()));
                            CellFormatter prDescription = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.NAME).intValue()));
                            CellFormatter prRank = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.N).intValue()));
                            CellFormatter prUnused = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.UNUSED).intValue()));
                            CellFormatter prTotal = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.TOTAL).intValue()));
                            //CellFormatter prCov = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.COV).intValue()));
                            CellFormatter prSpecies = new CellFormatter(protRows[i].getCell(protCols.get(ProtCol.SPECIES).intValue()));
                            Protein cpProt = new Protein(prAccession.asString, prDescription.asString, prSpecies.asString);
                            ProteinHit cpProtHit = new ProteinHit(cpProt,cpDSInfo,prRank.asInt,GroupMemberStatus.WINNER,prUnused.asDouble,prTotal.asDouble);
                            ProtHitPeptHitJoin cpProtPepHit = new ProtHitPeptHitJoin(cpProtHit,cpPepHit);
                            debug_prothitpepthitjoins++;
                            // adds all
                            cpFullDataSet.addProteinHitPeptideHitJoin(cpProtPepHit);
                        }
                    } else { // no protein hit for this peptide
                        if (p.includeUnassigned) {
                            cpFullDataSet.addPeptideHit(cpPepHit);
                        } else {
                            cpFullDataSet.addQuery(cpQuery);
                        }
                    }
                } else { // no peptide hit, only query
                    cpFullDataSet.addQuery(cpQuery);
                }
            }
            

        } catch (Exception ex) {
            throw new ImportException(ex);
        }
        System.out.println("DEBUG: Spectrums in sheet "+debug_spectrums);
        System.out.println("DEBUG: ProtHitPeptHits sheet "+debug_prothitpepthitjoins);
        return cpFullDataSet;
    }

    /**
     * Checks workbooks given as parameter and tries to find Spectrum Sheet
     * and Protein sheet. If sheets are found, they are saved as this object's
     * attributes, and so are the column numbers associated with each header.
     * 
     * @param workBooks workbooks to search in
     * @return true if both sheets found, false otherwise
     */
    private boolean findCoordinates(HSSFWorkbook[] workBooks) {

        boolean prSheetFound = false;
        boolean spSheetFound = false;

        // loop through all workbooks given as parameter
        for (int book=0; book<workBooks.length; ++book) {
            if (workBooks[book]==null) continue;
            
            // loop through all sheets in this workbook
            for (int sheetNr=0; sheetNr<workBooks[book].getNumberOfSheets();
                 ++sheetNr)
            {
                HSSFSheet sheet = workBooks[book].getSheetAt(sheetNr);
                boolean foundMeaningInThisSheet = false;
                Iterator rowIter = sheet.rowIterator();
                
                // loop through all rows in this sheet
                while (rowIter.hasNext() && !foundMeaningInThisSheet)
                {
                    if (!prSheetFound) protCols =
                            new HashMap<ProtCol,Integer>(ProtCol.values().length,1);
                    if (!spSheetFound) spectrCols =
                            new HashMap<SpectrCol,Integer>(SpectrCol.values().length,1);
                    HSSFRow row = (HSSFRow)rowIter.next();
                    if (row.getRowNum()>MAXHEADERROWDEPTH) break;
                    Iterator cellIter = row.cellIterator();
                    
                    // loop through cells in this row
                    while (cellIter.hasNext())
                    {
                        HSSFCell cell  = (HSSFCell)cellIter.next();
                        
                        // headers must be of string type
                        if (cell.getCellType()==HSSFCell.CELL_TYPE_STRING) {
                            String cellValue = cell.getRichStringCellValue()
                                                   .getString().trim()
                                                   .toUpperCase()
                                                   .replaceAll("[^A-Z]","");
                            
                            // check for protein summary headers
                            for (ProtCol pc : ProtCol.values()) {
                                if (cellValue.equals(pc.colHdr())) {
                                    protCols.put(pc,
                                             new Integer(cell.getColumnIndex()));
                                    if (protCols.size()>=PR_MINHEADERS) {
                                        protSheet = sheet;
                                        protHeaderRow = row.getRowNum();
                                        prSheetFound = true;
                                        foundMeaningInThisSheet = true;
                                    }
                                }
                            }
                            
                            // check for spectrum summary headers
                            for (SpectrCol sc : SpectrCol.values()) {
                                if (cellValue.equals(sc.colHdr())) {
                                    spectrCols.put(sc,
                                               new Integer(cell.getColumnIndex()));
                                    if (spectrCols.size()>=SP_MINHEADERS) {
                                        spectrSheet = sheet;
                                        spectrHeaderRow = row.getRowNum();
                                        spSheetFound = true;
                                        foundMeaningInThisSheet = true;
                                    }
                                }
                            } // for SpectrCol
                        } // if (cell.getCellType()==HFFSCell.CELL_TYPE_STRING)
                    } // while (cellIter.hasNext())
                } // while (rowIter.hasNext())
            } // for sheet
        } // for workBooks
        
        if (!prSheetFound) {
            protSheet = null;
            protCols = null;
            protHeaderRow = -1;
        }
        if (!spSheetFound) {
            spectrSheet = null;
            spectrCols = null;
            spectrHeaderRow = -1;
        }
        
        if (prSheetFound && spSheetFound) return true; // both sheets found
        return false;
    }
    
    private HSSFRow[] getProteinRowsFromProteinListString(String list) {
        if (list==null) return new HSSFRow[0];
        ArrayList<Integer> prNumbers = new ArrayList<Integer>();
        StringTokenizer st = new StringTokenizer(list," ,;-\t\n\r\f");
        int prN;
        while(st.hasMoreTokens()){
            String token = st.nextToken().trim();
            try {
                prN = Integer.parseInt(token);
            } catch (NumberFormatException ex) { prN = -1; }
            if (prN>0) {
                prNumbers.add(prN);
            }
        }
        HSSFRow[] resRows = new HSSFRow[prNumbers.size()];
        HSSFRow row;
        for (int i=0; i<prNumbers.size(); ++i) {
            row = this.getProtRowByRank(prNumbers.get(i).intValue());
            resRows[i] = row; // even if null
        }
        return resRows;
    }
    
    private HSSFRow getProtRowByRank(int rank) {
        HSSFRow row = protSheet.getRow(protHeaderRow+rank);
        CellFormatter cellN = new CellFormatter(row.getCell(protCols.get(ProtCol.N).intValue()));
        
        // if not found where it should have (order changed), go through all
        if (!cellN.goodInt || cellN.asInt!=rank) {
            // println for debugging
            System.out.println("DEBUG: correct N was not in expected row");
            Iterator iter = protSheet.rowIterator();
            while (iter.hasNext()) {
                row = (HSSFRow)iter.next();
                cellN.setValues(row.getCell(protCols.get(ProtCol.N).intValue()));
                if (cellN.goodInt && cellN.asInt==rank) break; // found it
            }
        }
        if (cellN.goodInt && cellN.asInt==rank) return row;
        // println for debugging
        System.out.println("DEBUG: and it was never found! ERROR.");
        return null;
    }
    
    private class CellFormatter {
        double asDouble = 0.0d;
        int asInt = 0;
        String asString = null;
        boolean goodDouble = false;
        boolean goodInt = false;
        boolean goodString = false;
        
        CellFormatter() {}
        
        CellFormatter(HSSFCell cell) {
            this.setValues(cell);
        }

        void setValues(HSSFCell cell) {
            if (cell==null) {
                asString = "";
                asDouble = 0.0d;
                asInt = 0;
                goodString = false;
                goodDouble = false;
                goodInt = false;
                return;
            }
            switch (cell.getCellType()) {
                case HSSFCell.CELL_TYPE_STRING:
                    asString = cell.getRichStringCellValue().getString();
                    goodString = true;
                    try {
                        asDouble = Double.parseDouble(asString.trim());
                        goodDouble = true;
                    } catch(NumberFormatException ex) {
                        asDouble = 0.0d;
                        goodDouble = false;
                    }
                    try {
                        asInt = Integer.parseInt(asString.trim());
                        goodInt = true;
                    } catch(NumberFormatException ex) {
                        asInt = 0;
                        goodInt = false;
                    }                    
                    break;
                case HSSFCell.CELL_TYPE_NUMERIC:
                    double numCellValue = cell.getNumericCellValue();
                    if ((int)numCellValue==numCellValue) {
                        asString = Integer.toString((int)numCellValue);
                    } else asString = Double.toString(numCellValue);
                    
                    goodString = true;
                    asDouble = numCellValue;
                    goodDouble = true;
                    asInt = (int)numCellValue;
                    if (asInt<asDouble) goodInt = false;
                    else goodInt = true;
                    break;
                case HSSFCell.CELL_TYPE_BLANK:
                    asString = "";     // should this be null?
                    goodString = true; // should this be false?
                    asDouble = 0.0d;
                    goodDouble = false;
                    asInt = 0;
                    goodInt = false;
                    break;
                case HSSFCell.CELL_TYPE_BOOLEAN:
                    if (cell.getBooleanCellValue()) {
                        asString = "TRUE";
                        asDouble = 1.0d;
                        asInt = 1;
                    } else {
                        asString = "FALSE";
                        asDouble = 0.0d;
                        asInt = 0;                        
                    }
                    goodString = goodDouble = goodInt = false;
                    break;
                case HSSFCell.CELL_TYPE_ERROR:
                    byte b = cell.getErrorCellValue();
                    asString = Integer.toString(b);
                    goodString = false;
                    asDouble = b;
                    goodDouble = false;
                    asInt = b;
                    goodInt = false;
                    break;
                case HSSFCell.CELL_TYPE_FORMULA:
                    asString = cell.getRichStringCellValue().getString();
                    goodString = false;
                    asDouble = 0.0d;
                    goodDouble = false;
                    asInt = 0;
                    goodInt = false;
                    break;
            } // switch
            
        } // setValues
    } // inner class CellFormatter

}
