/* -------------------------------------------------------------------------- */
/*                                                                            */
/*                      PHRASE BASED TEXT CLASSIFICATION                      */
/*                                                                            */
/*                                Frans Coenen                                */
/*                                                                            */
/*                          Tuesday 20 December 2005                          */
/*                           (Revised January 2006)                           */
/*                                                                            */
/*                       Department of Computer Science                       */
/*                        The University of Liverpool                         */
/*                                                                            */
/* -------------------------------------------------------------------------- */

/* Class structure

AssocRuleMining
	|
	+-- TextMining
		|
		+-- PhraseMining	*/
	
//package lucsKDD_ARM;

// Java packages

import java.io.*;
import java.util.*;

/** Class containing methods to coordinate phrase identification from training 
set of documents, recast the training and testset in terms of the identified 
phrases and then produce a classifier from the training set tested on the test 
set. 
@author Frans Coenen
@version 20 December 2005. */

public class PhraseMining extends TextMining { 

    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                               FIELDS                             */
    /*                                                                  */
    /* ---------------------------------------------------------------- */

    /** Instance of the class PhraseBinTree in which to store words. */
    protected PhraseBinTree phraseBinTree = null;
    
    // Flags
    /** Indicates request to output phrase bin tree. */
    protected boolean outputPhraseBinTreeFlag = false;
    /** Indicates request to output phrase bin tree statistics. */
    protected boolean outputPhraseBinTreeStatsFlag = false;
    /** Indicates request to output phraselist. */
    protected boolean outputPhraseListFlag = false;
    /** Indicates request to output phraselist (first 100 items). */
    protected boolean outputPhraseList100flag = false; 
        
    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                            CONSTRUCTORS                          */
    /*                                                                  */
    /* ---------------------------------------------------------------- */

    /** Constructor processes command line arguments.
    @param args the command line arguments (array of String instances). */

    public PhraseMining(String[] args) {
        super(args);	
	
        // Create phraseBinTree object for later use.
        phraseBinTree   = new PhraseBinTree();
	}
        
    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                              METHODS                             */
    /*                                                                  */
    /* ---------------------------------------------------------------- */
	
    /* ------------------------------------------------------ */
    /*                   TOP LEVEL METHODS                    */
    /* ------------------------------------------------------ */

    /** Top level method to: (i) reads/loads the words in the input document 
    base and stores in a "word" bin tree structure and (ii) cause the "word" 
    bin tree to be processed to identify key phrases.  
    @return true if OK to proceed and false otherwise. */

    public boolean findPhrasesInDB() throws IOException {
        final boolean OK_TO_PROCEED = true;
	
        System.out.println("START ANALYSING DOC BASE FOR PHRASES\n");
	if (docClasses==null) {
	    System.out.println("ERROR: No classes!");
	    System.exit(1);
	    }
	    
        // Load document base
        loadTrainingSetDocumentBase();
	if (outputNumDocsPerClassInTrainSetFlag) outputDocumentsPerClass();

        // Process document base and generate word and phrase bin trees. 
	// If there are not too many one itemsets process test set.
	wordBinTree.setDocsPerClass(docsPerClass);
	if (processTrainingSetDocumentBase()) {
	    // Process test document set
            loadTestSetDocumentBase();        
	    // Output attribute sets if requested
	    if (outputTrainSetAttNumFlag) {
	    	System.out.println("TRAINING DATA SET\n-----------------");	
	    	outputDataArray(dataArray);
	    	}
	    if (outputTestSetAttNumFlag) {
	    	System.out.println("TEST DATA SET\n-------------");	
            	outputDataArray(testDataArray);
		}
            return(OK_TO_PROCEED);
	    }
	else return(!OK_TO_PROCEED);
        }

    /** Commences processing of word bin tree (containing training set 
    document data) to ID: (1) noise words (words above/below noise thresholds),
    (2) ordinary words (words that do not serve to differentiate between
    classes), abd (3) significant words (words that do not serve to 
    differentiate between classes). <P> The desired phrases are defined
    as per methods in the sub-classes of this class. Once identified the input 
    training data is recast so that each document is represented according to 
    whether the phrase is contained in the document or not. Trainining set is 
    stored as a 2-D array with rows representing records and columns 
    attributes. 
    @return true if number of idcenrified 1-itemsets is less than the 
    specified maximum (a constant) and false otherwise. */

    private boolean processTrainingSetDocumentBase() {
        final boolean NUM_ONE_ITEMSETS_LESS_THAN_MAX = true;

	// ID noise and insignificant words from bin tree
	System.out.println("ID NOISE, ORDINARY AND SIGNIFICANT WORDS IN " +
		           "WORDS BIN TREE\nLNT = " + lowerNoiseThold + "% (" + 
		                        lowerNoiseDocThold + " docs), UNT = " +
                                                      upperNoiseThold + "% (" +
		                         upperNoiseDocThold + " docs), SI = " + 
			                                    significanceIndex);
	System.out.println("Max # sig words " + maxNumSigWords);  
        System.out.println("================================================");
	
	// Prune word bin tree
	wordBinTree.pruneWordBinTree(lowerNoiseDocThold,upperNoiseDocThold,
                                                                      numRows);
	
	// Generate potential significant words list
        wordBinTree.generatePotentialSigWordList();
	wordBinTree.idSignificantWords();

	// Output options for word bin tree
	outputWordBinTreeOptions();

	// Create phrase bin tree, throw away word bin tree and then create
	// training data array for DM algorithm.
	createPhraseBinTree();	
	wordBinTree=null;

	// Output options for phrase bin tree and assign training data array to
	// data array field.
	outputPhraseBinTreeOptions();	
	dataArray = phraseBinTree.genTrainingDataArray(trainingSetIDs.length,
		  		                                      docBase);
								   
	// Set numOneItemSets field contained in AssocRuleMining parent class. 
	int numAttributes  = phraseBinTree.getNumNodesInPhraseBinTree();
	numOneItemSets     = numAttributes+docClasses.length;
        if (numOneItemSets>MAX_NUM_ONE_ITEMSETS) {
	    System.out.println("Number of one itemsets (" + numOneItemSets +
	    		") exceeds maximum of " + MAX_NUM_ONE_ITEMSETS);
	    return(!NUM_ONE_ITEMSETS_LESS_THAN_MAX);
	    }
	else System.out.println("Number of one itemsets = " + numOneItemSets);
	if (outputTrainSetStatsFlag) outputTrainingSetStats(numOneItemSets,
		                                           docClasses.length);
	
        // Throw away training document base.
	docBase = null;
	return(NUM_ONE_ITEMSETS_LESS_THAN_MAX);
	}


    /** Creates phrase bin tree according to definition of phrase as defined in 
    sub class. <P>Overwritten in sub-classes. */
    
    protected void createPhraseBinTree() {
        /* STUBB */
	}
	
    /** Commences process of loading test document base and storing in an array
    of arrays. */

    private void loadTestSetDocumentBase() throws IOException {  
        System.out.println("READ TEST SET");
        System.out.println("=============");
        if (testSetIDs==null) {
	    System.out.println("ERROR: No test set!");
	    System.exit(1);
	    }

        //  Define testDocBase object
	int numRowsInTestSet = testSetIDs.length;
	testDocBase = new TestSetDocumentBase(numRowsInTestSet);
        System.out.println("Num. docs. in test set = " + numRowsInTestSet + 
	                                                                  "\n");

        //  loop through document base.
        for(int index=0;index<testSetIDs.length;index++) {
	    short docNumber = testSetIDs[index];
	    // Get size of document
            int size = getDocSize(docNumber);
            if (outputSizeOfDocInTestSetFlag) {
	        System.out.print(docNumber + " ");
                System.out.println("Test document " + docNumber + ", size = " + 
		                                                         size);
		}
	    testDocBase.setTestDocSize(size);
	    // Read document
	    readTestSetDoc(docNumber);
	    // Convert to attribute values.
            phraseBinTree.convertTestDocToAtts(testDocBase,(short) index,
	                                                        classNumber);
            // Output raw test set if requested
	    if (outputTestSetRawFlag) {
	        System.out.println();
                testDocBase.outputTestSetDoc(testSetIDs[index]);
	        System.out.println();
		}
            }
	    
	// Potential output
	if (outputTestSetStatsFlag) testDocBase.outputTestSetStats(numOneItemSets,
	                                                              numClasses);
	
	// Get test data array ready for DM, and throw away test document base. 
	testDataArray = testDocBase.getTestDataArray();
	testDocBase   = null;
	// Can throw away phrase bin tree unless we wish to output rules with
	// attribute literals. (Which we do in most cases!)
	//phraseBinTree = null;
	}
	
    /* ------------------------------------ */
    /*                                      */
    /*              SET METHODS             */
    /*                                      */
    /* ------------------------------------ */
    
    /** Sets output phrase bin tree flag. 
    @param flagValue the boolean value for the flag. */
    
    public void setOutputPhraseBinTree(boolean flagValue) {
        outputPhraseBinTreeFlag = flagValue;
	}
    
    /** Sets output phrase bin tree statistics flag. 
    @param flagValue the boolean value for the flag. */
    
    public void setOutputPhraseBinTreeStats(boolean flagValue) {
        outputPhraseBinTreeStatsFlag = flagValue;
	}
    
    /** Sets output phrase list flag. 
    @param flagValue the boolean value for the flag. */
    
    public void setOutputPhraseList(boolean flagValue) {
        outputPhraseListFlag = flagValue;
	}
    
    /** Sets output phrase list 100 flag. 
    @param flagValue the boolean value for the flag. */
    
    public void setOutputPhraseList100(boolean flagValue) {
        outputPhraseList100flag = flagValue;
	}
	
    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                            OUTPUT                                */
    /*                                                                  */
    /* ---------------------------------------------------------------- */
	
    /** Checks requests to output attributes of word bin tree and complies with
    request where indicated. */
    
    private void outputWordBinTreeOptions() {
	if (outputWordBinTreeFlag) wordBinTree.outputWordBinTree();
	if (outputWordBinTreeUNflag) wordBinTree.outputWordBinTreeUNW();
	if (outputWordBinTreeLNflag) wordBinTree.outputWordBinTreeLNW();
	if (outputWordBinTreeOWflag) wordBinTree.outputWordBinTreeOW();
	if (outputWordBinTreeSWflag) wordBinTree.outputWordBinTreeSW();
	if (outputCount1wrdsFlag) wordBinTree.outputWordsWithCount1();
	if (outputWordBinTreeStatsFlag) wordBinTree.outputWordBinTreeStats();
	if (outputSigWordsPerClassFlag) wordBinTree.outputSigWordsPerClass();
	if (outputPotSigWdsPerClassFlag) wordBinTree.outputPotSigWdsPerClass(); 
	if (outputPotSigWordsListFlag) wordBinTree.outputPotentialSigWords();
	if (outputTop10sigWordsListFlag) 
	                      wordBinTree.outputTopNpotSigWords(10,docClasses);
	if (outputTrainSetAttMarkedFlag) docBase.outputDocBaseWithMarks();
	if (outputTrainSetAttMarked10flag) docBase.outputDocBaseWithMarks10();
	}
	
    /** Checks requests to output attributes of phrase bin tree and complies 
    with request where indicated. */
    
    private void outputPhraseBinTreeOptions() {
	if (outputPhraseBinTreeFlag) phraseBinTree.outputPhraseBinTree();
	if (outputPhraseBinTreeStatsFlag) 
	                              phraseBinTree.outputPhraseBinTreeStats();
        if (outputPhraseListFlag) phraseBinTree.outputPhrases();
	if (outputPhraseList100flag) phraseBinTree.outputPhrases100();
	}
    
    /** Outputs entire pre-processed training document base where attributes 
    represent phrases as set of phrase literals. */
    
    public void outputTraingSetPhrases() {
        System.out.println("TRAINING ATTRIBUTE SET (PHRASES)");
	System.out.println("================================");
	
	// Loop through records
	for (int docIndex=0;docIndex<dataArray.length;docIndex++) {
	    System.out.print("Training Doc #" + (docIndex+1) + ": ");
	    outputTraingSetDocPhrases(dataArray[docIndex]);
	    }
	
	// End
        System.out.println("\n");
	} 

   /** Outputs first 10 pre-processed documents in training document base where 
   attributes represent phrases as set of phrase literals. */
    
    public void outputTraingSetPhrases10() {
        System.out.println("TRAINING ATTRIBUTE SET (PHRASES) -- 1st 10 DOC");
	System.out.println("==============================================");
	
	// Loop through records
	for (int docIndex=0;docIndex<dataArray.length;docIndex++) {
	    System.out.print("Training Doc #" + (docIndex+1) + ": ");
	    outputTraingSetDocPhrases(dataArray[docIndex]);
	    // Check for stop (only want to output first 10 documents if more
	    // than 10 exist).
	    if (docIndex==9) break;
	    }
	
	// End
        System.out.println("\n");
	}
	
    /** Outputs training set document as a sequence of identified phrases (if
    any).
    @param docItemSet array of short integers representong identified phrases
    included in the given document. */
     	
    private void outputTraingSetDocPhrases(short[] docItemSet) {
	// Test for empty document  
	if (docItemSet == null) System.out.print(" null ");

	// Otherwise process record
	else {
	    // Reconvert where input dataset has been reordered and possibly 
	    // pruned.
	    short[] tempItemSet = reconvertItemSet(docItemSet);
	    // Attributes
	    int length = tempItemSet.length-1;
            // Punctuation marker
	    boolean startOfRecord = true;
	    // Process record
	    for (int index=0;index<length;index++) {
	        if (startOfRecord) {
		    System.out.print(" {");
		    startOfRecord = false;
		    }
	        else System.out.print(", ");
	        phraseBinTree.outputPhraseN(tempItemSet[index]);
		}
	    if (length==0) System.out.print("{} ");
	    else System.out.print("} ");
	    // Add class
	    String ClassLiteral = getClassLiteral(tempItemSet[length]);
	    System.out.println(ClassLiteral + "\n");
	    }
	}
	
    /** Outputs test set IDs as set of phrases. */
    
    public void outputTestSetPhrases() {
        System.out.println("TEST ATTRIBUTE SET (PHRASES)");
	System.out.println("============================");	
	// Loop through records
	for (int docIndex=0;docIndex<testDataArray.length;docIndex++) {	
	    System.out.print("Test Doc #" + (docIndex+1) + ": ");
	    // No record  
	    if (testDataArray[docIndex] == null) System.out.print(" null ");
	    // Otherwise process record
	    else {
	        // Reconvert where input dataset has been reordered and 
		// possibly pruned.
	        short[] tempItemSet = reconvertItemSet(testDataArray[docIndex]);
	        // Attributes
	        int length = tempItemSet.length-1;
                // Punctuation marker
		boolean startOfRecord = true;
		// Process record
	        for (int index=0;index<length;index++) {
	            if (startOfRecord) {
		        System.out.print(" {");
			startOfRecord = false;
			}
	            else System.out.print(", ");
	            phraseBinTree.outputPhraseN(tempItemSet[index]);
		    }
	        if (length==0) System.out.print("{} ");
		else System.out.print("} ");
	        // Add class
	        String ClassLiteral = 
		               getClassLiteral(tempItemSet[length]);
	        System.out.println(ClassLiteral + "\n");
		}
	    }
	
	// End
        System.out.println("\n");
	}
		
    /* OUTPUT RULE LINKED LIST WITH ATTRIBUTE LITERALS AND DEFAULT */
    /** Outputs contents of rule linked list (if any), asuming that the list
    represents a set of CRs, such that (i) attribute numbers are replaced by
    attribute lirerals (phrases) and (ii) the last rule is the default rule. */

    public void outputCRsWithAttLiteralsAndDefault() {
	System.out.println("CLASSIFIER\n----------");
	
        // Check for empty rule list
	if (startRulelist==null) {
	    System.out.println("No classification rules generated!");
	    return;
	    }

	// Process list
        outputCRsWithAttLiteralsAndDefault(startRulelist);
	    
	// End
	System.out.println();
        }

    /** Continues process of outputting rules in binary tree as a list
    of attribute numbers.
    @param linkRuleNode the currentNode. */
    
    private void outputCRsWithAttLiteralsAndDefault(RuleNode linkRuleNode) {	
	// Process Tree
	if (linkRuleNode != null) {
	    // Left branch
            outputCRsWithAttLiteralsAndDefault(linkRuleNode.leftBranch);
	    // Node
	    outputCRwithAttLiteralsAndDefault(linkRuleNode);
	    // Right branch
            outputCRsWithAttLiteralsAndDefault(linkRuleNode.rightBranch);
	    }
	}
    
    /** Outputs first N records in rule linked list (if any), asuming that the 
    list represents a set of CRs, such that (i) attribute numbers are replaced 
    by attribute lirerals (phrases) and (ii) the last rule is the default rule. 
    @param numberOfRulesToOutput the number of rules to output.	*/

    public void outputCRsWithAttLiteralsAndDefault(int numberOfRulesToOutput) {
	System.out.println("CLASSIFIER (FIRST N RULES)\n--------------------");
	
        // Check for empty rule list
	if (startRulelist==null) {
	    System.out.println("NO CLASSIFICATION RULES GENERATED");
	    return;
	    }
	
	// Process list
	outputCRsWithAttLiteralsAndDefault(numberOfRulesToOutput,startRulelist);
	    
	// End
	System.out.println();
        }

    /** Continues process of outputting first N rules in binary tree as a list
    of attribute literals.
    @param numRtoOutput) the number of rule sto output
    @param linkRuleNode the currentNode. */
	
    private void outputCRsWithAttLiteralsAndDefault(int numRtoOutput,
    							RuleNode linkRuleNode) {	
	// Process node
        if (linkRuleNode != null) {
            // Check if at end
            if (numRtoOutput==linkRuleNode.ruleNumber+1) return;
	    // Left branch
            outputCRsWithAttLiteralsAndDefault(numRtoOutput,
	                                               linkRuleNode.leftBranch);
	    // Node
	    outputCRwithAttLiteralsAndDefault(linkRuleNode);
	    // Right branch
            outputCRsWithAttLiteralsAndDefault(numRtoOutput,
	                                              linkRuleNode.rightBranch);
	    }
	}
	
    /** Outputs a single classification rule with attribute literals.
    @param linkRuleNode the reference to the record in the rule list. */
    
    private void outputCRwithAttLiteralsAndDefault(RuleNode linkRuleNode) {
        // Output rule number
	System.out.print("(" + linkRuleNode.ruleNumber + ") ");
	    
	// Output antecedent
	if (linkRuleNode.ruleNumber==numRules) System.out.print("Default -> ");
	else {
	    outputItemSetWithAttLiterals(linkRuleNode.antecedent);
	    System.out.print(" -> ");
	    }
	    
	// Output class (first and only attribute in the rule consequent).
        String ClassLiteral = getClassLiteral(linkRuleNode.consequent[0]);
	System.out.print(ClassLiteral + " ");
        System.out.println(" " +
	    		   twoDecPlaces(linkRuleNode.confidenceForRule) + "%");
	}
	
	
    /* OUTPUT ITEMSET */  
    /** Outputs a given item set as a set of phrase literals. 
    @param itemSet the given item set. */
    
   private void outputItemSetWithAttLiterals(short[] itemSet) {
	// Check for empty set  	
	if (itemSet == null) System.out.print(" null "); 
	// Process
	else {
	    // Reconvert where input dataset has been reordered and possible 
	    // pruned.
	    short[] tempItemSet = reconvertItemSet(itemSet);
	    // Loop through item set elements
            int counter = 0;
	    for (int index=0;index<tempItemSet.length;index++) {
	        if (counter == 0) {
	    	    counter++;
		    System.out.print(" {");
		    }
	        else System.out.print(", ");
	        phraseBinTree.outputPhraseN(tempItemSet[index]);
		}
	    System.out.print("} ");
	    }
	}	
	
    /* OUTPUT PHRASE N */
    
    /** Outputs a particular phrase, identified by a sequential attribute 
    number, contained within the phrase bin tree (called from application code
    for diagnostic purposes only).
    @param num the given phrase id number. */
    
    public void outputPhraseN(int num) {
        phraseBinTree.outputPhraseN(num);
	}
    }