/* -------------------------------------------------------------------------- */
/*                                                                            */
/*                      KEY WORD BASED TEXT CLASSIFICATION                    */
/*                                                                            */
/*                                Frans Coenen                                */
/*                                                                            */
/*                           Friday 3 February 2006                           */
/*                                                                            */
/*                       Department of Computer Science                       */
/*                        The University of Liverpool                         */
/*                                                                            */
/* -------------------------------------------------------------------------- */

/* Class structure

AssocRuleMining
	|
	+-- TextMining
		|
		+-- KeyWordMining  */
	
//package lucsKDD_ARM;

// Java packages

import java.io.*;
import java.util.*;

/** Class containing methods to coordinate keyword identification from training 
set of documents, recast the training and testset in terms of the identified 
keywords and then produce a classifier from the training set tested on the test 
set. 
@author Frans Coenen
@version 3 February 2006. */

public class KeyWordMining extends TextMining { 

    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                               FIELDS                             */
    /*                                                                  */
    /* ---------------------------------------------------------------- */
    
    // None

    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                            CONSTRUCTORS                          */
    /*                                                                  */
    /* ---------------------------------------------------------------- */

    /** Constructor processes command line arguments.
    @param args the command line arguments (array of String instances). */

    public KeyWordMining(String[] args) {
        super(args);
	}
        
    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                              METHODS                             */
    /*                                                                  */
    /* ---------------------------------------------------------------- */
    
    /* ------------------------------------------------------ */
    /*                   TOP LEVEL METHODS                    */
    /* ------------------------------------------------------ */

    /** Top level method to: (i) reads the document base and stores in a bin
    tree structure and (ii) cause the bin tree to be processed to identify key 
    phrases.  
    @return true if OK to proceed and false otherwise.   */

    public boolean findKeyWordsInDB() throws IOException {
        final boolean OK_TO_PROCEED = true;
	
        System.out.println("START ANALYSING DOC BASE FOR KEYWORDS\n");
	if (docClasses==null) {
	    System.out.println("ERROR: No classes!");
	    System.exit(1);
	    }
	    
        // Load document base
        loadTrainingSetDocumentBase();
        if (outputNumDocsPerClassInTrainSetFlag) outputDocumentsPerClass(); 

        // Process document base and generate word bin tree provided there
	// are not too many one itemsets.
	wordBinTree.setDocsPerClass(docsPerClass);
	if (processTrainingSetDocumentBase()) {
	    // Process test document set
            loadTestSetDocumentBase();       
	    // Output attribute sets if requested
	    if (outputTrainSetAttNumFlag) {
	    	System.out.println("TRAINING DATA SET\n-----------------");	
	    	outputDataArray(dataArray);
	    	}
	    if (outputTestSetAttNumFlag) {
	    	System.out.println("TEST DATA SET\n-------------");	
            	outputDataArray(testDataArray);
		}
            return(OK_TO_PROCEED);
	    }
	else return(!OK_TO_PROCEED);
        }
	
    /** Commences processing of word bin tree (containing training set 
    document data) to ID: (1) noise words (words above/below noise thresholds),
    (2) ordinary words (words that do not serve to differentiate between
    classes), abd (3) significant words (words that do not serve to 
    differentiate between classes). <P> The desired key words are then the
    identified significant words and thus the input training data is 
    recast so that each document is represented according to whether the key
    word is contained in the document or not. Trainining set is stored as a 2-D
    array with rows representing records and columns attributes.
    @return true if OK to proceed and false otherwise.   */

    private boolean processTrainingSetDocumentBase() {
        final boolean NUM_ONE_ITEMSETS_LESS_THAN_MAX = true;
	
	// ID noise and insignificant words from bin tree
	System.out.println("ID NOISE, ORDINARY AND SIGNIFICANT WORDS IN " +
		           "WORDS BIN TREE\nLST = " + lowerNoiseThold + "% (" + 
		                        lowerNoiseDocThold + " docs), UST = " +
                                                      upperNoiseThold + "% (" +
		                         upperNoiseDocThold + " docs), SI = " + 
			                                    significanceIndex);
        System.out.println("Max # sig words " + maxNumSigWords);  
        System.out.println("================================================");
	
	// Prune word bin tree
	wordBinTree.pruneWordBinTree(lowerNoiseDocThold,upperNoiseDocThold,
                                                                      numRows);
								      								      
	// Generate potential significant words list
	wordBinTree.generatePotentialSigWordList();
	int numAttributes = wordBinTree.idSignificantWords(); 
     
	// Output options for word bin tree
	if (outputWordBinTreeFlag) wordBinTree.outputWordBinTree();
	if (outputWordBinTreeUNflag) wordBinTree.outputWordBinTreeUNW();
	if (outputWordBinTreeLNflag) wordBinTree.outputWordBinTreeLNW();
	if (outputWordBinTreeOWflag) wordBinTree.outputWordBinTreeOW();
	if (outputWordBinTreeSWflag) wordBinTree.outputWordBinTreeSW();
	if (outputCount1wrdsFlag) wordBinTree.outputWordsWithCount1();
	if (outputWordBinTreeStatsFlag) wordBinTree.outputWordBinTreeStats();
	if (outputSigWordsPerClassFlag) wordBinTree.outputSigWordsPerClass();
	if (outputPotSigWdsPerClassFlag) wordBinTree.outputPotSigWdsPerClass();
	if (outputTop10sigWordsListFlag) 
	                      wordBinTree.outputTopNpotSigWords(10,docClasses);	
	if (outputPotSigWordsListFlag) wordBinTree.outputPotentialSigWords();
	if (outputTrainSetAttMarkedFlag) docBase.outputDocBaseWithMarks();
	
	// Create training data array
	dataArray = wordBinTree.genTrainingDataArray(trainingSetIDs.length,
		  		                                      docBase);	
								      						   
	// Set numOneItemSets field contained in AssocRuleMining parent class.
	numOneItemSets = numAttributes+docClasses.length;
        if (numOneItemSets>MAX_NUM_ONE_ITEMSETS) {
	    System.out.println("Number of one itemsets (" + numOneItemSets +
	    		") exceeds maximum of " + MAX_NUM_ONE_ITEMSETS);
	    return(!NUM_ONE_ITEMSETS_LESS_THAN_MAX);
	    }
	else System.out.println("Number of one itemsets = " + numOneItemSets);
	if (outputTrainSetStatsFlag) outputTrainingSetStats(numOneItemSets,
		                                           docClasses.length);
	
        // Throw away training document base and return.
	docBase = null;
	return(NUM_ONE_ITEMSETS_LESS_THAN_MAX);
	}
	
    /** Commences process of loading test document base and storing in an array
    of arrays. */

    private void loadTestSetDocumentBase() throws IOException {  
        System.out.println("READ TEST SET");
        System.out.println("=============");
        if (testSetIDs==null) {
	    System.out.println("ERROR: No test set!");
	    System.exit(1);
	    }

        //  Define testDocBase object
	int numRowsInTestSet = testSetIDs.length;
	testDocBase = new TestSetDocumentBase(numRowsInTestSet);
System.out.println("Num. docs. in test set = " + numRowsInTestSet);

        //  loop through document base.
        for(int index=0;index<testSetIDs.length;index++) {
	    short docNumber = testSetIDs[index];
	    // Get size of document
            int size = getDocSize(docNumber);
//System.out.print(docNumber + " ");
//System.out.println("Test document " + docNumber + ", size = " + size);
	    testDocBase.setTestDocSize(size);
	    // Read document
	    readTestSetDoc(docNumber);
	    // Convert to attribute values.
            wordBinTree.convertTestDocToAtts(testDocBase,(short) index,
	                                                        classNumber);
//testDocBase.outputTestSetDocBase();
            }
//System.out.println();
	
	// Get test data array ready for DM, and throw away test document base. 
//testDocBase.outputTestDataArray(wordBinTree,this);
	testDataArray = testDocBase.getTestDataArray();
	testDocBase   = null;
	// Throw away word bin tree unless we wish to output rules with
	// attribute literals.
	//wordBinTree = null;
	}
	
    /* ---------------------------------------------------------------- */
    /*                                                                  */
    /*                       DIAGNOSTIC OUTPUT                          */
    /*                                                                  */
    /* ---------------------------------------------------------------- */
	
   /** Outputs training document base where attributes represent key word
   as set of word literals. */
    
    public void outputTraingSetKeyWords() {
        System.out.println("TRAINING ATTRIBUTE SET (KEY WORDS)");
	System.out.println("==================================");
	
	// Loop through records
	for (int docIndex=0;docIndex<dataArray.length;docIndex++) {
	    System.out.print("Training Doc #" + (docIndex+1) + ": ");		
	    outputTraingSetKeyWords(dataArray[docIndex]);
	    }
	
	// End
        System.out.println("\n");
	}

   /** Outputs first 10 pre-processed documwents in training document base 
   where attributes represent key word as set of word literals. */
    
    public void outputTraingSetKeyWords10() {
        System.out.println("TRAINING ATTRIBUTE SET (KEY WORDS) -- 1st 10 DOC");
	System.out.println("================================================");
	
	// Loop through records
	for (int docIndex=0;docIndex<dataArray.length;docIndex++) {
	    System.out.print("Training Doc #" + (docIndex+1) + ": ");
	    outputTraingSetKeyWords(dataArray[docIndex]);
	    // Check for stop (only want to output first 10 documents if more
	    // than 10 exist).
	    if (docIndex==9) break;
	    }
	
	// End
        System.out.println("\n");
	}
	
    /** Outputs training set document as a sequence of identified phrases (if
    any).
    @param docItemSet array of short integers representong identified phrases
    included in the given document. */
     	
    private void outputTraingSetKeyWords(short[] docItemSet) {
        // Test for empty document   
	if (docItemSet == null) System.out.print(" null ");
	
	// Otherwise process record
	else {
	    // Reconvert where input dataset has been reordered and possible 
	    // pruned.
	    short[] tempItemSet = reconvertItemSet(docItemSet);
	    // Attributes
	    int length = tempItemSet.length-1;
            // Punctuation marker
            boolean startOfRecord = true;
	    // Process record
	    for (int index=0;index<length;index++) {
	        if (startOfRecord) {
		    System.out.print(" {");
		    startOfRecord = false;
		    }
	        else System.out.print(", ");
	        wordBinTree.outputKeyWordN(tempItemSet[index]);
		}
	    if (length==0) System.out.print("{} ");
	    else System.out.print("} ");
	    // Add class
	    String ClassLiteral = getClassLiteral(tempItemSet[length]);
	    System.out.println(ClassLiteral + "\n");
            }
	}
	
    /** Outputs test set IDs as set of phrases. */
    
    public void outputTestSetKeyWords() {
        System.out.println("TEST ATTRIBUTE SET (KEY WORDS)");
	System.out.println("==============================");	
	// Loop through records
	for (int docIndex=0;docIndex<testDataArray.length;docIndex++) {
	    System.out.print("Training Doc #" + (docIndex+1) + ": ");	
	    // No record  
	    if (testDataArray[docIndex] == null) System.out.print(" null ");
	    // Otherwise process record
	    else {
	        // Reconvert where input dataset has been reordered and possible 
	        // pruned.
	        short[] tempItemSet = reconvertItemSet(dataArray[docIndex]);
	        // Attributes
	        int length = testDataArray[docIndex].length-1;
                // Punctuation marker
		boolean startOfRecord = true;
		// Process record
	        for (int index=0;index<length;index++) {
	            if (startOfRecord) {
		        System.out.print(" {");
			startOfRecord = false;
		        }
	            else System.out.print(", ");
	            wordBinTree.outputKeyWordN(testDataArray[docIndex][index]);
		    }
	        if (length==0) System.out.print("{} ");
		else System.out.print("} ");
	        // Add class
	        String ClassLiteral = 
		               getClassLiteral(testDataArray[docIndex][length]);
	        System.out.println(ClassLiteral + "\n");
		}
	    }
	
	// End
        System.out.println("\n");
	}
		
    /* OUTPUT RULE LINKED LIST WITH ATTRIBUTE LITERALS AND DEFAULT */
    /** Outputs contents of rule binary tree (if any), asuming that the tree
    nodes represent a set of CRs, such that (i) attribute numbers are replaced 
    by attribute lirerals (key words) and (ii) the last rule is the default 
    rule. */

    public void outputCRsWithAttLiteralsAndDefault() {
	System.out.println("CLASSIFIER\n----------");
	
        // Check for empty rule list
	if (startRulelist==null) {
	    System.out.println("NO CLASSIFICATION RULES GENERATED");
	    return;
	    }

	// Process list
        outputCRsWithAttLiteralsAndDefault(startRulelist);
	    
	// End
	System.out.println();
        }

    /** Continues process of outputting rules in binary tree as a list
    of attribute numbers.
    @param linkRuleNode the currentNode. */
    
    private void outputCRsWithAttLiteralsAndDefault(RuleNode linkRuleNode) {	
	// Process Tree
	if (linkRuleNode != null) {
	    // Left branch
            outputCRsWithAttLiteralsAndDefault(linkRuleNode.leftBranch);
	    // Node
	    outputCRwithAttLiteralsAndDefault(linkRuleNode);
	    // Right branch
            outputCRsWithAttLiteralsAndDefault(linkRuleNode.rightBranch);
	    }
	}

    /** Outputs first N records in rule linked list (if any), asuming that the 
    list represents a set of CRs, such that (i) attribute numbers are replaced 
    by attribute lirerals (key words) and (ii) the last rule is the default 
    rule. 
    @param numberOfRulesToOutput the number of rules to output.	*/

    public void outputCRsWithAttLiteralsAndDefault(int numberOfRulesToOutput) {
        System.out.println("CLASSIFIER (FIRST N RULES)\n----=---------------");
	
        // Check for empty rule list
	if (startRulelist==null) {
	    System.out.println("NO CLASSIFICATION RULES GENERATED");
	    return;
	    }
	
	// Process list
	outputCRsWithAttLiteralsAndDefault(numberOfRulesToOutput,startRulelist);
	    
	// End
	System.out.println();
        }

    /** Continues process of outputting first N rules in binary tree as a list
    of attribute literals.
    @param numRtoOutput) the number of rule sto output
    @param linkRuleNode the currentNode. */
	
    private void outputCRsWithAttLiteralsAndDefault(int numRtoOutput,
    							RuleNode linkRuleNode) {	
	// Process node
        if (linkRuleNode != null) {
            // Check if at end
            if (numRtoOutput==linkRuleNode.ruleNumber+1) return;
	    // Left branch
            outputCRsWithAttLiteralsAndDefault(numRtoOutput,
	                                               linkRuleNode.leftBranch);
	    // Node
	    outputCRwithAttLiteralsAndDefault(linkRuleNode);
	    // Right branch
            outputCRsWithAttLiteralsAndDefault(numRtoOutput,
	                                              linkRuleNode.rightBranch);
	    }
	}
    
    /** Outputs a single classification rule with attribute literals.
    @param number the rule number
    @param linkRuleNode the reference to the record in the rule list. */
    
    private void outputCRwithAttLiteralsAndDefault(RuleNode linkRuleNode) {
	// Output rule number
	System.out.print("(" + linkRuleNode.ruleNumber + ") ");
	    
	// Output antecedent
	if (linkRuleNode.ruleNumber==numRules) System.out.print("Default -> ");
	else {
	    outputItemSetWithAttWordLiterals(linkRuleNode.antecedent);
	    System.out.print(" -> ");
	    }
		
	// Output class (first and only attribute in the rule consequent).
        String ClassLiteral = getClassLiteral(linkRuleNode.consequent[0]);
	System.out.print(ClassLiteral + " ");
        System.out.println(" " +
	    		twoDecPlaces(linkRuleNode.confidenceForRule) + "%");
	}

    /* OUTPUT ITEMSET */  
    /** Outputs a given item set as a set of word literals. 
    @param itemSet the given item set. */
    
   private void outputItemSetWithAttWordLiterals(short[] itemSet) {
	// Check for empty set  	
	if (itemSet == null) System.out.print(" null "); 
	// Process
	else {
	    // Reconvert where input dataset has been reordered and possible 
	    // pruned.
	    short[] tempItemSet = reconvertItemSet(itemSet);
	    // Loop through item set elements
            int counter = 0;
	    for (int index=0;index<tempItemSet.length;index++) {
	        if (counter == 0) {
	    	    counter++;
		    System.out.print(" {");
		    }
	        else System.out.print(", ");
	        wordBinTree.outputKeyWordN(tempItemSet[index]);
		}
	    System.out.print("} ");
	    }
	}	
    }