/* -------------------------------------------------------------------------- */
/*                                                                            */
/*                         TRAINING SET DOCUMENT BASE                         */
/*                                                                            */
/*                                Frans Coenen                                */
/*                                                                            */
/*                          Tuesday 20 December 2005                          */
/*                                                                            */
/*                       Department of Computer Science                       */
/*                        The University of Liverpool                         */
/*                                                                            */
/* -------------------------------------------------------------------------- */

/* Class structure

SetDocumentBase
	|
    	+-- Training SetDocumentBase		*/
	
//package lucsKDD_ARM;

/** Class containing methods to describes a set of training documents in terms 
of references to words in the bin tree and to generate the phrase bin tree. Does
not produce the desired final  training set in terms of attribute numbers, this
is dome by methods in the PhraseBinTree class. Similarly methods to output the
training set (in what ever format) are contained in the PhraseBinTree class.
@author Frans Coenen
@version 20 December 2005   */

public class TrainingSetDocumentBase extends SetDocumentBase {

    /* ------------------------------- */
    /*                                 */
    /*              FIELDS             */
    /*                                 */
    /* ------------------------------- */

    /** Inner class describing document in terms of word bin tree nodes. */
    public class Document {
        /** List of words */
	private WordBinTreeNode[] content = null;	
	/** Class label */
	private short classLabel;
	}
		
    /** The document base (array of document objects) describing each document
    in terms of words. */
    public Document[] docBase = null;
    /** Content index for current document, */
    private int contentIndex = 0;
    
    /* ------------------------------------ */
    /*                                      */
    /*              CONSTRUCTORS            */
    /*                                      */
    /* ------------------------------------ */
    
    /** One argument constructor.
    @param numberOfDocs the number of documents in the training set document 
    base. */

    public TrainingSetDocumentBase(int numberOfDocs) {
	// Create instances of class Document.
        docBase = new Document[numberOfDocs];
	for(int index=0;index<docBase.length;index++) {
	    docBase[index] = new Document();
	    }
	}

    /* ------------------------------- */
    /*                                 */
    /*              METHODS            */
    /*                                 */
    /* ------------------------------- */

    /* --------------------------------------------------------- */
    /*              METHODS TO CONSTRUCT DOCUMENT BASE           */
    /* --------------------------------------------------------- */

    /** Intialises a content array for the given document.
    @param docNum the ID number of the current document.
    @param size the number of words and stop marks in the document   */

    public void createContentArray(int docNumber, int size) {
        docBase[docNumber].content = new WordBinTreeNode[size];
	contentIndex = 0;
	}

    /** Adds the given word (word bin tree node) to the document content array 
    (recorded as a set of word bin tree nodes).
    @param docNum the ID number of the current document.
    @param size the given words represented as a word bin tree node.   */  
   
    public void addWordToDocInDocBase(int docNumber, WordBinTreeNode word) {
    	docBase[docNumber].content[contentIndex] = word;
	contentIndex++;
	}

    /* ------------------------------------------------------------- */
    /*                   GENERATE PHRASE BIN TREE                    */
    /*                                                               */
    /* DELIMETERS = NOISE WORDS AND STOP MARKS                       */
    /* CONTENT    = DISTINGUSHING WORDS AND INCLUDED ORDINARY WORDS  */
    /* IGNORE     = EXCLUDED ORDINARY WORDS                          */
    /*                                                               */
    /* ------------------------------------------------------------- */
    
    /** Commences process of generating phrase bin tree. <P>Process each
    document in turn.
    @param phraseBinTree instance of the class PhraseBinTree.	*/
    
    public void genPhraseBinTree_DelSN_ContGO(PhraseBinTree phraseBinTree) {
        for (int index=0;index<docBase.length;index++) {
	    genPhraseBinTree_DelSN_ContGO(phraseBinTree,
	    				      index,docBase[index].content);
	    }
	}
    
    /** Continues process of generating phrase bin tree by processing a single
    document. <P> Phrase made up of ordinary words and a least one significant 
    word. Phrase delimited by stop mark and/or noise words.  
    @param phraseBinTree instance of the class PhraseBinTree.	
    @param docNum the current document number.
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects. 	*/
    
    private void genPhraseBinTree_DelSN_ContGO(PhraseBinTree phraseBinTree, 
                                       int docNum, WordBinTreeNode[] content) {
        String[] phrase               = null;
	boolean lookingForPhraseStart = true;
        boolean lookingForPhraseEnd   = false;
	String fstSigWordInPhrase     = null;
	int indexOfFstSigWordInPhrase = -1;
	
	// Process document word by word
        for (int index=0;index<content.length;index++) {
	    // Check for NOT delimiter (stop mark indictaed by "null" or noise 
	    // word)
	    if ((content[index]!=null) && (!content[index].isUpperNoiseWord) &&
	    		(!content[index].isLowerNoiseWord)) {
	        // Check if start of phrase
		if (lookingForPhraseStart) {
		     // First word in phrase
		     phrase                = new String[1];
                     phrase[0]             = content[index].word; 
                     lookingForPhraseStart = false;
                     lookingForPhraseEnd   = true;
		     }
		 // Otherwise part of phrase
		 else phrase = addToPhrase(phrase,content[index].word);
		 // Check for significant word (not ordinary word) 
		 if ((fstSigWordInPhrase==null) && 
		                                  (content[index].isSigWord)) {
		     fstSigWordInPhrase = content[index].word;
		     indexOfFstSigWordInPhrase = phrase.length-1;
		     }
		 }
	    // Otherwise found delimiter, if end of phrase and phrase contains 
	    // at least one significant word then add to phrase bin tree, 
	    // otherwise ignore
	    else {
		if (lookingForPhraseEnd) {
		    if (indexOfFstSigWordInPhrase>=0) {
			phraseBinTree.addToPhraseBinTree(phrase,docNum,
				fstSigWordInPhrase,indexOfFstSigWordInPhrase);
			}    
                     lookingForPhraseStart     = true;
                     lookingForPhraseEnd       = false;
		     fstSigWordInPhrase        = null;
		     indexOfFstSigWordInPhrase = -1;
		     }
		 }    
	    }
	
	// Got to end of document, may have a last phrase
	if (indexOfFstSigWordInPhrase>=0) 
	                             phraseBinTree.addToPhraseBinTree(phrase,
	                docNum,fstSigWordInPhrase,indexOfFstSigWordInPhrase); 
	}

    /** Append a word to a phrase so far.
    @param phrase the phrase so far.
    @param newWord the word to be appended.
    @return the newly concatinated phrase. */
    
    private String[] addToPhrase(String[] phrase, String newWord) {
        String[] newPhrase;
	
	// Empty phrase
	if (phrase==null) {
	    newPhrase    = new String[1];
	    newPhrase[0] = newWord;
	    }
	// Not empty    
	else {
	    newPhrase = new String[phrase.length+1];
	    // Copy
	    int index=0;
	    for (;index<phrase.length;index++) newPhrase[index] = phrase[index];
	    // Add new word 
	    newPhrase[index] = newWord;
	    }
	    
	// Return
	return(newPhrase);
	}
	
    /* ------------------------------------------------------------- */
    /*                   GENERATE PHRASE BIN TREE                    */
    /*                                                               */
    /* DELIMETERS = ORDINARY WORDS AND STOP MARKS                    */
    /* CONTENT    = DISTINGUSHING WORDS AND INCLUDED NOISE WORDS     */
    /* IGNORE     = EXCLUDED NOISE WORDS                             */
    /*                                                               */
    /* ------------------------------------------------------------- */
    
    /** Commences process of generating phrase bin tree. <P>Process each
    document in turn.
    @param phraseBinTree instance of the class PhraseBinTree.	*/
    
    public void genPhraseBinTree_DelSO_ContGN(PhraseBinTree phraseBinTree) {
        for (int index=0;index<docBase.length;index++) {
	    genPhraseBinTree_DelSO_ContGN(phraseBinTree,
	    				      index,docBase[index].content);
	    }
	}
    
    /** Continues process of generating phrase bin tree by processing a single
    document. <P> Phrase made up of noise words and a least one significant 
    word. Phrase delimited by stop mark and/or ordinary words.  
    @param phraseBinTree instance of the class PhraseBinTree.	
    @param docNum the current document number.
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects. 	*/
    
    private void genPhraseBinTree_DelSO_ContGN(PhraseBinTree phraseBinTree, 
                                        int docNum,WordBinTreeNode[] content) {
        String[] phrase               = null;
	boolean lookingForPhraseStart = true;
        boolean lookingForPhraseEnd   = false;
	String fstSigWordInPhrase     = null;
	int indexOfFstSigWordInPhrase = -1;
	
	// Process document word by word
        for (int index=0;index<content.length;index++) {
	    // Check for NOT delimiter (stop mark or ordinary word)
	    if ((content[index]!=null) && (!content[index].isOrdWord)) {
	        // Check if start of phrase
		if (lookingForPhraseStart) {
		     // First word in phrase
		     phrase                = null; 
                     lookingForPhraseStart = false;
                     lookingForPhraseEnd   = true;
		     }
		 // Otherwise part of phrase
		 phrase = addToPhrase(phrase,content[index].word); 
		 if ((fstSigWordInPhrase==null) &&
		                                  (content[index].isSigWord)) {
		     fstSigWordInPhrase        = content[index].word;
		     indexOfFstSigWordInPhrase = phrase.length-1;
		     }
		 }
	    // Otherwise found delimiter, if end of phrase and phrase contains 
	    // at least one significant word then add to phrase bin tree, 
	    // otherwise ignore
	    else {
		if (lookingForPhraseEnd) {
		    if (indexOfFstSigWordInPhrase>=0) {
			phraseBinTree.addToPhraseBinTree(phrase,docNum,
				fstSigWordInPhrase,indexOfFstSigWordInPhrase);
			}    
                    lookingForPhraseStart     = true;
                    lookingForPhraseEnd       = false;
		    fstSigWordInPhrase        = null;
		    indexOfFstSigWordInPhrase = -1;
		    }
		}    
	    }
	
	// Got to end of document, may have a last phrase
	if (indexOfFstSigWordInPhrase>=0) 
	                             phraseBinTree.addToPhraseBinTree(phrase,
	                docNum,fstSigWordInPhrase,indexOfFstSigWordInPhrase); 
	} 

    /* ------------------------------------------------------------- */
    /*                   GENERATE PHRASE BIN TREE                    */
    /*                                                               */
    /* DELIMETERS = NOISE WORDS AND STOP MARKS                       */
    /* CONTENT    = DISTINGUSHING WORDS AND INCLUDED ORDINARY WORDS  */
    /*              REPRESENTED AS WILD CARDS                        */
    /* IGNORE     = EXCLUDED ORDINARY WORDS                          */
    /*                                                               */
    /* ------------------------------------------------------------- */
    
    /** Commences process of generating phrase bin tree. <P>Process each
    document in turn.
    @param phraseBinTree instance of the class PhraseBinTree.	*/
    
    public void genPhraseBinTree_DelSN_ContGW(PhraseBinTree phraseBinTree) {
        for (int index=0;index<docBase.length;index++) {
	    genPhraseBinTree_DelSN_ContGW(phraseBinTree,
	    				      index,docBase[index].content);
	    }
	}
    
    /** Continues process of generating phrase bin tree by processing a single
    document. <P> Phrase made up of wild cards and a least one significant 
    word. Phrase delimited by stop mark and/or noise words.  
    @param phraseBinTree instance of the class PhraseBinTree.	
    @param docNum the current document number.
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects. 	*/
    
    private void genPhraseBinTree_DelSN_ContGW(PhraseBinTree phraseBinTree, int docNum,
    				WordBinTreeNode[] content) {
        String[] phrase               = null;
	boolean lookingForPhraseStart = true;
        boolean lookingForPhraseEnd   = false;
	String fstSigWordInPhrase     = null;
	int indexOfFstSigWordInPhrase = -1;

//System.out.println("NEW DOCUMENT\n***********************");	
	// Process document word by word
        for (int index=0;index<content.length;index++) {
	    // Check fot non-stop mark
	    if ((content[index]!=null) && (!content[index].isUpperNoiseWord) &&
	    		                  (!content[index].isLowerNoiseWord)) {
//System.out.print("\nNEW WORD \"" + content[index].word + "\"");
	        // Check for ordinary word
	        if (content[index].isOrdWord) {
//System.out.print(" Ordinary Word (add to phrase)");
	            phrase = addToPhrase(phrase,"?");
	            // Check if start of phrase
	            if (lookingForPhraseStart) { 
                        lookingForPhraseStart = false;
                        lookingForPhraseEnd   = true;
		        }
		    }
	        // Check for significant word
	        else if (content[index].isSigWord) {
//System.out.print(" Significant Word (add to phrase)");
	            phrase = addToPhrase(phrase,content[index].word);
		    // Check if start of phrase, in which case significant 
		    // word is first significant word found.
	            if (lookingForPhraseStart) { 
                        lookingForPhraseStart     = false;
                        lookingForPhraseEnd       = true;
		        fstSigWordInPhrase        = content[index].word;
		        indexOfFstSigWordInPhrase = phrase.length-1;
	                }
		    else if (fstSigWordInPhrase==null) {
		        fstSigWordInPhrase        = content[index].word;
		        indexOfFstSigWordInPhrase = phrase.length-1;
		        }
		    }
//System.out.print(" Noise Word");
		}
	    // Otherwise found delimiter, if end of phrase and phrase contains 
	    // at least one significant word then add to phrase bin tree, 
	    // otherwise ignore
	    else {
//System.out.print(" Stop Mark");
		if (lookingForPhraseEnd) { 
		    if (fstSigWordInPhrase!=null) {
//System.out.println("\nPHRASE = ");
//for(int i=0;i<phrase.length;i++) System.out.print(phrase[i] + " ");
			phraseBinTree.addToPhraseBinTree(phrase,docNum,
				fstSigWordInPhrase,indexOfFstSigWordInPhrase); 
			}  
//else System.out.print(" NON SIG. PHRASE");  
                    lookingForPhraseStart     = true;
                    lookingForPhraseEnd       = false;
		    fstSigWordInPhrase        = null;
		    indexOfFstSigWordInPhrase = -1;
		    phrase                    = null;
		    }
//else System.out.print(" NO PHRASE");
		}    
	    }
	
	// Got to end of document, may have a last phrase
	if (fstSigWordInPhrase!=null) 
	                             phraseBinTree.addToPhraseBinTree(phrase,
	                docNum,fstSigWordInPhrase,indexOfFstSigWordInPhrase); 
	}
	
    /* ------------------------------------------------------------- */
    /*                   GENERATE PHRASE BIN TREE                    */
    /*                                                               */
    /* DELIMETERS = ORDINARY WORDS AND STOP MARKS                    */
    /* CONTENT    = DISTINGUSHING WORDS AND INCLUDED NOISE WORDS     */
    /*              REPRESENTED AS WILD CARDS                        */
    /* IGNORE     = EXCLUDED NOISE WORDS                             */
    /*                                                               */
    /* ------------------------------------------------------------- */
    
    /** Commences process of generating phrase bin tree. <P>Process each
    document in turn.
    @param phraseBinTree instance of the class PhraseBinTree.	*/
    
    public void genPhraseBinTree_DelSO_ContGW(PhraseBinTree phraseBinTree) {
        for (int index=0;index<docBase.length;index++) {
	    genPhraseBinTree_DelSO_ContGW(phraseBinTree,
	    				      index,docBase[index].content);
	    }
	}
    
    /** Continues process of generating phrase bin tree by processing a single
    document. <P> Phrase made up of wild cards and a least one significant 
    word. Phrase delimited by stop mark and/or ordinary words.  
    @param phraseBinTree instance of the class PhraseBinTree.	
    @param docNum the current document number.
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects. 	*/
    
    private void genPhraseBinTree_DelSO_ContGW(PhraseBinTree phraseBinTree, 
                                        int docNum,WordBinTreeNode[] content) {
        String[] phrase               = null;
	boolean lookingForPhraseStart = true;
        boolean lookingForPhraseEnd   = false;
	String fstSigWordInPhrase     = null;
	int indexOfFstSigWordInPhrase = -1;

//System.out.println("NEW DOCUMENT\n***********************");	
	// Process document word by word
        for (int index=0;index<content.length;index++) {
	    // Check for NOT delimiter or ordinary word (not stop mark and not 
	    // ordinary word)
	    if ((content[index]!=null) && (!content[index].isOrdWord)) {
//System.out.print("\nNEW WORD \"" + content[index].word + "\". ");
	        // Found sig or noise word, first check if start of phrase
		if (lookingForPhraseStart) {
//System.out.print("Start of phrase. ");
		    // First word in phrase
		    phrase = null;
                    // Update flags
		    lookingForPhraseStart = false;
                    lookingForPhraseEnd   = true;
		    }
		// Update phrase and check for significant word
		if (content[index].isSigWord) {
//System.out.print(" Significant Word (add to phrase). ");
		    phrase = addToPhrase(phrase,content[index].word); 
		    // Check if first significant word
		    if (fstSigWordInPhrase==null) {
//System.out.print(" First Significant Word. ");
			fstSigWordInPhrase = content[index].word;
		        indexOfFstSigWordInPhrase = phrase.length-1;
			}
		    }
                else phrase = addToPhrase(phrase,"?"); 
		}
	    // Otherwise found delimiter, if end of phrase and phrase contains 
	    // at least one significant word then add to phrase bin tree, 
	    // otherwise ignore
	    else {
//System.out.print("STOP Found stop mark or ordinary word. ");
		if (lookingForPhraseEnd) {
//System.out.print("Looking for end of phrase. ");
		    if (indexOfFstSigWordInPhrase>=0) {
//System.out.print("Add Phrase:");
//for(int i=0;i<phrase.length;i++) System.out.print(" " + phrase[i]);
//System.out.print(" to tree.");
			phraseBinTree.addToPhraseBinTree(phrase,docNum,
				fstSigWordInPhrase,indexOfFstSigWordInPhrase);
			}    
                    // Reset
		    lookingForPhraseStart = true;
                    lookingForPhraseEnd   = false;
		    fstSigWordInPhrase        = null;
		    indexOfFstSigWordInPhrase = -1;
		    }
		}
//System.out.println();  
	    }
	
	// Got to end of document, may have a last phrase
	if (indexOfFstSigWordInPhrase>=0) 
	                             phraseBinTree.addToPhraseBinTree(phrase,
	                docNum,fstSigWordInPhrase,indexOfFstSigWordInPhrase); 
	} 

	
    /* ------------------------------------------- */
    /*              GET AND SET METHODS            */
    /* ------------------------------------------- */
    
    /** Gets number of documents in document base. 
    @return number of documents in document base.	*/

    public int getSizeOfDocBase() {
        return(docBase.length);
	}

    /** Gets class label for given document.
    @param docNumber the ID number for the given document. 
    @return the class lebel.	*/
    
    public short getClassLabel(int docNumber) {
        return(docBase[docNumber].classLabel);
	} 

    /** Sets the class label field for the relevant document in the document
    base.
    @param docNumber the ID number for the given document.
    @param label the class label  */

    public void setClassLabel(int docNumber, short label) {
        docBase[docNumber].classLabel = label;
	}

    /* ----------------------------------- */
    /*              DIAGNOSTICS            */
    /* ----------------------------------- */
    
    /** Commences process of outputting document base as a set of phrases. */
    
    public void outputDocBase() {
        // Process document base
	for (int index=0;index<docBase.length;index++) {
	    System.out.println("TRAINING SET DOCUMENT " + (index+1) + 
	                                            "\n================");
	    outputDocument(docBase[index].content);
	    System.out.println("\n");
	    }

	//End
	System.out.println("\n");
	} 

    /** Commences process of outputting first 10 documents in document base as 
    a set of phrases. */

    public void outputDocBase10() {
        // Process document base
	for (int index=0;index<docBase.length;index++) {
	    System.out.println("TRAINING SET DOCUMENT " + (index+1) + 
	                                            "\n================");
	    outputDocument(docBase[index].content);
	    System.out.println("\n");
	    // Check for stop (only want to output first 10 documents if more
	    // than 10 exist).
	    if (index==9) break;
	    }

	//End
	System.out.println("\n");
	} 

    /** Outputs document as a set of phrases. 
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects.  */
    
    private void outputDocument(WordBinTreeNode[] content) {
    	for (int index=0;index<content.length;index++) {
        if (content[index] != null)
                        System.out.print(content[index].word + " ");
	    else System.out.print("# ");
	    }
	} 

    /** Commences process of outputting document base with indicators for stop
    marks and pruned words' "#" indicates stop mark, "N-N" a noise word and 
    "O-O" an ordinary word. */
    
    public void outputDocBaseWithMarks() {  
        System.out.println("TRAINING DOC BASE");
        System.out.println("=================");
	System.out.println("\"#\" = stop mark, \"L-L\" = lower threshold " +
	             "noise word,\n\"U-U\" = upper threshold noise word, " +
		     "\"O-O\" = ordinary word,\n\"S-S\" = significant word.");
	
    	// Process document base
	for (int index=0;index<docBase.length;index++) {
	    System.out.println("DOCUMENT " + (index+1) + "\n================");
	    outputDocumentWithStopMarks(docBase[index].content);
	    System.out.println("\n");
	    }
	
	//End
	System.out.println("\n");
	} 

    /** Commences process of outputting first 10 documents in document base 
    with indicators for stop marks and pruned words' "#" indicates stop mark, "N-N" a noise word and 
    "O-O" an ordinary word. */
    
    public void outputDocBaseWithMarks10() {  
        System.out.println("TRAINING DOC BASE (FIRST 10 DOCUMENTS)");
        System.out.println("======================================");
	System.out.println("\"#\" = stop mark, \"L-L\" = lower threshold " +
	             "noise word,\n\"U-U\" = upper threshold noise word, " +
		     "\"O-O\" = ordinary word,\n\"S-S\" = significant word.");
	
    	// Process document base
	for (int index=0;index<docBase.length;index++) {
	    System.out.println("DOCUMENT " + (index+1) + "\n================");
	    outputDocumentWithStopMarks(docBase[index].content);
	    System.out.println("\n");
	    // Check for stop (only want to output first 10 documents if more
	    // than 10 exist).
	    if (index==9) break;
	    }
	
	//End
	System.out.println("\n");
	}
    
    /** Outputs document base with indicators for stop marks and pruned words, 
    "null" indicates stop mark and "###" a pruned word. 
    @param content the content of the document in terms of a set of references
    to WordBinTreeNode objects.  */
    
    private void outputDocumentWithStopMarks(WordBinTreeNode[] content) {
        for (int index=0;index<content.length;index++) {
	    if (content[index] == null) System.out.print("# ");
	    else System.out.print(content[index] + " ");
	    }
	}
    }