View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import java.io.Reader;
7   import java.io.StringReader;
8   
9   import org.apache.commons.io.IOUtils;
10  
11  import net.sourceforge.pmd.lang.LanguageRegistry;
12  import net.sourceforge.pmd.lang.LanguageVersionHandler;
13  import net.sourceforge.pmd.lang.TokenManager;
14  import net.sourceforge.pmd.lang.jsp.JspLanguageModule;
15  import net.sourceforge.pmd.lang.jsp.ast.Token;
16  import net.sourceforge.pmd.util.IOUtil;
17  
18  public class JSPTokenizer implements Tokenizer {
19  
20  	public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21  		StringBuilder buffer = sourceCode.getCodeBuffer();
22  		LanguageVersionHandler languageVersionHandler = 
23  				LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
24  		Reader reader = null;
25  
26  		try {
27  			reader = new StringReader(buffer.toString());
28  			reader = IOUtil.skipBOM(reader);
29  			TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
30  					.getTokenManager(sourceCode.getFileName(), reader);
31  			Token currentToken = (Token) tokenMgr.getNextToken();
32  
33  			while (currentToken.image.length() > 0) {
34  				tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
35  						currentToken.beginLine));
36  				currentToken = (Token) tokenMgr.getNextToken();
37  			}
38  		} finally {
39  			IOUtils.closeQuietly(reader);
40  		}
41  		tokenEntries.add(TokenEntry.getEOF());
42  	}
43  }