1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.StringReader;
7
8 import net.sourceforge.pmd.lang.LanguageVersion;
9 import net.sourceforge.pmd.lang.LanguageVersionHandler;
10 import net.sourceforge.pmd.lang.TokenManager;
11 import net.sourceforge.pmd.lang.ast.TokenMgrError;
12 import net.sourceforge.pmd.lang.cpp.ast.Token;
13
14 import org.apache.commons.io.IOUtils;
15
16 public class CPPTokenizer implements Tokenizer {
17
18 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
19 StringBuilder buffer = sourceCode.getCodeBuffer();
20 StringReader reader = null;
21 try {
22 LanguageVersionHandler languageVersionHandler = LanguageVersion.CPP.getLanguageVersionHandler();
23 reader = new StringReader(buffer.toString());
24 TokenManager tokenManager = languageVersionHandler.getParser(
25 languageVersionHandler.getDefaultParserOptions())
26 .getTokenManager(sourceCode.getFileName(), reader);
27 Token currentToken = (Token) tokenManager.getNextToken();
28 while (currentToken.image.length() > 0) {
29 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
30 currentToken = (Token) tokenManager.getNextToken();
31 }
32 tokenEntries.add(TokenEntry.getEOF());
33 System.err.println("Added " + sourceCode.getFileName());
34 } catch (TokenMgrError err) {
35 err.printStackTrace();
36 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
37 tokenEntries.add(TokenEntry.getEOF());
38 } finally {
39 IOUtils.closeQuietly(reader);
40 }
41 }
42 }