1
2
3
4
5 package net.sourceforge.pmd.cpd;
6
7 import static org.junit.Assert.assertEquals;
8 import static org.junit.Assert.assertNotEquals;
9
10 import java.util.List;
11
12 import org.junit.Before;
13 import org.junit.Test;
14
15 public class CsTokenizerTest {
16
17 private CsTokenizer tokenizer;
18
19 private Tokens tokens;
20
21 @Before
22 public void init() {
23 tokenizer = new CsTokenizer();
24 tokens = new Tokens();
25 TokenEntry.clearImages();
26 }
27
28 @Test
29 public void testSimpleClass() {
30 tokenizer.tokenize(toSourceCode("class Foo {}"), tokens);
31 assertEquals(5, tokens.size());
32 }
33
34 @Test
35 public void testSimpleClassDuplicatedTokens() {
36 tokenizer.tokenize(toSourceCode("class Foo { class Foo { } }"), tokens);
37 assertEquals(9, tokens.size());
38 List<TokenEntry> tokenList = tokens.getTokens();
39 assertEquals(tokenList.get(0).getIdentifier(), tokenList.get(3).getIdentifier());
40 assertEquals(tokenList.get(1).getIdentifier(), tokenList.get(4).getIdentifier());
41 assertEquals(tokenList.get(2).getIdentifier(), tokenList.get(5).getIdentifier());
42 assertEquals(tokenList.get(6).getIdentifier(), tokenList.get(7).getIdentifier());
43 }
44
45 @Test
46 public void testSimpleClassMethodMultipleLines() {
47 tokenizer.tokenize(toSourceCode(
48 "class Foo {\n"
49 + " public String foo(int a) {\n"
50 + " int i = a;\n"
51 + " return \"x\" + a;\n"
52 + " }\n"
53 + "}"), tokens);
54 assertEquals(22, tokens.size());
55 List<TokenEntry> tokenList = tokens.getTokens();
56 assertEquals(1, tokenList.get(0).getBeginLine());
57 assertEquals(2, tokenList.get(4).getBeginLine());
58 assertEquals(3, tokenList.get(11).getBeginLine());
59 }
60
61 @Test
62 public void testStrings() {
63 tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
64 assertEquals(5, tokens.size());
65 }
66
67 @Test
68 public void testOpenString() {
69 tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
70 assertEquals(5, tokens.size());
71 }
72
73
74 @Test
75 public void testCommentsIgnored1() {
76 tokenizer.tokenize(toSourceCode("class Foo { /* class * ** X */ }"), tokens);
77 assertEquals(5, tokens.size());
78 }
79
80 @Test
81 public void testCommentsIgnored2() {
82 tokenizer.tokenize(toSourceCode("class Foo { // class X /* aaa */ \n }"), tokens);
83 assertEquals(5, tokens.size());
84 }
85
86 @Test
87 public void testCommentsIgnored3() {
88 tokenizer.tokenize(toSourceCode("class Foo { /// class X /* aaa */ \n }"), tokens);
89 assertEquals(5, tokens.size());
90 }
91
92 @Test
93 public void testMoreTokens() {
94 tokenizer.tokenize(toSourceCode(
95 "class Foo {\n"
96 + " void bar() {\n"
97 + " int a = 1 >> 2; \n"
98 + " a += 1; \n"
99 + " a++; \n"
100 + " a /= 3e2; \n"
101 + " float f = -3.1; \n"
102 + " f *= 2; \n"
103 + " bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n"
104 + " }\n"
105 + "}"
106 ), tokens);
107 assertEquals(50, tokens.size());
108 }
109
110 @Test
111 public void testLineNumberAfterMultilineComment() {
112 tokenizer.tokenize(toSourceCode(
113 "/* This is a multiline comment \n"
114 + " * \n"
115 + " * Lorem ipsum dolor sit amet, \n"
116 + " * consectetur adipiscing elit \n"
117 + " */\n"
118 + "\n"
119 + "class Foo {\n"
120 + "\n"
121 + "}"
122 ), tokens);
123 assertEquals(5, tokens.size());
124 assertEquals(7, tokens.getTokens().get(0).getBeginLine());
125 }
126
127 @Test
128 public void testLineNumberAfterMultilineString() {
129 tokenizer.tokenize(toSourceCode(
130 "class Foo {\n"
131 + " void bar() {\n"
132 + " String query = \n"
133 + " @\"SELECT foo, bar\n"
134 + " FROM table \n"
135 + " WHERE id = 42\"; \n"
136 + " }\n"
137 + "}"
138 ), tokens);
139 assertEquals(16, tokens.size());
140 assertEquals(8, tokens.getTokens().get(14).getBeginLine());
141 }
142
143 @Test
144 public void testIgnoreUsingDirectives() {
145 tokenizer.setIgnoreUsings(true);
146 tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens);
147 assertNotEquals("using", tokens.getTokens().get(0).toString());
148 assertEquals(2, tokens.size());
149 }
150
151 @Test
152 public void testUsingStatementsAreNotIgnored() {
153 tokenizer.setIgnoreUsings(true);
154 tokenizer.tokenize(toSourceCode(
155 "using (Font font1 = new Font(\"Arial\", 10.0f)) {\n"
156 + " byte charset = font1.GdiCharSet;\n"
157 + "}\n"
158 ), tokens);
159 assertEquals("using", tokens.getTokens().get(0).toString());
160 }
161
162 private SourceCode toSourceCode(String source) {
163 return new SourceCode(new SourceCode.StringCodeLoader(source));
164 }
165 }