Attachment #64308 for bug #78063

View | Details | Raw Unified | Return to bug 78063
Collapse All | Expand All




 */
public class JavaTokenComparator implements ITokenComparator {
		
	private static final boolean DEBUG = false;
	private String fText;
	private boolean fShouldEscape= true;
	private int fCount;

		
		IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
		scanner.setSource(fText.toCharArray());
		int endPos= 0;
		try {
			int tokenType;
			while ((tokenType = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
				int start= scanner.getCurrentTokenStartPosition();
				int end= scanner.getCurrentTokenEndPosition()+1;
				// Comments are treated as a single token (see bug 78063)
				if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
						|| tokenType == ITerminalSymbols.TokenNameCOMMENT_BLOCK
						|| tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					int dl = getCommentDelimeterLength(tokenType);
					recordToken(start, dl);
					parseComment(start + dl, text.substring(start+dl, end), shouldEscape);
				} else {
					recordToken(start, end - start);
				}
				endPos= end;
				fCount++;
			}
			// workaround for #13907
			if (endPos < length) {
				fStarts[fCount]= endPos;
				fLengths[fCount]= length-endPos;
				fCount++;
			}
		} catch (InvalidInputException ex) {
			// We couldn't parse part of the input. Fall through and make the rest a single token
		}
		// workaround for #13907
		if (endPos < length) {
			recordToken(endPos, length-endPos);
		}
	}

	private void recordToken(int start, int length) {
		fStarts[fCount]= start;
		fLengths[fCount]= length;
		fCount++;
		if (DEBUG)
			System.out.println(fText.substring(start, start + length));
	}

	private void parseComment(int start, String commentText, boolean shouldEscape) {
		JavaTokenComparator subTokenizer = new JavaTokenComparator(commentText, shouldEscape);
		int count = subTokenizer.getRangeCount();
		for (int i = 0; i < count; i++) {
			int subStart = subTokenizer.getTokenStart(i);
			int subLength = subTokenizer.getTokenLength(i);
			recordToken(start + subStart, subLength);
		}
	}

	private int getCommentDelimeterLength(int tokenType) {
		if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC)
			return 3;
		return 2;
	}

	/**
	 * Returns the number of token in the string.

Return to bug 78063

Lines 24-29 Link Here

(-)ui/org/eclipse/jdt/internal/ui/compare/JavaTokenComparator.java (-13 / +44 lines)
24	*/	24	*/
25	public class JavaTokenComparator implements ITokenComparator {	25	public class JavaTokenComparator implements ITokenComparator {
26		26
		27	private static final boolean DEBUG = false;
27	private String fText;	28	private String fText;
28	private boolean fShouldEscape= true;	29	private boolean fShouldEscape= true;
29	private int fCount;	30	private int fCount;
Lines 47-72 Link Here
47		48
48	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace	49	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
49	scanner.setSource(fText.toCharArray());	50	scanner.setSource(fText.toCharArray());
		51	int endPos= 0;
50	try {	52	try {
51	int endPos= 0;	53	int tokenType;
52	while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {	54	while ((tokenType = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
53	int start= scanner.getCurrentTokenStartPosition();	55	int start= scanner.getCurrentTokenStartPosition();
54	int end= scanner.getCurrentTokenEndPosition()+1;	56	int end= scanner.getCurrentTokenEndPosition()+1;
55	fStarts[fCount]= start;	57	// Comments are treated as a single token (see bug 78063)
56	fLengths[fCount]= end - start;	58	if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
		59	\|\| tokenType == ITerminalSymbols.TokenNameCOMMENT_BLOCK
		60	\|\| tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		61	int dl = getCommentDelimeterLength(tokenType);
		62	recordToken(start, dl);
		63	parseComment(start + dl, text.substring(start+dl, end), shouldEscape);
		64	} else {
		65	recordToken(start, end - start);
		66	}
57	endPos= end;	67	endPos= end;
58	fCount++;
59	}
60	// workaround for #13907
61	if (endPos < length) {
62	fStarts[fCount]= endPos;
63	fLengths[fCount]= length-endPos;
64	fCount++;
65	}	68	}
66	} catch (InvalidInputException ex) {	69	} catch (InvalidInputException ex) {
67	// NeedWork	70	// We couldn't parse part of the input. Fall through and make the rest a single token
		71	}
		72	// workaround for #13907
		73	if (endPos < length) {
		74	recordToken(endPos, length-endPos);
		75	}
		76	}
		77
		78	private void recordToken(int start, int length) {
		79	fStarts[fCount]= start;
		80	fLengths[fCount]= length;
		81	fCount++;
		82	if (DEBUG)
		83	System.out.println(fText.substring(start, start + length));
		84	}
		85
		86	private void parseComment(int start, String commentText, boolean shouldEscape) {
		87	JavaTokenComparator subTokenizer = new JavaTokenComparator(commentText, shouldEscape);
		88	int count = subTokenizer.getRangeCount();
		89	for (int i = 0; i < count; i++) {
		90	int subStart = subTokenizer.getTokenStart(i);
		91	int subLength = subTokenizer.getTokenLength(i);
		92	recordToken(start + subStart, subLength);
68	}	93	}
69	}	94	}
		95
		96	private int getCommentDelimeterLength(int tokenType) {
		97	if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC)
		98	return 3;
		99	return 2;
		100	}
70		101
71	/**	102	/**
72	* Returns the number of token in the string.	103	* Returns the number of token in the string.