/* To quickly fix the public API of the scanner I think this should work fine: */ public int getCurrentTokenEndPosition(){ // CHANGED LINE: return this.currentPosition - 1; return this.currentPosition > eofPosition ? eofPosition-1 : currentPosition-1; } public char[] getCurrentTokenSource() { // Return the token REAL source (aka unicodes are precomputed) char[] result; if (this.withoutUnicodePtr != 0) // 0 is used as a fast test flag so the real first char is in position 1 System.arraycopy( this.withoutUnicodeBuffer, 1, result = new char[this.withoutUnicodePtr], 0, this.withoutUnicodePtr); else { int length; System.arraycopy( this.source, this.startPosition, // CHANGED LINE: result = new char[length = this.currentPosition - this.startPosition], result = new char[length = (this.currentPosition > eofPosition ? eofPosition : currentPosition) - this.startPosition], 0, length); } return result; }