Attachment 168076 Details for Bug 312249 – Unicode support patch

[patch] Unicode support patch

patch.txt (text/plain), 89.02 KB, created by Jeffrey Overbey

on 2010-05-11 22:37:40 EDT

(hide)

Description:

Filename:

MIME Type:

Creator: Jeffrey Overbey

Created: 2010-05-11 22:37:40 EDT

Size: 89.02 KB

patch

obsolete

>### Eclipse Workspace Patch 1.0
>#P org.eclipse.photran.cdtinterface
>Index: model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.cdtinterface/model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java,v
>retrieving revision 1.6
>diff -u -r1.6 SimpleFortranModelBuilder.java
>--- model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java	2 Nov 2009 21:19:03 -0000	1.6
>+++ model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java	12 May 2010 02:33:56 -0000
>@@ -10,8 +10,8 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.core.model;
> 
>-import java.io.ByteArrayInputStream;
>-import java.io.InputStream;
>+import java.io.Reader;
>+import java.io.StringReader;
> import java.util.ArrayList;
> import java.util.HashMap;
> import java.util.Map;
>@@ -73,7 +73,7 @@
> {
> this.newElements = new HashMap();
> 
>- InputStream in = new ByteArrayInputStream(translationUnit.getBuffer().getContents().getBytes());
>+ Reader in = new StringReader(translationUnit.getBuffer().getContents());
> String filename = translationUnit.getElementName();
> boolean wasSuccessful = true;
> 
>#P org.eclipse.photran.cdtinterface.vpg
>Index: model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.cdtinterface.vpg/model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java,v
>retrieving revision 1.15
>diff -u -r1.15 FortranModelBuilder.java
>--- model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java	12 Mar 2010 16:47:05 -0000	1.15
>+++ model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java	12 May 2010 02:34:01 -0000
>@@ -1,6 +1,6 @@
> package org.eclipse.photran.internal.core.model;
> 
>-import java.io.ByteArrayInputStream;
>+import java.io.StringReader;
> import java.util.HashMap;
> import java.util.Map;
> 
>@@ -74,7 +74,7 @@
> SourceForm sourceForm = determineSourceForm(file);
> String filename = determineFilename(file);
> lexer = LexerFactory.createLexer(
>- new ByteArrayInputStream(translationUnit.getBuffer().getContents().getBytes()),
>+ new StringReader(translationUnit.getBuffer().getContents()),
> file,
> filename,
> sourceForm,
>#P org.eclipse.photran.core
>Index: lexer/FixedFormLexerPhase1.flex
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex,v
>retrieving revision 1.15
>diff -u -r1.15 FixedFormLexerPhase1.flex
>--- lexer/FixedFormLexerPhase1.flex	8 Apr 2010 01:02:05 -0000	1.15
>+++ lexer/FixedFormLexerPhase1.flex	12 May 2010 02:34:09 -0000
>@@ -34,7 +34,7 @@
> package org.eclipse.photran.internal.core.lexer;
> 
> import java.util.regex.Pattern;
>-import java.io.InputStream;
>+import java.io.Reader;
> import org.eclipse.core.resources.IFile;
> 
> @SuppressWarnings("unused")
>@@ -58,8 +58,8 @@
> 
> private TokenFactory tokenFactory;
> 
>- public FixedFormLexerPhase1(InputStream in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
>- this(new LineAppendingInputStream(in));
>+ public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
>+ this(new LineAppendingReader(in));
> this.prepass=_prepass;
> this.tokenFactory = tokenFactory;
> }
>@@ -188,7 +188,7 @@
> 
> private String filename = "<stdin>";
> 
>- public FixedFormLexerPhase1(java.io.InputStream in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
>+ public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
> {
> this(in, _prepass, tokenFactory);
> this.lastTokenFile = new FileOrIFile(file);
>Index: lexer/FreeFormLexerPhase1.flex
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/FreeFormLexerPhase1.flex,v
>retrieving revision 1.14
>diff -u -r1.14 FreeFormLexerPhase1.flex
>--- lexer/FreeFormLexerPhase1.flex	5 Mar 2010 16:47:00 -0000	1.14
>+++ lexer/FreeFormLexerPhase1.flex	12 May 2010 02:34:10 -0000
>@@ -27,8 +27,10 @@
> */
> 
> package org.eclipse.photran.internal.core.lexer;
>+
> import java.io.FileNotFoundException;
> import java.io.IOException;
>+import java.io.Reader;
> import org.eclipse.core.resources.IFile;
> 
> @SuppressWarnings("unused")
>@@ -120,9 +122,9 @@
> private String filename = "<stdin>";
> protected TokenFactory tokenFactory;
> 
>- public FreeFormLexerPhase1(java.io.InputStream in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
>+ public FreeFormLexerPhase1(Reader in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
> {
>- this(new LineAppendingInputStream(in));
>+ this(new LineAppendingReader(in));
> this.lastTokenFile = new FileOrIFile(file);
> this.filename = filename;
> this.tokenFactory = tokenFactory;
>Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java,v
>retrieving revision 1.16
>diff -u -r1.16 FixedFormLexerPhase1.java
>--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java	8 Apr 2010 01:02:05 -0000	1.16
>+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java	12 May 2010 02:34:13 -0000
>@@ -1,4 +1,4 @@
>-/* The following code was generated by JFlex 1.4.1 on 3/18/10 4:23 PM */
>+/* The following code was generated by JFlex 1.4.1 on 5/11/10 8:56 PM */
> 
> /*******************************************************************************
> * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>@@ -36,7 +36,7 @@
> package org.eclipse.photran.internal.core.lexer;
> 
> import java.util.regex.Pattern;
>-import java.io.InputStream;
>+import java.io.Reader;
> import org.eclipse.core.resources.IFile;
> 
> @SuppressWarnings("unused")
>@@ -44,7 +44,7 @@
> /**
> * This class is a scanner generated by 
> * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
>- * on 3/18/10 4:23 PM from the specification file
>+ * on 5/11/10 8:56 PM from the specification file
> * <tt>FixedFormLexerPhase1.flex</tt>
> */
> class FixedFormLexerPhase1 implements ILexer {
>@@ -916,8 +916,8 @@
> 
> private TokenFactory tokenFactory;
> 
>- public FixedFormLexerPhase1(InputStream in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
>- this(new LineAppendingInputStream(in));
>+ public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
>+ this(new LineAppendingReader(in));
> this.prepass=_prepass;
> this.tokenFactory = tokenFactory;
> }
>@@ -1046,7 +1046,7 @@
> 
> private String filename = "<stdin>";
> 
>- public FixedFormLexerPhase1(java.io.InputStream in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
>+ public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
> {
> this(in, _prepass, tokenFactory);
> this.lastTokenFile = new FileOrIFile(file);
>Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java,v
>retrieving revision 1.7
>diff -u -r1.7 FixedFormLexerPhase2.java
>--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java	18 Mar 2010 20:00:59 -0000	1.7
>+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java	12 May 2010 02:34:13 -0000
>@@ -11,7 +11,7 @@
> package org.eclipse.photran.internal.core.lexer;
> 
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> 
>@@ -29,12 +29,13 @@
> 
> private IToken nextToken = null;
> 
>- public FixedFormLexerPhase2(InputStream in, IFile file, String filename, TokenFactory tokenFactory)
>+ public FixedFormLexerPhase2(Reader in, IFile file, String filename, TokenFactory tokenFactory)
> {
>- in = new LineAppendingInputStream(in);
>- final FixedFormLexerPrepass prepass = new FixedFormLexerPrepass(in);
>- InputStream prepassReader = new InputStream()
>+ final Reader input = new LineAppendingReader(in);
>+ final FixedFormLexerPrepass prepass = new FixedFormLexerPrepass(input);
>+ Reader prepassReader = new SingleCharReader()
> {
>+ @Override
> public int read() throws IOException
> {
> try
>@@ -47,6 +48,12 @@
> return -1;
> }
> }
>+ 
>+ @Override
>+ public void close() throws IOException
>+ {
>+ input.close();
>+ }
> };
> 
> FixedFormLexerPhase1 fixedLexer1 = new FixedFormLexerPhase1(prepassReader, prepass, file, filename, tokenFactory);
>Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java,v
>retrieving revision 1.8
>diff -u -r1.8 FixedFormLexerPrepass.java
>--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java	27 Apr 2010 19:42:48 -0000	1.8
>+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java	12 May 2010 02:34:14 -0000
>@@ -64,9 +64,9 @@
> 		this.in = new OffsetLineReader(in);
> 	}
> 	
>-	public FixedFormLexerPrepass(InputStream in) {
>-		this(new InputStreamReader(in));
>-	}
>+//	public FixedFormLexerPrepass(InputStream in) {
>+//		this(new InputStreamReader(in));
>+//	}
> 	
> 	public int getLine(int absCharPos) {
> 		if (absCharPos<0) return 0;
>Index: lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java,v
>retrieving revision 1.16
>diff -u -r1.16 FreeFormLexerPhase1.java
>--- lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java	5 Mar 2010 16:46:59 -0000	1.16
>+++ lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java	12 May 2010 02:34:20 -0000
>@@ -1,4 +1,4 @@
>-/* The following code was generated by JFlex 1.4.1 on 3/5/10 10:46 AM */
>+/* The following code was generated by JFlex 1.4.1 on 5/11/10 8:56 PM */
> 
> /*******************************************************************************
> * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>@@ -29,8 +29,10 @@
> */
> 
> package org.eclipse.photran.internal.core.lexer;
>+
> import java.io.FileNotFoundException;
> import java.io.IOException;
>+import java.io.Reader;
> import org.eclipse.core.resources.IFile;
> 
> @SuppressWarnings("unused")
>@@ -38,7 +40,7 @@
> /**
> * This class is a scanner generated by 
> * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
>- * on 3/5/10 10:46 AM from the specification file
>+ * on 5/11/10 8:56 PM from the specification file
> * <tt>FreeFormLexerPhase1.flex</tt>
> */
> public class FreeFormLexerPhase1 implements ILexer {
>@@ -2517,9 +2519,9 @@
> private String filename = "<stdin>";
> protected TokenFactory tokenFactory;
> 
>- public FreeFormLexerPhase1(java.io.InputStream in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
>+ public FreeFormLexerPhase1(Reader in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
> {
>- this(new LineAppendingInputStream(in));
>+ this(new LineAppendingReader(in));
> this.lastTokenFile = new FileOrIFile(file);
> this.filename = filename;
> this.tokenFactory = tokenFactory;
>Index: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java,v
>retrieving revision 1.3
>diff -u -r1.3 LineAppendingInputStream.java
>--- lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java	3 Nov 2009 16:11:00 -0000	1.3
>+++ lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java	12 May 2010 02:34:20 -0000
>@@ -36,6 +36,7 @@
> this.currentInputStream = originalInputStream;
> }
> 
>+ @Override
> public int read() throws IOException
> {
> int result = currentInputStream.read();
>@@ -44,7 +45,6 @@
> {
> currentInputStream = eolInputStream;
> result = currentInputStream.read();
>- currentInputStream.available();
> }
> else
> {
>@@ -64,6 +64,7 @@
> private static final byte[] EOL = System.getProperty("line.separator").getBytes();
> private int eolByte = 0;
> 
>+ @Override
> public int read() throws IOException
> {
> return eolByte < EOL.length ? EOL[eolByte++] : -1;
>Index: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
>===================================================================
>RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
>diff -N lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,84 @@
>+/*******************************************************************************
>+ * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ *
>+ * Contributors:
>+ * UIUC - Initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.photran.internal.core.lexer;
>+
>+import java.io.IOException;
>+import java.io.Reader;
>+
>+/**
>+ * Wraps another <code>Reader</code>, and, if the wrapped stream does not end
>+ * with an end-of-line character, appends the OS-dependent end-of-line sequence to
>+ * the end of the stream.
>+ * 
>+ * @author Jeff Overbey
>+ */
>+public final class LineAppendingReader extends SingleCharReader
>+{
>+ private Reader originalReader;
>+ private Reader eolReader;
>+ private Reader currentReader;
>+ 
>+ private int lastChar = -1;
>+ 
>+ public LineAppendingReader(Reader streamToWrap)
>+ {
>+ this.originalReader = streamToWrap;
>+ this.eolReader = new EOLReader();
>+ 
>+ this.currentReader = originalReader;
>+ }
>+
>+ @Override
>+ public int read() throws IOException
>+ {
>+ int result = currentReader.read();
>+ 
>+ if (result == -1 && currentReader == originalReader && !isEOL(lastChar))
>+ {
>+ currentReader = eolReader;
>+ result = currentReader.read();
>+ }
>+ else
>+ {
>+ lastChar = result;
>+ }
>+ 
>+ return result;
>+ }
>+
>+ private boolean isEOL(int c)
>+ {
>+ return c =='\n' || c == '\r';
>+ }
>+
>+ @Override
>+ public void close() throws IOException
>+ {
>+ originalReader.close();
>+ }
>+
>+ private static class EOLReader extends SingleCharReader
>+ {
>+ private static final byte[] EOL = System.getProperty("line.separator").getBytes();
>+ private int eolByte = 0;
>+
>+ @Override
>+ public int read() throws IOException
>+ {
>+ return eolByte < EOL.length ? EOL[eolByte++] : -1;
>+ }
>+
>+ @Override
>+ public void close() throws IOException
>+ {
>+ }
>+ }
>+}
>Index: lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
>===================================================================
>RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
>diff -N lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
>--- lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java	16 Aug 2007 22:58:21 -0000	1.2
>+++ /dev/null	1 Jan 1970 00:00:00 -0000
>@@ -1,260 +0,0 @@
>-/*******************************************************************************
>- * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
>- * All rights reserved. This program and the accompanying materials
>- * are made available under the terms of the Eclipse Public License v1.0
>- * which accompanies this distribution, and is available at
>- * http://www.eclipse.org/legal/epl-v10.html
>- *
>- * Contributors:
>- * UIUC - Initial API and implementation
>- *******************************************************************************/
>-package org.eclipse.photran.internal.core.lexer;
>-
>-import java.io.IOException;
>-import java.io.InputStream;
>-
>-/**
>- * Java's <code>BufferedReader</code> can read a line of text from another
>- * reader, but it omits the end-of-line character(s). This one doesn't.
>- * Subclass if necessary.
>- * 
>- * @author Jeff Overbey
>- */
>-public final class LineInputStream extends InputStream implements CharSequence
>-{
>- protected InputStream in;
>- 
>- protected String filename;
>- protected int startOffset;
>- protected int startLineNum;
>- protected int restartLine = -1;
>- protected int restartOffset = -1;
>- 
>- protected char[] currentLine = new char[4096];
>- protected int currentLineLength = -1;
>- protected int offsetInCurrentLine = -1;
>- protected int offsetInStream = 0;
>- protected int lineNumInCurrentStream = 0;
>- protected boolean atEOF = false;
>-
>- public LineInputStream(InputStream readFrom, String filename) throws IOException
>- {
>- this(readFrom, filename, 0, 0);
>- }
>-
>- public LineInputStream(InputStream readFrom, String filename, int startOffset, int startLineNum) throws IOException
>- {
>- this.in = readFrom;
>- this.filename = filename;
>- this.startOffset = startOffset;
>- this.startLineNum = startLineNum;
>- advanceToNextLine();
>- }
>-
>- /**
>- * THIS IS EXPENSIVE -- The text of the line is COPIED into a <code>String</code>.
>- * @return
>- * @throws IOException
>- */
>- public String currentLine() throws IOException
>- {
>- ensureLineLoaded();
>- return atEOF() ? null : String.valueOf(currentLine, offsetInCurrentLine, currentLineLength-offsetInCurrentLine);
>- }
>- 
>- public boolean atBOL() throws IOException
>- {
>- ensureLineLoaded();
>- return offsetInCurrentLine == 0;
>- }
>- 
>- public boolean atEOF() throws IOException
>- {
>- return offsetInCurrentLine >= currentLineLength && atEOF;
>- }
>-
>- protected void ensureLineLoaded() throws IOException
>- {
>- if (!atEOF && offsetInCurrentLine >= currentLineLength)
>- advanceToNextLine();
>- }
>-
>- protected int peek() throws IOException
>- {
>- return atEOF() ? -1 : currentLine[offsetInCurrentLine];
>- }
>-
>- public void advanceToNextLine() throws IOException
>- {
>- int c;
>- 
>- offsetInCurrentLine = 0;
>- offsetInStream += Math.max(currentLineLength-offsetInCurrentLine, 0);
>- 
>- if (atEOF) return;
>- 
>- lineNumInCurrentStream++;
>- 
>- if (currentLineLength < 0)
>- {
>- // First attempt to read a line
>- currentLineLength = 0;
>- }
>- else
>- {
>- // Start with the extra character from the last read
>- currentLine[0] = currentLine[currentLineLength];
>- currentLineLength = 1;
>- }
>- 
>- for (c = in.read(); c != -1 && c != '\n' && c != '\r'; c = in.read())
>- append(c);
>- 
>- if (c == -1) append('\n'); // Always end files with a newline
>- 
>- for ( ; c != -1 && (c == '\n' || c == '\r'); c = in.read())
>- append(c);
>- 
>- if (c == -1) atEOF = true;
>-
>- // To find the end of the line, we had to read an extra character; don't lose it!
>- currentLine[currentLineLength] = (char)c;
>- }
>- 
>- private void append(int c)
>- {
>- if (currentLineLength + 1 == currentLine.length)
>- {
>- char[] newCurrentLine = new char[currentLine.length*2];
>- System.arraycopy(currentLine, 0, newCurrentLine, 0, currentLine.length);
>- currentLine = newCurrentLine;
>- }
>- 
>- currentLine[currentLineLength++] = (char)c;
>- }
>- 
>- public String getFilename()
>- {
>- return filename;
>- }
>- 
>- public int getFileOffsetOfNextRead()
>- {
>- return offsetInStream;
>- }
>- 
>- public int getFileLineOfNextRead()
>- {
>- return lineNumInCurrentStream;
>- }
>- 
>- public int getTotalOffsetOfNextRead()
>- {
>- return startOffset + offsetInStream;
>- }
>- 
>- public int getTotalLineOfNextRead()
>- {
>- return startLineNum + lineNumInCurrentStream;
>- }
>-
>- public int getStartLine()
>- {
>- return startLineNum;
>- }
>-
>- public int getStartOffset()
>- {
>- return startOffset;
>- }
>-
>- public int getRestartLine()
>- {
>- return restartLine;
>- }
>-
>- public void setRestartLine(int restartLine)
>- {
>- this.restartLine = restartLine;
>- }
>-
>- public int getRestartOffset()
>- {
>- return restartOffset;
>- }
>-
>- public void setRestartOffset(int restartOffset)
>- {
>- this.restartOffset = restartOffset;
>- }
>- 
>- ///////////////////////////////////////////////////////////////////////////
>- // INPUTSTREAM IMPLEMENTATION
>- ///////////////////////////////////////////////////////////////////////////
>-
>- public int read() throws IOException
>- {
>- ensureLineLoaded();
>- int result = peek();
>- if (result >= 0)
>- {
>- offsetInCurrentLine++;
>- offsetInStream++;
>- }
>- return result;
>- }
>-
>- ///////////////////////////////////////////////////////////////////////////
>- // CHARSEQUENCE IMPLEMENTATION
>- ///////////////////////////////////////////////////////////////////////////
>- 
>- public char charAt(int index)
>- {
>- return currentLine[offsetInCurrentLine+index];
>- }
>-
>- public int length()
>- {
>- return currentLineLength-offsetInCurrentLine;
>- }
>-
>- public CharSequence subSequence(final int start, final int end)
>- {
>- if (start < offsetInCurrentLine || end >= length())
>- throw new IllegalArgumentException();
>- 
>- return new SubSequence(start, end);
>- }
>- 
>- private final class SubSequence implements CharSequence
>- {
>- private final int start;
>- private final int end;
>-
>- private SubSequence(int start, int end)
>- {
>- this.start = start;
>- this.end = end;
>- }
>-
>- public char charAt(int index)
>- {
>- return currentLine[offsetInCurrentLine+start];
>- }
>-
>- public int length()
>- {
>- return end-start;
>- }
>-
>- public CharSequence subSequence(int subseqStart, int subseqEnd)
>- {
>- return new SubSequence(start+subseqStart, start+subseqEnd);
>- }
>- 
>- public String toString()
>- {
>- return String.valueOf(currentLine, start, end-start);
>- }
>- }
>-}
>Index: lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
>===================================================================
>RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
>diff -N lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ lexer/org/eclipse/photran/internal/core/lexer/LineReader.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,266 @@
>+/*******************************************************************************
>+ * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ *
>+ * Contributors:
>+ * UIUC - Initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.photran.internal.core.lexer;
>+
>+import java.io.IOException;
>+import java.io.Reader;
>+
>+/**
>+ * Java's <code>BufferedReader</code> can read a line of text from another
>+ * reader, but it omits the end-of-line character(s). This one doesn't.
>+ * Subclass if necessary.
>+ * 
>+ * @author Jeff Overbey
>+ */
>+public final class LineReader extends SingleCharReader implements CharSequence
>+{
>+ protected Reader in;
>+ 
>+ protected String filename;
>+ protected int startOffset;
>+ protected int startLineNum;
>+ protected int restartLine = -1;
>+ protected int restartOffset = -1;
>+ 
>+ protected char[] currentLine = new char[4096];
>+ protected int currentLineLength = -1;
>+ protected int offsetInCurrentLine = -1;
>+ protected int offsetInStream = 0;
>+ protected int lineNumInCurrentStream = 0;
>+ protected boolean atEOF = false;
>+
>+ public LineReader(Reader readFrom, String filename) throws IOException
>+ {
>+ this(readFrom, filename, 0, 0);
>+ }
>+
>+ public LineReader(Reader readFrom, String filename, int startOffset, int startLineNum) throws IOException
>+ {
>+ this.in = readFrom;
>+ this.filename = filename;
>+ this.startOffset = startOffset;
>+ this.startLineNum = startLineNum;
>+ advanceToNextLine();
>+ }
>+
>+ /**
>+ * THIS IS EXPENSIVE -- The text of the line is COPIED into a <code>String</code>.
>+ * @return
>+ * @throws IOException
>+ */
>+ public String currentLine() throws IOException
>+ {
>+ ensureLineLoaded();
>+ return atEOF() ? null : String.valueOf(currentLine, offsetInCurrentLine, currentLineLength-offsetInCurrentLine);
>+ }
>+ 
>+ public boolean atBOL() throws IOException
>+ {
>+ ensureLineLoaded();
>+ return offsetInCurrentLine == 0;
>+ }
>+ 
>+ public boolean atEOF() throws IOException
>+ {
>+ return offsetInCurrentLine >= currentLineLength && atEOF;
>+ }
>+
>+ protected void ensureLineLoaded() throws IOException
>+ {
>+ if (!atEOF && offsetInCurrentLine >= currentLineLength)
>+ advanceToNextLine();
>+ }
>+
>+ protected int peek() throws IOException
>+ {
>+ return atEOF() ? -1 : currentLine[offsetInCurrentLine];
>+ }
>+
>+ public void advanceToNextLine() throws IOException
>+ {
>+ int c;
>+ 
>+ offsetInCurrentLine = 0;
>+ offsetInStream += Math.max(currentLineLength-offsetInCurrentLine, 0);
>+ 
>+ if (atEOF) return;
>+ 
>+ lineNumInCurrentStream++;
>+ 
>+ if (currentLineLength < 0)
>+ {
>+ // First attempt to read a line
>+ currentLineLength = 0;
>+ }
>+ else
>+ {
>+ // Start with the extra character from the last read
>+ currentLine[0] = currentLine[currentLineLength];
>+ currentLineLength = 1;
>+ }
>+ 
>+ for (c = in.read(); c != -1 && c != '\n' && c != '\r'; c = in.read())
>+ append(c);
>+ 
>+ if (c == -1) append('\n'); // Always end files with a newline
>+ 
>+ for ( ; c != -1 && (c == '\n' || c == '\r'); c = in.read())
>+ append(c);
>+ 
>+ if (c == -1) atEOF = true;
>+
>+ // To find the end of the line, we had to read an extra character; don't lose it!
>+ currentLine[currentLineLength] = (char)c;
>+ }
>+ 
>+ private void append(int c)
>+ {
>+ if (currentLineLength + 1 == currentLine.length)
>+ {
>+ char[] newCurrentLine = new char[currentLine.length*2];
>+ System.arraycopy(currentLine, 0, newCurrentLine, 0, currentLine.length);
>+ currentLine = newCurrentLine;
>+ }
>+ 
>+ currentLine[currentLineLength++] = (char)c;
>+ }
>+ 
>+ public String getFilename()
>+ {
>+ return filename;
>+ }
>+ 
>+ public int getFileOffsetOfNextRead()
>+ {
>+ return offsetInStream;
>+ }
>+ 
>+ public int getFileLineOfNextRead()
>+ {
>+ return lineNumInCurrentStream;
>+ }
>+ 
>+ public int getTotalOffsetOfNextRead()
>+ {
>+ return startOffset + offsetInStream;
>+ }
>+ 
>+ public int getTotalLineOfNextRead()
>+ {
>+ return startLineNum + lineNumInCurrentStream;
>+ }
>+
>+ public int getStartLine()
>+ {
>+ return startLineNum;
>+ }
>+
>+ public int getStartOffset()
>+ {
>+ return startOffset;
>+ }
>+
>+ public int getRestartLine()
>+ {
>+ return restartLine;
>+ }
>+
>+ public void setRestartLine(int restartLine)
>+ {
>+ this.restartLine = restartLine;
>+ }
>+
>+ public int getRestartOffset()
>+ {
>+ return restartOffset;
>+ }
>+
>+ public void setRestartOffset(int restartOffset)
>+ {
>+ this.restartOffset = restartOffset;
>+ }
>+ 
>+ ///////////////////////////////////////////////////////////////////////////
>+ // INPUTSTREAM IMPLEMENTATION
>+ ///////////////////////////////////////////////////////////////////////////
>+
>+ public int read() throws IOException
>+ {
>+ ensureLineLoaded();
>+ int result = peek();
>+ if (result >= 0)
>+ {
>+ offsetInCurrentLine++;
>+ offsetInStream++;
>+ }
>+ return result;
>+ }
>+
>+ ///////////////////////////////////////////////////////////////////////////
>+ // CHARSEQUENCE IMPLEMENTATION
>+ ///////////////////////////////////////////////////////////////////////////
>+ 
>+ public char charAt(int index)
>+ {
>+ return currentLine[offsetInCurrentLine+index];
>+ }
>+
>+ public int length()
>+ {
>+ return currentLineLength-offsetInCurrentLine;
>+ }
>+
>+ public CharSequence subSequence(final int start, final int end)
>+ {
>+ if (start < offsetInCurrentLine || end >= length())
>+ throw new IllegalArgumentException();
>+ 
>+ return new SubSequence(start, end);
>+ }
>+ 
>+ private final class SubSequence implements CharSequence
>+ {
>+ private final int start;
>+ private final int end;
>+
>+ private SubSequence(int start, int end)
>+ {
>+ this.start = start;
>+ this.end = end;
>+ }
>+
>+ public char charAt(int index)
>+ {
>+ return currentLine[offsetInCurrentLine+start];
>+ }
>+
>+ public int length()
>+ {
>+ return end-start;
>+ }
>+
>+ public CharSequence subSequence(int subseqStart, int subseqEnd)
>+ {
>+ return new SubSequence(start+subseqStart, start+subseqEnd);
>+ }
>+ 
>+ public String toString()
>+ {
>+ return String.valueOf(currentLine, start, end-start);
>+ }
>+ }
>+
>+ @Override
>+ public void close() throws IOException
>+ {
>+ in.close();
>+ }
>+}
>Index: lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
>===================================================================
>RCS file: lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
>diff -N lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,66 @@
>+/*******************************************************************************
>+ * Copyright (c) 2000, 2007 IBM Corporation and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ *
>+ * Contributors:
>+ * IBM Corporation - initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.photran.internal.core.lexer;
>+
>+import java.io.IOException;
>+import java.io.Reader;
>+
>+
>+/**
>+ * Copied from org.eclipse.jface.internal.text.html.SingleCharReader
>+ * 
>+ * Moved into this package from <code>org.eclipse.jface.internal.text.revisions</code>.
>+ */
>+public abstract class SingleCharReader extends Reader {
>+
>+	/**
>+	 * @see Reader#read()
>+	 */
>+	public abstract int read() throws IOException;
>+
>+	/**
>+	 * @see Reader#read(char[],int,int)
>+	 */
>+	public int read(char cbuf[], int off, int len) throws IOException {
>+		int end= off + len;
>+		for (int i= off; i < end; i++) {
>+			int ch= read();
>+			if (ch == -1) {
>+				if (i == off)
>+					return -1;
>+				return i - off;
>+			}
>+			cbuf[i]= (char)ch;
>+		}
>+		return len;
>+	}
>+
>+	/**
>+	 * @see Reader#ready()
>+	 */
>+ public boolean ready() throws IOException {
>+		return true;
>+	}
>+
>+	/**
>+	 * Returns the readable content as string.
>+	 * @return the readable content as string
>+	 * @exception IOException in case reading fails
>+	 */
>+	public String getString() throws IOException {
>+		StringBuffer buf= new StringBuffer();
>+		int ch;
>+		while ((ch= read()) != -1) {
>+			buf.append((char)ch);
>+		}
>+		return buf.toString();
>+	}
>+}
>#P org.eclipse.photran.core.vpg
>Index: parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java,v
>retrieving revision 1.5
>diff -u -r1.5 LexerFactory.java
>--- parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java	11 Mar 2010 21:41:10 -0000	1.5
>+++ parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java	12 May 2010 02:34:36 -0000
>@@ -10,11 +10,12 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.core.lexer;
> 
>-import java.io.BufferedInputStream;
>+import java.io.BufferedReader;
> import java.io.File;
>-import java.io.FileInputStream;
>+import java.io.FileReader;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.InputStreamReader;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.core.runtime.CoreException;
>@@ -29,19 +30,23 @@
> {
> private LexerFactory() {;}
> 
>- public static IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
>+ public static IAccumulatingLexer createLexer(Reader in, IFile file, String filename, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
> {
> return sourceForm.createLexer(in, file, filename, accumulateWhitetext);
> }
> 
> public static IAccumulatingLexer createLexer(File file, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
> {
>- return createLexer(new BufferedInputStream(new FileInputStream(file)), null, file.getAbsolutePath(), sourceForm, accumulateWhitetext);
>+ return createLexer(new BufferedReader(new FileReader(file)), null, file.getAbsolutePath(), sourceForm, accumulateWhitetext);
> }
> 
> public static IAccumulatingLexer createLexer(IFile file, SourceForm sourceForm, boolean accumulateWhitetext) throws CoreException, IOException
> {
>- return createLexer(file.getContents(true), file, determineFilename(file), sourceForm, accumulateWhitetext);
>+ return createLexer(new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset())),
>+ file,
>+ determineFilename(file),
>+ sourceForm,
>+ accumulateWhitetext);
> }
> 
> private static String determineFilename(IFile file)
>Index: parser/org/eclipse/photran/internal/core/lexer/SourceForm.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/SourceForm.java,v
>retrieving revision 1.12
>diff -u -r1.12 SourceForm.java
>--- parser/org/eclipse/photran/internal/core/lexer/SourceForm.java	5 Mar 2010 21:44:11 -0000	1.12
>+++ parser/org/eclipse/photran/internal/core/lexer/SourceForm.java	12 May 2010 02:34:36 -0000
>@@ -12,6 +12,7 @@
> 
> import java.io.IOException;
> import java.io.InputStream;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.core.runtime.CoreException;
>@@ -21,7 +22,6 @@
> import org.eclipse.photran.internal.core.FortranCorePlugin;
> import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.IncludeLoaderCallback;
> import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.PreprocessingFreeFormLexerPhase1;
>-import org.eclipse.photran.internal.core.vpg.PhotranVPG;
> 
> /**
> * Contains constants enumerating the various Fortran source forms (fixed form, free form, free
>@@ -63,13 +63,13 @@
> * </ol>
> * </ol>
> */
>- public abstract IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException;
>+ public abstract IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException;
> 
> public abstract String getDescription(String filename);
> 
> public static final SourceForm UNPREPROCESSED_FREE_FORM = new SourceForm()
> {
>- @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
>+ @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
> {
> return new LexerPhase3(new FreeFormLexerPhase2(new FreeFormLexerPhase1(in, file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext)));
> }
>@@ -79,7 +79,7 @@
> 
> public static final SourceForm FIXED_FORM = new SourceForm()
> {
>- @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
>+ @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
> {
> return new LexerPhase3(new FixedFormLexerPhase2(in, file, filename, ASTTokenFactory.getInstance()));
> }
>@@ -96,7 +96,7 @@
> {
> return new SourceForm()
> {
>- @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
>+ @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
> {
> SourceForm sf = findPreferredSourceForm(filename);
> if (sf != null)
>Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java,v
>retrieving revision 1.3
>diff -u -r1.3 FortranPreprocessor.java
>--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java	13 Oct 2009 20:25:21 -0000	1.3
>+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java	12 May 2010 02:34:36 -0000
>@@ -12,7 +12,7 @@
> 
> import java.io.FileNotFoundException;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.Reader;
> import java.util.ArrayList;
> import java.util.LinkedList;
> import java.util.Stack;
>@@ -21,15 +21,16 @@
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.photran.internal.core.lexer.FreeFormLexerPhase1;
>-import org.eclipse.photran.internal.core.lexer.LineInputStream;
>+import org.eclipse.photran.internal.core.lexer.LineReader;
>+import org.eclipse.photran.internal.core.lexer.SingleCharReader;
> 
> /**
>- * An <code>InputStream</code> that recognizes and processes Fortran INCLUDE lines.
>+ * An <code>Reader</code> that recognizes and processes Fortran INCLUDE lines.
> * 
>- * This class implements an {@link InputStream} the provides the contents of a
>+ * This class implements an {@link Reader} the provides the contents of a
> * Fortran file after INCLUDE lines have been processed. Photran's
> * machine-constructed, free-form lexer ({@link FreeFormLexerPhase1}) tokenizes
>- * this InputStream, oblivious to the fact that it is actually tokenizing
>+ * this Reader, oblivious to the fact that it is actually tokenizing
> * preprocessed text. However, a special subclass
> * ({@link PreprocessingFreeFormLexerPhase1}) takes care of mapping the
> * preprocessed tokens back to their images and locations in the original files.
>@@ -41,10 +42,10 @@
> * 
> * @author Jeff Overbey
> */
>-public final class FortranPreprocessor extends InputStream
>+public final class FortranPreprocessor extends SingleCharReader
> {
> /**
>- * Utility class. Originally, we simply used a <pre>Stack<LineInputStream></pre>,
>+ * Utility class. Originally, we simply used a <pre>Stack<LineReader></pre>,
> * but a profile revealed that Stack#peek was consuming a large amount of time
> * due to repeated invocations. Now, the #topStream field in this class is
> * accessed instead, resulting in a significant performance improvement
>@@ -52,13 +53,13 @@
> */
> private static final class StreamStack
> {
>- private Stack<LineInputStream> streamStack = new Stack<LineInputStream>();
>- private LineInputStream topStream = null;
>+ private Stack<LineReader> streamStack = new Stack<LineReader>();
>+ private LineReader topStream = null;
> 
>- public void push(LineInputStream lineInputStream)
>+ public void push(LineReader lineReader)
> {
>- streamStack.push(lineInputStream);
>- topStream = lineInputStream;
>+ streamStack.push(lineReader);
>+ topStream = lineReader;
> }
> 
> public int size()
>@@ -93,13 +94,13 @@
> private ArrayList<Integer> fileStartLines;
> private ArrayList<Integer> fileStartLineAdjustments;
> 
>- public FortranPreprocessor(InputStream readFrom, IFile file, String filename, IncludeLoaderCallback callback) throws IOException
>+ public FortranPreprocessor(Reader readFrom, IFile file, String filename, IncludeLoaderCallback callback) throws IOException
> {
> this.topLevelFile = file;
> this.callback = callback;
> 
> streamStack = new StreamStack();
>- streamStack.push(new LineInputStream(readFrom, filename));
>+ streamStack.push(new LineReader(readFrom, filename));
> 
> directivesInTopLevelFile = new LinkedList<FortranIncludeDirective>();
> directivesInTopLevelFile.add(null);
>@@ -167,9 +168,10 @@
> throw new IllegalArgumentException();
> }
> 
>+ @Override
> public int read() throws IOException
> {
>- LineInputStream currentStream = streamStack.topStream;
>+ LineReader currentStream = streamStack.topStream;
> if (currentStream.atBOL()) checkForInclude();
> currentStream = streamStack.topStream;
> if (currentStream.atEOF()) finishInclude();
>@@ -192,11 +194,11 @@
> Matcher m = INCLUDE_LINE.matcher(streamStack.topStream);
> if (m.matches())
> {
>- LineInputStream origStream = streamStack.topStream;
>+ LineReader origStream = streamStack.topStream;
> String includeLine = origStream.currentLine();
> String fileToInclude = m.group(INCLUDE_LINE_CAPTURING_GROUP_OF_FILENAME);
> 
>- InputStream newStream = findIncludedFile(fileToInclude);
>+ Reader newStream = findIncludedFile(fileToInclude);
> if (newStream != null)
> {
> if (inTopLevelFile())
>@@ -208,7 +210,7 @@
> origStream.setRestartOffset(offset + includeLine.length());
> origStream.setRestartLine(line + 1);
> 
>- streamStack.push(new LineInputStream(newStream, fileToInclude, offset, line));
>+ streamStack.push(new LineReader(newStream, fileToInclude, offset, line));
> 
> fileNames.add(fileToInclude);
> fileStartOffsets.add(new Integer(offset));
>@@ -222,7 +224,7 @@
> }
> }
> 
>- protected InputStream findIncludedFile(String fileToInclude) throws IOException
>+ protected Reader findIncludedFile(String fileToInclude) throws IOException
> {
> try
> {
>@@ -282,4 +284,10 @@
> {
> return topLevelFile;
> }
>+
>+ @Override
>+ public void close() throws IOException
>+ {
>+ streamStack.topStream.close();
>+ }
> }
>Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java,v
>retrieving revision 1.2
>diff -u -r1.2 IncludeLoaderCallback.java
>--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java	11 Mar 2010 21:41:10 -0000	1.2
>+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java	12 May 2010 02:34:36 -0000
>@@ -10,8 +10,11 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include;
> 
>+import java.io.BufferedReader;
> import java.io.FileNotFoundException;
>-import java.io.InputStream;
>+import java.io.InputStreamReader;
>+import java.io.Reader;
>+import java.io.UnsupportedEncodingException;
> 
> import org.eclipse.core.resources.IContainer;
> import org.eclipse.core.resources.IFile;
>@@ -44,11 +47,19 @@
> * @return <code>InputStream</code>, not null
> * @throws FileNotFoundException if the file cannot be found
> */
>- public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
>+ public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
> {
> try
> {
>- 	return getIncludedFile(fileToInclude).getContents(true);
>+ 	IFile file = getIncludedFile(fileToInclude);
>+ try
>+ {
>+ return new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
>+ }
>+ catch (UnsupportedEncodingException e)
>+ {
>+ return new BufferedReader(new InputStreamReader(file.getContents(true)));
>+ }
> }
> catch (CoreException e)
> {
>Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java,v
>retrieving revision 1.2
>diff -u -r1.2 PreprocessingFreeFormLexerPhase1.java
>--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java	13 Oct 2009 20:25:21 -0000	1.2
>+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java	12 May 2010 02:34:36 -0000
>@@ -12,12 +12,13 @@
> 
> import java.io.IOException;
> import java.io.InputStream;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.photran.internal.core.lexer.ASTTokenFactory;
> import org.eclipse.photran.internal.core.lexer.FreeFormLexerPhase1;
> import org.eclipse.photran.internal.core.lexer.LexerException;
>-import org.eclipse.photran.internal.core.lexer.LineAppendingInputStream;
>+import org.eclipse.photran.internal.core.lexer.LineAppendingReader;
> import org.eclipse.photran.internal.core.lexer.Token;
> import org.eclipse.photran.internal.core.lexer.TokenFactory;
> 
>@@ -39,9 +40,9 @@
> private FortranPreprocessor preprocessor;
> private FortranIncludeDirective lastDirective;
> 
>- public PreprocessingFreeFormLexerPhase1(InputStream in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
>+ public PreprocessingFreeFormLexerPhase1(Reader in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
> {
>- this(new FortranPreprocessor(new LineAppendingInputStream(in), file, filename, callback), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
>+ this(new FortranPreprocessor(new LineAppendingReader(in), file, filename, callback), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
> }
> 
> // This would not be here if we could assign the preprocessor to a variable in the above ctor (grrr)
>Index: src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java,v
>retrieving revision 1.2
>diff -u -r1.2 FortranResourceRefactoring.java
>--- src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java	8 May 2010 08:02:37 -0000	1.2
>+++ src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java	12 May 2010 02:34:39 -0000
>@@ -11,6 +11,8 @@
> package org.eclipse.photran.internal.core.refactoring.infrastructure;
> 
> import java.io.ByteArrayInputStream;
>+import java.io.InputStreamReader;
>+import java.nio.charset.Charset;
> import java.util.ArrayList;
> import java.util.Arrays;
> import java.util.Collection;
>@@ -271,8 +273,9 @@
> {
> try
> {
>+ final String charset = Charset.defaultCharset().name();
> IAccumulatingLexer lexer = LexerFactory.createLexer(
>- new ByteArrayInputStream(string.getBytes()), null, "(none)",
>+ new InputStreamReader(new ByteArrayInputStream(string.getBytes(charset)), charset), null, "(none)",
> SourceForm.UNPREPROCESSED_FREE_FORM, true);
> Parser parser = new Parser();
> 
>Index: src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java,v
>retrieving revision 1.14
>diff -u -r1.14 PhotranVPGBuilder.java
>--- src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java	27 Apr 2010 16:32:01 -0000	1.14
>+++ src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java	12 May 2010 02:34:39 -0000
>@@ -10,11 +10,15 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.core.vpg;
> 
>+import java.io.BufferedReader;
> import java.io.ByteArrayInputStream;
> import java.io.ByteArrayOutputStream;
> import java.io.FileNotFoundException;
>-import java.io.InputStream;
>+import java.io.InputStreamReader;
> import java.io.PrintStream;
>+import java.io.Reader;
>+import java.io.UnsupportedEncodingException;
>+import java.nio.charset.Charset;
> import java.util.List;
> 
> import org.eclipse.core.resources.IFile;
>@@ -26,9 +30,9 @@
> import org.eclipse.photran.internal.core.SyntaxException;
> import org.eclipse.photran.internal.core.analysis.binding.Binder;
> import org.eclipse.photran.internal.core.analysis.binding.Definition;
>+import org.eclipse.photran.internal.core.analysis.binding.Definition.Visibility;
> import org.eclipse.photran.internal.core.analysis.binding.ImplicitSpec;
> import org.eclipse.photran.internal.core.analysis.binding.ScopingNode;
>-import org.eclipse.photran.internal.core.analysis.binding.Definition.Visibility;
> import org.eclipse.photran.internal.core.lexer.IAccumulatingLexer;
> import org.eclipse.photran.internal.core.lexer.LexerException;
> import org.eclipse.photran.internal.core.lexer.LexerFactory;
>@@ -213,7 +217,7 @@
> sourceForm = SourceForm.preprocessedFreeForm(new IncludeLoaderCallback(file.getProject())
> {
> @Override
>- public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
>+ public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
> {
> // When we encounter an INCLUDE directive, set up a file dependency in the VPG
> 
>@@ -303,11 +307,19 @@
> if (ast == null)
> throw new IllegalArgumentException(filename + " returned null AST");
> 
>- ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
>- ast.getRoot().printOn(new PrintStream(out), null);
>- ast = parse(filename, new ByteArrayInputStream(out.toByteArray()));
>+ try
>+ {
>+ final String charset = Charset.defaultCharset().name(); // Platform's default charset
>+ ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
>+ ast.getRoot().printOn(new PrintStream(out, false, charset), null);
>+ ast = parse(filename, new InputStreamReader(new ByteArrayInputStream(out.toByteArray()), charset));
> 
>- computeEdgesAndAnnotations(filename, ast);
>+ computeEdgesAndAnnotations(filename, ast);
>+ }
>+ catch (UnsupportedEncodingException e)
>+ {
>+ throw new Error(e);
>+ }
> }
> 
> @Override public String getSourceCodeFromAST(IFortranAST ast)
>@@ -321,7 +333,7 @@
> return parse(filename, null);
> }
> 
>- private IFortranAST parse(final String filename, InputStream stream)
>+ private IFortranAST parse(final String filename, Reader stream)
> {
> if (filename == null || isVirtualFile(filename)) return null;
> 
>@@ -331,7 +343,7 @@
> SourceForm sourceForm = determineSourceForm(filename);
> try
> {
>- if (stream == null) stream = file.getContents(true);
>+ if (stream == null) stream = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
> IAccumulatingLexer lexer = LexerFactory.createLexer(stream, file, filename, sourceForm, true);
> long start = System.currentTimeMillis();
> ASTExecutableProgramNode ast = parser.parse(lexer);
>#P org.eclipse.photran.core.vpg.preprocessor.c
>Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java,v
>retrieving revision 1.2
>diff -u -r1.2 CPreprocessedSourceForm.java
>--- src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java	12 Mar 2010 16:47:09 -0000	1.2
>+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java	12 May 2010 02:34:46 -0000
>@@ -11,7 +11,7 @@
> package org.eclipse.photran.internal.core.lexer;
> 
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.IncludeLoaderCallback;
>@@ -30,7 +30,7 @@
> 	this.callback = callback;
> }
> 
>- @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
>+ @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
> {
> return new LexerPhase3(new FreeFormLexerPhase2(new CPreprocessingFreeFormLexerPhase1(in, file, filename, callback, accumulateWhitetext)));
> }
>Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java,v
>retrieving revision 1.1
>diff -u -r1.1 CPreprocessingFreeFormLexerPhase1.java
>--- src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java	2 Mar 2010 22:01:25 -0000	1.1
>+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java	12 May 2010 02:34:46 -0000
>@@ -10,8 +10,12 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.core.lexer;
> 
>+import java.io.BufferedReader;
> import java.io.IOException;
> import java.io.InputStream;
>+import java.io.InputStreamReader;
>+import java.io.Reader;
>+import java.nio.charset.Charset;
> import java.util.ArrayList;
> 
> import org.eclipse.core.resources.IFile;
>@@ -20,7 +24,7 @@
> /**
> * Phase 1 lexer that handles C preprocessor directives.
> * 
>- * This class feeds {@link FreeFormLexerPhase1} with a {@link CPreprocessingInputStream},
>+ * This class feeds {@link FreeFormLexerPhase1} with a {@link CPreprocessingReader},
> * an {@link InputStream} that interprets C preprocessor directives.
> * {@link FreeFormLexerPhase1} tokenizes the preprocessed stream. However, this
> * class subclasses {@link FreeFormLexerPhase1} and overrides its tokenization
>@@ -49,13 +53,13 @@
> 	
> 	private Token lastReadInFile = null;
> 
>- public CPreprocessingFreeFormLexerPhase1(InputStream in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
>+ public CPreprocessingFreeFormLexerPhase1(Reader in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
> {
>- this(new CPreprocessingInputStream(file, filename, new LineAppendingInputStream(in)), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
>+ this(new CPreprocessingReader(file, filename, new LineAppendingReader(in)), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
> }
> 
> // This would not be here if we could assign the preprocessor to a variable in the above ctor (grrr)
>- private CPreprocessingFreeFormLexerPhase1(CPreprocessingInputStream cpp, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
>+ private CPreprocessingFreeFormLexerPhase1(CPreprocessingReader cpp, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
> {
> super(cpp, file, filename, tokenFactory, accumulateWhitetext);
> this.producerMap = cpp.getProducerMap();
>Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
>===================================================================
>RCS file: src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
>diff -N src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
>--- src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java	2 Mar 2010 22:01:25 -0000	1.1
>+++ /dev/null	1 Jan 1970 00:00:00 -0000
>@@ -1,86 +0,0 @@
>-/*******************************************************************************
>- * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>- * All rights reserved. This program and the accompanying materials
>- * are made available under the terms of the Eclipse Public License v1.0
>- * which accompanies this distribution, and is available at
>- * http://www.eclipse.org/legal/epl-v10.html
>- *
>- * Contributors:
>- * UIUC - Initial API and implementation
>- *******************************************************************************/
>-package org.eclipse.photran.internal.core.lexer;
>-
>-import java.io.IOException;
>-import java.io.InputStream;
>-
>-import org.eclipse.core.resources.IFile;
>-import org.eclipse.photran.internal.core.preprocessor.c.CppHelper;
>-import org.eclipse.photran.internal.core.preprocessor.c.IToken;
>-import org.eclipse.photran.internal.core.preprocessor.c.OffsetLimitReachedException;
>-
>-/**
>- * An InputStream to read tokens obtained from the CPP. This class will
>- * also make a ProducerMap when constructed that can be used to find
>- * directives and macros.
>- * 
>- * @author Matthew Michelotti
>- */
>-public class CPreprocessingInputStream extends InputStream
>-{
>-	/**ProducerMap constructed from the CPP tokens*/
>-	private ProducerMap producerMap;
>-	
>-	private IncludeMap includeMap;
>-
>-	/**current CPP token being read*/
>-	private IToken curToken;
>-	/**full image of curToken*/
>-	private String curTokenImage;
>-	/**the offset in curTokenImage to be read next*/
>-	private int curTokenOffset = 0;
>-	
>-	/**
>-	 * Basic constructor. Parses the whole file in the constructor
>-	 * and turns the result into CPP tokens. A ProducerMap is made
>-	 * from these tokens. The tokens are remembered, so that they
>-	 * can be read as an input stream.
>-	 * @param filename - name of file that inputStream corresponds to
>-	 * @param inputStream - InputStream to read file data from
>-	 * @throws IOException
>-	 */
>-	public CPreprocessingInputStream(IFile file, String filename, InputStream inputStream) throws IOException
>-	{
>-		try
>-		{
>-			CppHelper cpp = new CppHelper(file, filename, inputStream);
>-			curToken = cpp.getRemainingTokens();
>-			curTokenImage = CppHelper.getFullImage(curToken);
>-			producerMap = new ProducerMap(curToken);
>-			includeMap = new IncludeMap(curToken, file);
>-		}catch(OffsetLimitReachedException e) {throw new Error(e);}
>-	}
>-	
>-	@Override public int read() throws IOException
>-	{
>-		if(curToken == null) return -1;
>-		while(curTokenOffset >= curTokenImage.length()) {
>-			curToken = curToken.getNext();
>-			if(curToken == null) return -1;
>-			curTokenImage = CppHelper.getFullImage(curToken);
>-			curTokenOffset = 0;
>-		}
>-		char result = curTokenImage.charAt(curTokenOffset);
>-		curTokenOffset++;
>-		
>-		return result;
>-	}
>-	
>-	/**@return the ProducerMap object made at construction*/
>-	public ProducerMap getProducerMap() {
>-		return producerMap;
>-	}
>-	
>-	public IncludeMap getIncludeMap() {
>-		return includeMap;
>-	}
>-}
>Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
>===================================================================
>RCS file: src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
>diff -N src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,91 @@
>+/*******************************************************************************
>+ * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ *
>+ * Contributors:
>+ * UIUC - Initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.photran.internal.core.lexer;
>+
>+import java.io.IOException;
>+import java.io.Reader;
>+
>+import org.eclipse.core.resources.IFile;
>+import org.eclipse.photran.internal.core.preprocessor.c.CppHelper;
>+import org.eclipse.photran.internal.core.preprocessor.c.IToken;
>+import org.eclipse.photran.internal.core.preprocessor.c.OffsetLimitReachedException;
>+
>+/**
>+ * An InputStream to read tokens obtained from the CPP. This class will
>+ * also make a ProducerMap when constructed that can be used to find
>+ * directives and macros.
>+ * 
>+ * @author Matthew Michelotti
>+ */
>+public class CPreprocessingReader extends SingleCharReader
>+{
>+	/**ProducerMap constructed from the CPP tokens*/
>+	private ProducerMap producerMap;
>+	
>+	private IncludeMap includeMap;
>+
>+	/**current CPP token being read*/
>+	private IToken curToken;
>+	/**full image of curToken*/
>+	private String curTokenImage;
>+	/**the offset in curTokenImage to be read next*/
>+	private int curTokenOffset = 0;
>+	
>+	/**
>+	 * Basic constructor. Parses the whole file in the constructor
>+	 * and turns the result into CPP tokens. A ProducerMap is made
>+	 * from these tokens. The tokens are remembered, so that they
>+	 * can be read as an input stream.
>+	 * @param filename - name of file that inputStream corresponds to
>+	 * @param reader - Reader to read file data from
>+	 * @throws IOException
>+	 */
>+	public CPreprocessingReader(IFile file, String filename, Reader reader) throws IOException
>+	{
>+		try
>+		{
>+			CppHelper cpp = new CppHelper(file, filename, reader);
>+			curToken = cpp.getRemainingTokens();
>+			curTokenImage = CppHelper.getFullImage(curToken);
>+			producerMap = new ProducerMap(curToken);
>+			includeMap = new IncludeMap(curToken, file);
>+		}catch(OffsetLimitReachedException e) {throw new Error(e);}
>+	}
>+	
>+	@Override public int read() throws IOException
>+	{
>+		if(curToken == null) return -1;
>+		while(curTokenOffset >= curTokenImage.length()) {
>+			curToken = curToken.getNext();
>+			if(curToken == null) return -1;
>+			curTokenImage = CppHelper.getFullImage(curToken);
>+			curTokenOffset = 0;
>+		}
>+		char result = curTokenImage.charAt(curTokenOffset);
>+		curTokenOffset++;
>+		
>+		return result;
>+	}
>+	
>+	/**@return the ProducerMap object made at construction*/
>+	public ProducerMap getProducerMap() {
>+		return producerMap;
>+	}
>+	
>+	public IncludeMap getIncludeMap() {
>+		return includeMap;
>+	}
>+
>+ @Override
>+ public void close() throws IOException
>+ {
>+ }
>+}
>Index: src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java,v
>retrieving revision 1.2
>diff -u -r1.2 CppHelper.java
>--- src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java	4 Mar 2010 08:01:44 -0000	1.2
>+++ src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java	12 May 2010 02:34:51 -0000
>@@ -12,9 +12,8 @@
> 
> import java.io.File;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.Reader;
> import java.util.Arrays;
>-import java.util.Map;
> 
> import org.eclipse.cdt.core.CCorePlugin;
> import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
>@@ -24,7 +23,6 @@
> import org.eclipse.cdt.core.parser.NullLogService;
> import org.eclipse.cdt.core.parser.ParserLanguage;
> import org.eclipse.cdt.core.parser.ScannerInfo;
>-import org.eclipse.core.resources.IFile;
> import org.eclipse.core.resources.IProject;
> import org.eclipse.core.resources.IResource;
> 
>@@ -64,11 +62,11 @@
> * @param includeSearchPaths - file paths to search in for included files
> * @throws IOException
> */
>- public CppHelper(String filename, InputStream in, String[] includeSearchPaths)
>+ public CppHelper(String filename, Reader in, String[] includeSearchPaths)
> 		throws IOException
> {
> 	//NOTE: it is important that the CodeReader gets an ABSOLUTE file path
>- cpp = new CPreprocessor(new InputStreamCodeReader(filename, in),
>+ cpp = new CPreprocessor(new ReaderBasedCodeReader(filename, in),
> new ScannerInfo(null, includeSearchPaths), ParserLanguage.C,
> new NullLogService(), new GCCScannerExtensionConfiguration(),
> FileCodeReaderFactory.getInstance());
>@@ -85,7 +83,7 @@
> * Also, @param in could be different than @param file, if for example there
> * are un-saved changes in the editor.
> */
>- public CppHelper(IResource infoProvider, String filename, InputStream in) throws IOException {
>+ public CppHelper(IResource infoProvider, String filename, Reader in) throws IOException {
> //NOTE: it is important that the CodeReader gets an ABSOLUTE file path
> if (filename == null) 
> filename = "";
>@@ -104,7 +102,7 @@
> }
> 
> //Map<String, String> temp = scanInfo.getDefinedSymbols();
>- cpp = new CPreprocessor(new InputStreamCodeReader(filename, in),
>+ cpp = new CPreprocessor(new ReaderBasedCodeReader(filename, in),
> new ScannerInfo(scanInfo.getDefinedSymbols(), scanInfo.getIncludePaths()), ParserLanguage.C, new NullLogService(), 
> new GCCScannerExtensionConfiguration(), FileCodeReaderFactory.getInstance());
> }
>Index: src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
>===================================================================
>RCS file: src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
>diff -N src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
>--- src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java	2 Mar 2010 22:01:24 -0000	1.1
>+++ /dev/null	1 Jan 1970 00:00:00 -0000
>@@ -1,63 +0,0 @@
>-/*******************************************************************************
>- * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>- * All rights reserved. This program and the accompanying materials
>- * are made available under the terms of the Eclipse Public License v1.0
>- * which accompanies this distribution, and is available at
>- * http://www.eclipse.org/legal/epl-v10.html
>- *
>- * Contributors:
>- * UIUC - Initial API and implementation
>- *******************************************************************************/
>-package org.eclipse.photran.internal.core.preprocessor.c;
>-
>-import java.io.IOException;
>-import java.io.InputStream;
>-
>-import org.eclipse.cdt.core.parser.CodeReader;
>-
>-public class InputStreamCodeReader extends CodeReader 
>-{
>-
>-	public InputStreamCodeReader(String filename, InputStream stream)
>-			throws IOException 
>-	{
>-		super(filename, convertToCharBuffer(stream));
>-		
>-		// TODO Auto-generated constructor stub
>-	}
>-	
>-	public InputStreamCodeReader(String filename, String charSet, InputStream stream)
>-			throws IOException 
>-	{
>-		//We are dropping out the charSet variable because it is used internally by 
>-		// CodeReader to read from the FileStream. Since we want to read from the 
>-		// InputStream, and we do it byte-wise, we don't really care for encoding
>-		super(filename, convertToCharBuffer(stream));
>-		// TODO Auto-generated constructor stub
>-	}
>-	
>-	private static char[] convertToCharBuffer(InputStream stream)
>-	{
>-		int b = 0;
>-		StringBuffer aggregate = new StringBuffer();
>-		while(b != -1)
>-		{
>-			try 
>-			{
>-				b = stream.read();
>-			} 
>-			catch (IOException e) 
>-			{
>-				e.printStackTrace();
>-				b = -1;
>-				break;
>-			}
>-			if(b != -1)
>-			{
>-				aggregate.append((char)b);
>-			}
>-		}
>-		return aggregate.toString().toCharArray();
>-	}
>-
>-}
>Index: src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
>===================================================================
>RCS file: src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
>diff -N src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,59 @@
>+/*******************************************************************************
>+ * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ *
>+ * Contributors:
>+ * UIUC - Initial API and implementation
>+ * (Jeff Overbey) Modified to use Reader rather than InputStream
>+ *******************************************************************************/
>+package org.eclipse.photran.internal.core.preprocessor.c;
>+
>+import java.io.IOException;
>+import java.io.Reader;
>+
>+import org.eclipse.cdt.core.parser.CodeReader;
>+
>+public class ReaderBasedCodeReader extends CodeReader 
>+{
>+
>+	public ReaderBasedCodeReader(String filename, Reader stream)
>+			throws IOException 
>+	{
>+		super(filename, convertToCharBuffer(stream));
>+	}
>+	
>+	private static char[] convertToCharBuffer(Reader stream)
>+	{
>+		int b = 0;
>+		StringBuffer aggregate = new StringBuffer();
>+		while(b != -1)
>+		{
>+			try 
>+			{
>+				b = stream.read();
>+			} 
>+			catch (IOException e) 
>+			{
>+				e.printStackTrace();
>+				b = -1;
>+				break;
>+			}
>+			if(b != -1)
>+			{
>+				aggregate.append((char)b);
>+			}
>+		}
>+		try
>+ {
>+ stream.close();
>+ }
>+ catch (IOException e)
>+ {
>+ e.printStackTrace();
>+ }
>+		return aggregate.toString().toCharArray();
>+	}
>+}
>#P org.eclipse.photran.core.vpg.preprocessor.c.tests
>Index: src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c.tests/src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java,v
>retrieving revision 1.1
>diff -u -r1.1 CppIntegratedSourceReproductionTestCase.java
>--- src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java	2 Mar 2010 22:06:58 -0000	1.1
>+++ src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java	12 May 2010 02:34:54 -0000
>@@ -15,7 +15,7 @@
> import java.io.FileNotFoundException;
> import java.io.FileReader;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.Reader;
> 
> import org.eclipse.photran.internal.core.lexer.CPreprocessedSourceForm;
> import org.eclipse.photran.internal.core.lexer.SourceForm;
>@@ -35,7 +35,7 @@
> {
> return new CPreprocessedSourceForm(new IncludeLoaderCallback(null)
> {
>- @Override public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
>+ @Override public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
> {
> throw new FileNotFoundException("#include lines not supported in JUnit tests");
> }
>#P org.eclipse.photran.core.vpg.tests
>Index: src/org/eclipse/photran/internal/tests/BaseTestCase.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/BaseTestCase.java,v
>retrieving revision 1.3
>diff -u -r1.3 BaseTestCase.java
>--- src/org/eclipse/photran/internal/tests/BaseTestCase.java	25 Feb 2010 18:11:29 -0000	1.3
>+++ src/org/eclipse/photran/internal/tests/BaseTestCase.java	12 May 2010 02:34:59 -0000
>@@ -10,9 +10,8 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.tests;
> 
>-import java.io.ByteArrayInputStream;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.StringReader;
> 
> import junit.framework.TestCase;
> 
>@@ -63,8 +62,7 @@
> protected <T extends IBodyConstruct> T parseStmt(String stmt) throws IOException, LexerException, SyntaxException
> {
> String program = stmt + "\nend program\n";
>- InputStream in = new ByteArrayInputStream(program.getBytes());
>- ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(in, null, null, SourceForm.UNPREPROCESSED_FREE_FORM, true));
>+ ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(program), null, null, SourceForm.UNPREPROCESSED_FREE_FORM, true));
> assertTrue(ast != null);
> LoopReplacer.replaceAllLoopsIn(ast);
> return (T)((ASTMainProgramNode)ast.getProgramUnitList().get(0)).getBody().get(0);
>Index: src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java,v
>retrieving revision 1.1
>diff -u -r1.1 TypeCheckerTestCase.java
>--- src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java	11 Nov 2009 17:48:12 -0000	1.1
>+++ src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java	12 May 2010 02:35:00 -0000
>@@ -10,7 +10,7 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.tests.analysis.types;
> 
>-import java.io.ByteArrayInputStream;
>+import java.io.StringReader;
> 
> import junit.framework.TestCase;
> 
>@@ -39,7 +39,7 @@
> // a parseable program, then extract the expression from the program's AST
> 
> String program = "a = " + expression + "\nend";
>- ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new ByteArrayInputStream(program.getBytes()), null, "<literal text>", SourceForm.UNPREPROCESSED_FREE_FORM, true /*false*/));
>+ ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(program), null, "<literal text>", SourceForm.UNPREPROCESSED_FREE_FORM, true /*false*/));
> assertNotNull(ast);
> ASTMainProgramNode mainProg = (ASTMainProgramNode)ast.getProgramUnitList().get(0);
> ASTAssignmentStmtNode assignmentStmt = (ASTAssignmentStmtNode)mainProg.getBody().get(0);
>Index: src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java,v
>retrieving revision 1.3
>diff -u -r1.3 TestOpenMPComments.java
>--- src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java	4 May 2010 19:58:05 -0000	1.3
>+++ src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java	12 May 2010 02:35:00 -0000
>@@ -10,8 +10,8 @@
> *******************************************************************************/
> package org.eclipse.photran.internal.tests.parser;
> 
>-import java.io.ByteArrayInputStream;
> import java.io.IOException;
>+import java.io.StringReader;
> import java.util.ArrayList;
> import java.util.List;
> 
>@@ -129,8 +129,7 @@
> 
> private ASTExecutableProgramNode parse(SourceForm sourceForm, String string) throws IOException, LexerException, SyntaxException
> {
>- ByteArrayInputStream in = new ByteArrayInputStream(string.getBytes());
>- ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(in, null, "<stdin>", sourceForm, true));
>+ ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(string), null, "<stdin>", sourceForm, true));
> assertTrue(ast != null);
> return ast;
> }
>#P org.eclipse.photran.ui.vpg
>Index: src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.ui.vpg/src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java,v
>retrieving revision 1.14
>diff -u -r1.14 FortranEditorTasks.java
>--- src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java	11 Mar 2010 20:19:30 -0000	1.14
>+++ src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java	12 May 2010 02:35:04 -0000
>@@ -1,6 +1,6 @@
> package org.eclipse.photran.internal.ui.editor_vpg;
> 
>-import java.io.ByteArrayInputStream;
>+import java.io.StringReader;
> import java.util.HashSet;
> import java.util.Set;
> 
>@@ -179,7 +179,7 @@
> String editorContents = editor.getDocumentProvider().getDocument(editor.getEditorInput()).get();
> 
> SourceForm sourceForm = determineSourceForm();
>- IAccumulatingLexer lexer = LexerFactory.createLexer(new ByteArrayInputStream(editorContents.getBytes()),
>+ IAccumulatingLexer lexer = LexerFactory.createLexer(new StringReader(editorContents),
> editor.getIFile(),
> editor.getIFile().getName(),
> sourceForm,
>#P org.eclipse.photran.ui.vpg.preprocessor.c
>Index: src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.ui.vpg.preprocessor.c/src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java,v
>retrieving revision 1.1
>diff -u -r1.1 DisplayPreprocessedSource.java
>--- src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java	2 Mar 2010 22:07:38 -0000	1.1
>+++ src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java	12 May 2010 02:35:09 -0000
>@@ -13,15 +13,15 @@
> import java.io.BufferedOutputStream;
> import java.io.File;
> import java.io.FileOutputStream;
>-import java.io.InputStream;
> import java.io.PrintStream;
>-import java.io.StringBufferInputStream;
>+import java.io.Reader;
>+import java.io.StringReader;
> import java.lang.reflect.InvocationTargetException;
> 
> import org.eclipse.core.runtime.IProgressMonitor;
> import org.eclipse.jface.dialogs.MessageDialog;
> import org.eclipse.jface.text.IDocument;
>-import org.eclipse.photran.internal.core.lexer.CPreprocessingInputStream;
>+import org.eclipse.photran.internal.core.lexer.CPreprocessingReader;
> import org.eclipse.photran.internal.ui.editor.FixedFormFortranEditor;
> import org.eclipse.photran.internal.ui.editor.FreeFormFortranEditor;
> import org.eclipse.ui.ide.IDE;
>@@ -32,7 +32,7 @@
> *
> * @author Jeff Overbey
> */
>-@SuppressWarnings({ "deprecation", "restriction" })
>+@SuppressWarnings({ "restriction" })
> public class DisplayPreprocessedSource extends FortranEditorActionDelegate
> {
> public void run(IProgressMonitor progressMonitor) throws InvocationTargetException, InterruptedException
>@@ -42,8 +42,8 @@
> 	boolean isFixedForm = getFortranEditor().isFixedForm();
> 	
> IDocument doc = getFortranEditor().getIDocument();
>- InputStream in = new StringBufferInputStream(doc.get());
>- InputStream cppIn = new CPreprocessingInputStream(getFortranEditor().getIFile(), null, in);
>+ Reader in = new StringReader(doc.get());
>+ Reader cppIn = new CPreprocessingReader(getFortranEditor().getIFile(), null, in);
> 
> File tempFile = File.createTempFile(
> 	"tmp",
>#P org.eclipse.rephraserengine.core
>Index: src/org/eclipse/rephraserengine/core/util/StringUtil.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.core/src/org/eclipse/rephraserengine/core/util/StringUtil.java,v
>retrieving revision 1.4
>diff -u -r1.4 StringUtil.java
>--- src/org/eclipse/rephraserengine/core/util/StringUtil.java	25 Feb 2010 14:06:01 -0000	1.4
>+++ src/org/eclipse/rephraserengine/core/util/StringUtil.java	12 May 2010 02:35:12 -0000
>@@ -67,7 +67,7 @@
> 
> public static String read(IFile file) throws IOException, CoreException
> {
>- return read(file.getContents(true));
>+ return read(new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset())));
> }
> 
> public static String read(File file) throws IOException
>#P org.eclipse.rephraserengine.core.vpg
>Index: src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.core.vpg/src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java,v
>retrieving revision 1.2
>diff -u -r1.2 VPGRefactoring.java
>--- src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java	11 Mar 2010 21:41:07 -0000	1.2
>+++ src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java	12 May 2010 02:35:16 -0000
>@@ -1,7 +1,9 @@
> package org.eclipse.rephraserengine.core.vpg.refactoring;
> 
>+import java.io.BufferedReader;
> import java.io.IOException;
>-import java.io.InputStream;
>+import java.io.InputStreamReader;
>+import java.io.Reader;
> 
> import org.eclipse.core.resources.IFile;
> import org.eclipse.core.runtime.CoreException;
>@@ -237,9 +239,10 @@
> return ast.toString();
> }
> 
>- private int getSizeOf(IFile file) throws CoreException, IOException {
>+ private int getSizeOf(IFile file) throws CoreException, IOException
>+ {
> int size = 0;
>- InputStream in = file.getContents(true);
>+ Reader in = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
> while (in.read() > -1)
> size++;
> in.close();
>#P org.eclipse.rephraserengine.examples.text
>Index: src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.examples.text/src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java,v
>retrieving revision 1.1
>diff -u -r1.1 ConvertTabsToSpacesRefactoring.java
>--- src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java	3 Nov 2009 17:00:37 -0000	1.1
>+++ src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java	12 May 2010 02:35:20 -0000
>@@ -10,8 +10,10 @@
> *******************************************************************************/
> package org.eclipse.rephraserengine.internal.examples.text;
> 
>-import java.io.BufferedInputStream;
>+import java.io.BufferedReader;
> import java.io.IOException;
>+import java.io.InputStreamReader;
>+import java.io.Reader;
> import java.util.List;
> 
> import org.eclipse.core.resources.IFile;
>@@ -127,7 +129,7 @@
> TextFileChange change = new TextFileChange("Replace tabs in " + file.getName(), file);
> change.setEdit(new MultiTextEdit());
> 
>- BufferedInputStream in = new BufferedInputStream(file.getContents(true));
>+ Reader in = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
> int offset = 0;
> int column = 0;
> int numberOfTabsReplaced = 0;
>#P org.eclipse.rephraserengine.ui.vpg
>Index: src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java
>===================================================================
>RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.ui.vpg/src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java,v
>retrieving revision 1.4
>diff -u -r1.4 DisplayModelAction.java
>--- src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java	4 Mar 2010 18:19:14 -0000	1.4
>+++ src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java	12 May 2010 02:35:24 -0000
>@@ -12,9 +12,9 @@
> 
> import java.io.BufferedReader;
> import java.io.IOException;
>-import java.io.InputStream;
> import java.io.InputStreamReader;
> import java.io.PrintStream;
>+import java.io.Reader;
> import java.util.ArrayList;
> 
> import org.eclipse.core.runtime.NullProgressMonitor;
>@@ -42,7 +42,8 @@
> if (filename != null)
> {
> ArrayList<Integer> lineMap = new ArrayList<Integer>();
>- String fileContents = readStream(lineMap, info.getFileInEditor().getContents(true));
>+ String fileContents = readStream(lineMap,
>+ new BufferedReader(new InputStreamReader(info.getFileInEditor().getContents(true), info.getFileInEditor().getCharset())));
> ps.println(filename);
> ps.println();
> Model model = new Model("edge model", new NullProgressMonitor(), 0, vpg, filename);
>@@ -60,10 +61,9 @@
> }
> }
> 
>- protected String readStream(ArrayList<Integer> lineMap, InputStream inputStream) throws IOException
>+ protected String readStream(ArrayList<Integer> lineMap, Reader in) throws IOException
> {
> StringBuffer sb = new StringBuffer(4096);
>- BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
> for (int offset = 0, ch = in.read(); ch >= 0; ch = in.read())
> {
> sb.append((char)ch);

Flags:

g.watson: iplog+

Actions: View | Diff

Attachments on bug 312249: 167699 | 168076