### Eclipse Workspace Patch 1.0
#P org.eclipse.photran.cdtinterface
Index: model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.cdtinterface/model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java,v
retrieving revision 1.6
diff -u -r1.6 SimpleFortranModelBuilder.java
--- model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java	2 Nov 2009 21:19:03 -0000	1.6
+++ model/org/eclipse/photran/internal/core/model/SimpleFortranModelBuilder.java	12 May 2010 02:33:56 -0000
@@ -10,8 +10,8 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.core.model;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
@@ -73,7 +73,7 @@
     {
         this.newElements = new HashMap();
         
-        InputStream in = new ByteArrayInputStream(translationUnit.getBuffer().getContents().getBytes());
+        Reader in = new StringReader(translationUnit.getBuffer().getContents());
         String filename = translationUnit.getElementName();
         boolean wasSuccessful = true;
 
#P org.eclipse.photran.cdtinterface.vpg
Index: model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.cdtinterface.vpg/model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java,v
retrieving revision 1.15
diff -u -r1.15 FortranModelBuilder.java
--- model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java	12 Mar 2010 16:47:05 -0000	1.15
+++ model/org/eclipse/photran/internal/core/model/FortranModelBuilder.java	12 May 2010 02:34:01 -0000
@@ -1,6 +1,6 @@
 package org.eclipse.photran.internal.core.model;
 
-import java.io.ByteArrayInputStream;
+import java.io.StringReader;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -74,7 +74,7 @@
             SourceForm sourceForm = determineSourceForm(file);
             String filename = determineFilename(file);
             lexer = LexerFactory.createLexer(
-                new ByteArrayInputStream(translationUnit.getBuffer().getContents().getBytes()),
+                new StringReader(translationUnit.getBuffer().getContents()),
                 file,
                 filename,
                 sourceForm,
#P org.eclipse.photran.core
Index: lexer/FixedFormLexerPhase1.flex
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/FixedFormLexerPhase1.flex,v
retrieving revision 1.15
diff -u -r1.15 FixedFormLexerPhase1.flex
--- lexer/FixedFormLexerPhase1.flex	8 Apr 2010 01:02:05 -0000	1.15
+++ lexer/FixedFormLexerPhase1.flex	12 May 2010 02:34:09 -0000
@@ -34,7 +34,7 @@
 package org.eclipse.photran.internal.core.lexer;
 
 import java.util.regex.Pattern;
-import java.io.InputStream;
+import java.io.Reader;
 import org.eclipse.core.resources.IFile;
 
 @SuppressWarnings("unused")
@@ -58,8 +58,8 @@
 
     private TokenFactory tokenFactory;
 
-    public FixedFormLexerPhase1(InputStream in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
-        this(new LineAppendingInputStream(in));
+    public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
+        this(new LineAppendingReader(in));
         this.prepass=_prepass;
         this.tokenFactory = tokenFactory;
     }
@@ -188,7 +188,7 @@
 
     private String filename = "<stdin>";
 
-    public FixedFormLexerPhase1(java.io.InputStream in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
+    public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
     {
         this(in, _prepass, tokenFactory);
         this.lastTokenFile = new FileOrIFile(file);
Index: lexer/FreeFormLexerPhase1.flex
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/FreeFormLexerPhase1.flex,v
retrieving revision 1.14
diff -u -r1.14 FreeFormLexerPhase1.flex
--- lexer/FreeFormLexerPhase1.flex	5 Mar 2010 16:47:00 -0000	1.14
+++ lexer/FreeFormLexerPhase1.flex	12 May 2010 02:34:10 -0000
@@ -27,8 +27,10 @@
  */
 
 package org.eclipse.photran.internal.core.lexer;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.Reader;
 import org.eclipse.core.resources.IFile;
 
 @SuppressWarnings("unused")
@@ -120,9 +122,9 @@
     private String filename = "<stdin>";
     protected TokenFactory tokenFactory;
 
-    public FreeFormLexerPhase1(java.io.InputStream in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
+    public FreeFormLexerPhase1(Reader in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
     {
-        this(new LineAppendingInputStream(in));
+        this(new LineAppendingReader(in));
         this.lastTokenFile = new FileOrIFile(file);
         this.filename = filename;
         this.tokenFactory = tokenFactory;
Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java,v
retrieving revision 1.16
diff -u -r1.16 FixedFormLexerPhase1.java
--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java	8 Apr 2010 01:02:05 -0000	1.16
+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase1.java	12 May 2010 02:34:13 -0000
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 3/18/10 4:23 PM */
+/* The following code was generated by JFlex 1.4.1 on 5/11/10 8:56 PM */
 
 /*******************************************************************************
  * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
@@ -36,7 +36,7 @@
 package org.eclipse.photran.internal.core.lexer;
 
 import java.util.regex.Pattern;
-import java.io.InputStream;
+import java.io.Reader;
 import org.eclipse.core.resources.IFile;
 
 @SuppressWarnings("unused")
@@ -44,7 +44,7 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 3/18/10 4:23 PM from the specification file
+ * on 5/11/10 8:56 PM from the specification file
  * <tt>FixedFormLexerPhase1.flex</tt>
  */
 class FixedFormLexerPhase1 implements ILexer {
@@ -916,8 +916,8 @@
 
     private TokenFactory tokenFactory;
 
-    public FixedFormLexerPhase1(InputStream in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
-        this(new LineAppendingInputStream(in));
+    public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, TokenFactory tokenFactory) {
+        this(new LineAppendingReader(in));
         this.prepass=_prepass;
         this.tokenFactory = tokenFactory;
     }
@@ -1046,7 +1046,7 @@
 
     private String filename = "<stdin>";
 
-    public FixedFormLexerPhase1(java.io.InputStream in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
+    public FixedFormLexerPhase1(Reader in, FixedFormLexerPrepass _prepass, IFile file, String filename, TokenFactory tokenFactory)
     {
         this(in, _prepass, tokenFactory);
         this.lastTokenFile = new FileOrIFile(file);
Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java,v
retrieving revision 1.7
diff -u -r1.7 FixedFormLexerPhase2.java
--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java	18 Mar 2010 20:00:59 -0000	1.7
+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPhase2.java	12 May 2010 02:34:13 -0000
@@ -11,7 +11,7 @@
 package org.eclipse.photran.internal.core.lexer;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 
@@ -29,12 +29,13 @@
 
     private IToken nextToken = null;
 
-    public FixedFormLexerPhase2(InputStream in, IFile file, String filename, TokenFactory tokenFactory)
+    public FixedFormLexerPhase2(Reader in, IFile file, String filename, TokenFactory tokenFactory)
     {
-        in = new LineAppendingInputStream(in);
-        final FixedFormLexerPrepass prepass = new FixedFormLexerPrepass(in);
-        InputStream prepassReader = new InputStream()
+        final Reader input = new LineAppendingReader(in);
+        final FixedFormLexerPrepass prepass = new FixedFormLexerPrepass(input);
+        Reader prepassReader = new SingleCharReader()
         {
+            @Override
             public int read() throws IOException
             {
                 try
@@ -47,6 +48,12 @@
                     return -1;
                 }
             }
+            
+            @Override
+            public void close() throws IOException
+            {
+                input.close();
+            }
         };
 
         FixedFormLexerPhase1 fixedLexer1 = new FixedFormLexerPhase1(prepassReader, prepass, file, filename, tokenFactory);
Index: lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java,v
retrieving revision 1.8
diff -u -r1.8 FixedFormLexerPrepass.java
--- lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java	27 Apr 2010 19:42:48 -0000	1.8
+++ lexer/org/eclipse/photran/internal/core/lexer/FixedFormLexerPrepass.java	12 May 2010 02:34:14 -0000
@@ -64,9 +64,9 @@
 		this.in = new OffsetLineReader(in);
 	}
 	
-	public FixedFormLexerPrepass(InputStream in) {
-		this(new InputStreamReader(in));
-	}
+//	public FixedFormLexerPrepass(InputStream in) {
+//		this(new InputStreamReader(in));
+//	}
 	
 	public int getLine(int absCharPos) {
 		if (absCharPos<0) return 0;
Index: lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java,v
retrieving revision 1.16
diff -u -r1.16 FreeFormLexerPhase1.java
--- lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java	5 Mar 2010 16:46:59 -0000	1.16
+++ lexer/org/eclipse/photran/internal/core/lexer/FreeFormLexerPhase1.java	12 May 2010 02:34:20 -0000
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 3/5/10 10:46 AM */
+/* The following code was generated by JFlex 1.4.1 on 5/11/10 8:56 PM */
 
 /*******************************************************************************
  * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
@@ -29,8 +29,10 @@
  */
 
 package org.eclipse.photran.internal.core.lexer;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.Reader;
 import org.eclipse.core.resources.IFile;
 
 @SuppressWarnings("unused")
@@ -38,7 +40,7 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 3/5/10 10:46 AM from the specification file
+ * on 5/11/10 8:56 PM from the specification file
  * <tt>FreeFormLexerPhase1.flex</tt>
  */
 public class FreeFormLexerPhase1 implements ILexer {
@@ -2517,9 +2519,9 @@
     private String filename = "<stdin>";
     protected TokenFactory tokenFactory;
 
-    public FreeFormLexerPhase1(java.io.InputStream in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
+    public FreeFormLexerPhase1(Reader in, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
     {
-        this(new LineAppendingInputStream(in));
+        this(new LineAppendingReader(in));
         this.lastTokenFile = new FileOrIFile(file);
         this.filename = filename;
         this.tokenFactory = tokenFactory;
Index: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core/lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java,v
retrieving revision 1.3
diff -u -r1.3 LineAppendingInputStream.java
--- lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java	3 Nov 2009 16:11:00 -0000	1.3
+++ lexer/org/eclipse/photran/internal/core/lexer/LineAppendingInputStream.java	12 May 2010 02:34:20 -0000
@@ -36,6 +36,7 @@
         this.currentInputStream = originalInputStream;
     }
 
+    @Override
     public int read() throws IOException
     {
         int result = currentInputStream.read();
@@ -44,7 +45,6 @@
         {
             currentInputStream = eolInputStream;
             result = currentInputStream.read();
-            currentInputStream.available();
         }
         else
         {
@@ -64,6 +64,7 @@
         private static final byte[] EOL = System.getProperty("line.separator").getBytes();
         private int eolByte = 0;
 
+        @Override
         public int read() throws IOException
         {
             return eolByte < EOL.length ? EOL[eolByte++] : -1;
Index: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
===================================================================
RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
diff -N lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ lexer/org/eclipse/photran/internal/core/lexer/LineAppendingReader.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,84 @@
+/*******************************************************************************
+ * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     UIUC - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Wraps another <code>Reader</code>, and, if the wrapped stream does not end
+ * with an end-of-line character, appends the OS-dependent end-of-line sequence to
+ * the end of the stream.
+ * 
+ * @author Jeff Overbey
+ */
+public final class LineAppendingReader extends SingleCharReader
+{
+    private Reader originalReader;
+    private Reader eolReader;
+    private Reader currentReader;
+    
+    private int lastChar = -1;
+    
+    public LineAppendingReader(Reader streamToWrap)
+    {
+        this.originalReader = streamToWrap;
+        this.eolReader = new EOLReader();
+        
+        this.currentReader = originalReader;
+    }
+
+    @Override
+    public int read() throws IOException
+    {
+        int result = currentReader.read();
+        
+        if (result == -1 && currentReader == originalReader && !isEOL(lastChar))
+        {
+            currentReader = eolReader;
+            result = currentReader.read();
+        }
+        else
+        {
+            lastChar = result;
+        }
+        
+        return result;
+    }
+
+    private boolean isEOL(int c)
+    {
+        return c =='\n' || c == '\r';
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        originalReader.close();
+    }
+
+    private static class EOLReader extends SingleCharReader
+    {
+        private static final byte[] EOL = System.getProperty("line.separator").getBytes();
+        private int eolByte = 0;
+
+        @Override
+        public int read() throws IOException
+        {
+            return eolByte < EOL.length ? EOL[eolByte++] : -1;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+        }
+    }
+}
Index: lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
===================================================================
RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
diff -N lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java
--- lexer/org/eclipse/photran/internal/core/lexer/LineInputStream.java	16 Aug 2007 22:58:21 -0000	1.2
+++ /dev/null	1 Jan 1970 00:00:00 -0000
@@ -1,260 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- *     UIUC - Initial API and implementation
- *******************************************************************************/
-package org.eclipse.photran.internal.core.lexer;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Java's <code>BufferedReader</code> can read a line of text from another
- * reader, but it omits the end-of-line character(s).  This one doesn't.
- * Subclass if necessary.
- * 
- * @author Jeff Overbey
- */
-public final class LineInputStream extends InputStream implements CharSequence
-{
-    protected InputStream in;
-    
-    protected String filename;
-    protected int startOffset;
-    protected int startLineNum;
-    protected int restartLine = -1;
-    protected int restartOffset = -1;
-    
-    protected char[] currentLine = new char[4096];
-    protected int currentLineLength = -1;
-    protected int offsetInCurrentLine = -1;
-    protected int offsetInStream = 0;
-    protected int lineNumInCurrentStream = 0;
-    protected boolean atEOF = false;
-
-    public LineInputStream(InputStream readFrom, String filename) throws IOException
-    {
-        this(readFrom, filename, 0, 0);
-    }
-
-    public LineInputStream(InputStream readFrom, String filename, int startOffset, int startLineNum) throws IOException
-    {
-        this.in = readFrom;
-        this.filename = filename;
-        this.startOffset = startOffset;
-        this.startLineNum = startLineNum;
-        advanceToNextLine();
-    }
-
-    /**
-     * THIS IS EXPENSIVE -- The text of the line is COPIED into a <code>String</code>.
-     * @return
-     * @throws IOException
-     */
-    public String currentLine() throws IOException
-    {
-        ensureLineLoaded();
-        return atEOF() ? null : String.valueOf(currentLine, offsetInCurrentLine, currentLineLength-offsetInCurrentLine);
-    }
-    
-    public boolean atBOL() throws IOException
-    {
-        ensureLineLoaded();
-        return offsetInCurrentLine == 0;
-    }
-    
-    public boolean atEOF() throws IOException
-    {
-        return offsetInCurrentLine >= currentLineLength && atEOF;
-    }
-
-    protected void ensureLineLoaded() throws IOException
-    {
-        if (!atEOF && offsetInCurrentLine >= currentLineLength)
-            advanceToNextLine();
-    }
-
-    protected int peek() throws IOException
-    {
-        return atEOF() ? -1 : currentLine[offsetInCurrentLine];
-    }
-
-    public void advanceToNextLine() throws IOException
-    {
-        int c;
-        
-        offsetInCurrentLine = 0;
-        offsetInStream += Math.max(currentLineLength-offsetInCurrentLine, 0);
-        
-        if (atEOF) return;
-        
-        lineNumInCurrentStream++;
-        
-        if (currentLineLength < 0)
-        {
-            // First attempt to read a line
-            currentLineLength = 0;
-        }
-        else
-        {
-            // Start with the extra character from the last read
-            currentLine[0] = currentLine[currentLineLength];
-            currentLineLength = 1;
-        }
-        
-        for (c = in.read(); c != -1 && c != '\n' && c != '\r'; c = in.read())
-            append(c);
-        
-        if (c == -1) append('\n'); // Always end files with a newline
-        
-        for ( ; c != -1 && (c == '\n' || c == '\r'); c = in.read())
-            append(c);
-        
-        if (c == -1) atEOF = true;
-
-        // To find the end of the line, we had to read an extra character; don't lose it!
-        currentLine[currentLineLength] = (char)c;
-    }
-    
-    private void append(int c)
-    {
-        if (currentLineLength + 1 == currentLine.length)
-        {
-            char[] newCurrentLine = new char[currentLine.length*2];
-            System.arraycopy(currentLine, 0, newCurrentLine, 0, currentLine.length);
-            currentLine = newCurrentLine;
-        }
-        
-        currentLine[currentLineLength++] = (char)c;
-    }
-    
-    public String getFilename()
-    {
-        return filename;
-    }
-    
-    public int getFileOffsetOfNextRead()
-    {
-        return offsetInStream;
-    }
-    
-    public int getFileLineOfNextRead()
-    {
-        return lineNumInCurrentStream;
-    }
-    
-    public int getTotalOffsetOfNextRead()
-    {
-        return startOffset + offsetInStream;
-    }
-    
-    public int getTotalLineOfNextRead()
-    {
-        return startLineNum + lineNumInCurrentStream;
-    }
-
-    public int getStartLine()
-    {
-        return startLineNum;
-    }
-
-    public int getStartOffset()
-    {
-        return startOffset;
-    }
-
-    public int getRestartLine()
-    {
-        return restartLine;
-    }
-
-    public void setRestartLine(int restartLine)
-    {
-        this.restartLine = restartLine;
-    }
-
-    public int getRestartOffset()
-    {
-        return restartOffset;
-    }
-
-    public void setRestartOffset(int restartOffset)
-    {
-        this.restartOffset = restartOffset;
-    }
-    
-    ///////////////////////////////////////////////////////////////////////////
-    // INPUTSTREAM IMPLEMENTATION
-    ///////////////////////////////////////////////////////////////////////////
-
-    public int read() throws IOException
-    {
-        ensureLineLoaded();
-        int result = peek();
-        if (result >= 0)
-        {
-            offsetInCurrentLine++;
-            offsetInStream++;
-        }
-        return result;
-    }
-
-    ///////////////////////////////////////////////////////////////////////////
-    // CHARSEQUENCE IMPLEMENTATION
-    ///////////////////////////////////////////////////////////////////////////
-    
-    public char charAt(int index)
-    {
-        return currentLine[offsetInCurrentLine+index];
-    }
-
-    public int length()
-    {
-        return currentLineLength-offsetInCurrentLine;
-    }
-
-    public CharSequence subSequence(final int start, final int end)
-    {
-        if (start < offsetInCurrentLine || end >= length())
-            throw new IllegalArgumentException();
-        
-        return new SubSequence(start, end);
-    }
-    
-    private final class SubSequence implements CharSequence
-    {
-        private final int start;
-        private final int end;
-
-        private SubSequence(int start, int end)
-        {
-            this.start = start;
-            this.end = end;
-        }
-
-        public char charAt(int index)
-        {
-            return currentLine[offsetInCurrentLine+start];
-        }
-
-        public int length()
-        {
-            return end-start;
-        }
-
-        public CharSequence subSequence(int subseqStart, int subseqEnd)
-        {
-            return new SubSequence(start+subseqStart, start+subseqEnd);
-        }
-        
-        public String toString()
-        {
-            return String.valueOf(currentLine, start, end-start);
-        }
-    }
-}
Index: lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
===================================================================
RCS file: lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
diff -N lexer/org/eclipse/photran/internal/core/lexer/LineReader.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ lexer/org/eclipse/photran/internal/core/lexer/LineReader.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,266 @@
+/*******************************************************************************
+ * Copyright (c) 2007 University of Illinois at Urbana-Champaign and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     UIUC - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Java's <code>BufferedReader</code> can read a line of text from another
+ * reader, but it omits the end-of-line character(s).  This one doesn't.
+ * Subclass if necessary.
+ * 
+ * @author Jeff Overbey
+ */
+public final class LineReader extends SingleCharReader implements CharSequence
+{
+    protected Reader in;
+    
+    protected String filename;
+    protected int startOffset;
+    protected int startLineNum;
+    protected int restartLine = -1;
+    protected int restartOffset = -1;
+    
+    protected char[] currentLine = new char[4096];
+    protected int currentLineLength = -1;
+    protected int offsetInCurrentLine = -1;
+    protected int offsetInStream = 0;
+    protected int lineNumInCurrentStream = 0;
+    protected boolean atEOF = false;
+
+    public LineReader(Reader readFrom, String filename) throws IOException
+    {
+        this(readFrom, filename, 0, 0);
+    }
+
+    public LineReader(Reader readFrom, String filename, int startOffset, int startLineNum) throws IOException
+    {
+        this.in = readFrom;
+        this.filename = filename;
+        this.startOffset = startOffset;
+        this.startLineNum = startLineNum;
+        advanceToNextLine();
+    }
+
+    /**
+     * THIS IS EXPENSIVE -- The text of the line is COPIED into a <code>String</code>.
+     * @return
+     * @throws IOException
+     */
+    public String currentLine() throws IOException
+    {
+        ensureLineLoaded();
+        return atEOF() ? null : String.valueOf(currentLine, offsetInCurrentLine, currentLineLength-offsetInCurrentLine);
+    }
+    
+    public boolean atBOL() throws IOException
+    {
+        ensureLineLoaded();
+        return offsetInCurrentLine == 0;
+    }
+    
+    public boolean atEOF() throws IOException
+    {
+        return offsetInCurrentLine >= currentLineLength && atEOF;
+    }
+
+    protected void ensureLineLoaded() throws IOException
+    {
+        if (!atEOF && offsetInCurrentLine >= currentLineLength)
+            advanceToNextLine();
+    }
+
+    protected int peek() throws IOException
+    {
+        return atEOF() ? -1 : currentLine[offsetInCurrentLine];
+    }
+
+    public void advanceToNextLine() throws IOException
+    {
+        int c;
+        
+        offsetInCurrentLine = 0;
+        offsetInStream += Math.max(currentLineLength-offsetInCurrentLine, 0);
+        
+        if (atEOF) return;
+        
+        lineNumInCurrentStream++;
+        
+        if (currentLineLength < 0)
+        {
+            // First attempt to read a line
+            currentLineLength = 0;
+        }
+        else
+        {
+            // Start with the extra character from the last read
+            currentLine[0] = currentLine[currentLineLength];
+            currentLineLength = 1;
+        }
+        
+        for (c = in.read(); c != -1 && c != '\n' && c != '\r'; c = in.read())
+            append(c);
+        
+        if (c == -1) append('\n'); // Always end files with a newline
+        
+        for ( ; c != -1 && (c == '\n' || c == '\r'); c = in.read())
+            append(c);
+        
+        if (c == -1) atEOF = true;
+
+        // To find the end of the line, we had to read an extra character; don't lose it!
+        currentLine[currentLineLength] = (char)c;
+    }
+    
+    private void append(int c)
+    {
+        if (currentLineLength + 1 == currentLine.length)
+        {
+            char[] newCurrentLine = new char[currentLine.length*2];
+            System.arraycopy(currentLine, 0, newCurrentLine, 0, currentLine.length);
+            currentLine = newCurrentLine;
+        }
+        
+        currentLine[currentLineLength++] = (char)c;
+    }
+    
+    public String getFilename()
+    {
+        return filename;
+    }
+    
+    public int getFileOffsetOfNextRead()
+    {
+        return offsetInStream;
+    }
+    
+    public int getFileLineOfNextRead()
+    {
+        return lineNumInCurrentStream;
+    }
+    
+    public int getTotalOffsetOfNextRead()
+    {
+        return startOffset + offsetInStream;
+    }
+    
+    public int getTotalLineOfNextRead()
+    {
+        return startLineNum + lineNumInCurrentStream;
+    }
+
+    public int getStartLine()
+    {
+        return startLineNum;
+    }
+
+    public int getStartOffset()
+    {
+        return startOffset;
+    }
+
+    public int getRestartLine()
+    {
+        return restartLine;
+    }
+
+    public void setRestartLine(int restartLine)
+    {
+        this.restartLine = restartLine;
+    }
+
+    public int getRestartOffset()
+    {
+        return restartOffset;
+    }
+
+    public void setRestartOffset(int restartOffset)
+    {
+        this.restartOffset = restartOffset;
+    }
+    
+    ///////////////////////////////////////////////////////////////////////////
+    // INPUTSTREAM IMPLEMENTATION
+    ///////////////////////////////////////////////////////////////////////////
+
+    public int read() throws IOException
+    {
+        ensureLineLoaded();
+        int result = peek();
+        if (result >= 0)
+        {
+            offsetInCurrentLine++;
+            offsetInStream++;
+        }
+        return result;
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+    // CHARSEQUENCE IMPLEMENTATION
+    ///////////////////////////////////////////////////////////////////////////
+    
+    public char charAt(int index)
+    {
+        return currentLine[offsetInCurrentLine+index];
+    }
+
+    public int length()
+    {
+        return currentLineLength-offsetInCurrentLine;
+    }
+
+    public CharSequence subSequence(final int start, final int end)
+    {
+        if (start < offsetInCurrentLine || end >= length())
+            throw new IllegalArgumentException();
+        
+        return new SubSequence(start, end);
+    }
+    
+    private final class SubSequence implements CharSequence
+    {
+        private final int start;
+        private final int end;
+
+        private SubSequence(int start, int end)
+        {
+            this.start = start;
+            this.end = end;
+        }
+
+        public char charAt(int index)
+        {
+            return currentLine[offsetInCurrentLine+start];
+        }
+
+        public int length()
+        {
+            return end-start;
+        }
+
+        public CharSequence subSequence(int subseqStart, int subseqEnd)
+        {
+            return new SubSequence(start+subseqStart, start+subseqEnd);
+        }
+        
+        public String toString()
+        {
+            return String.valueOf(currentLine, start, end-start);
+        }
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        in.close();
+    }
+}
Index: lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
===================================================================
RCS file: lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
diff -N lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ lexer/org/eclipse/photran/internal/core/lexer/SingleCharReader.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,66 @@
+/*******************************************************************************
+ * Copyright (c) 2000, 2007 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     IBM Corporation - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+
+/**
+ * Copied from org.eclipse.jface.internal.text.html.SingleCharReader
+ * <p>
+ * Moved into this package from <code>org.eclipse.jface.internal.text.revisions</code>.</p>
+ */
+public abstract class SingleCharReader extends Reader {
+
+	/**
+	 * @see Reader#read()
+	 */
+	public abstract int read() throws IOException;
+
+	/**
+	 * @see Reader#read(char[],int,int)
+	 */
+	public int read(char cbuf[], int off, int len) throws IOException {
+		int end= off + len;
+		for (int i= off; i < end; i++) {
+			int ch= read();
+			if (ch == -1) {
+				if (i == off)
+					return -1;
+				return i - off;
+			}
+			cbuf[i]= (char)ch;
+		}
+		return len;
+	}
+
+	/**
+	 * @see Reader#ready()
+	 */
+    public boolean ready() throws IOException {
+		return true;
+	}
+
+	/**
+	 * Returns the readable content as string.
+	 * @return the readable content as string
+	 * @exception IOException in case reading fails
+	 */
+	public String getString() throws IOException {
+		StringBuffer buf= new StringBuffer();
+		int ch;
+		while ((ch= read()) != -1) {
+			buf.append((char)ch);
+		}
+		return buf.toString();
+	}
+}
#P org.eclipse.photran.core.vpg
Index: parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java,v
retrieving revision 1.5
diff -u -r1.5 LexerFactory.java
--- parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java	11 Mar 2010 21:41:10 -0000	1.5
+++ parser/org/eclipse/photran/internal/core/lexer/LexerFactory.java	12 May 2010 02:34:36 -0000
@@ -10,11 +10,12 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.core.lexer;
 
-import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.FileReader;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.runtime.CoreException;
@@ -29,19 +30,23 @@
 {
     private LexerFactory() {;}
     
-    public static IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
+    public static IAccumulatingLexer createLexer(Reader in, IFile file, String filename, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
     {
         return sourceForm.createLexer(in, file, filename, accumulateWhitetext);
     }
     
     public static IAccumulatingLexer createLexer(File file, SourceForm sourceForm, boolean accumulateWhitetext) throws IOException
     {
-        return createLexer(new BufferedInputStream(new FileInputStream(file)), null, file.getAbsolutePath(), sourceForm, accumulateWhitetext);
+        return createLexer(new BufferedReader(new FileReader(file)), null, file.getAbsolutePath(), sourceForm, accumulateWhitetext);
     }
     
     public static IAccumulatingLexer createLexer(IFile file, SourceForm sourceForm, boolean accumulateWhitetext) throws CoreException, IOException
     {
-        return createLexer(file.getContents(true), file, determineFilename(file), sourceForm, accumulateWhitetext);
+        return createLexer(new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset())),
+            file,
+            determineFilename(file),
+            sourceForm,
+            accumulateWhitetext);
     }
 
     private static String determineFilename(IFile file)
Index: parser/org/eclipse/photran/internal/core/lexer/SourceForm.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/SourceForm.java,v
retrieving revision 1.12
diff -u -r1.12 SourceForm.java
--- parser/org/eclipse/photran/internal/core/lexer/SourceForm.java	5 Mar 2010 21:44:11 -0000	1.12
+++ parser/org/eclipse/photran/internal/core/lexer/SourceForm.java	12 May 2010 02:34:36 -0000
@@ -12,6 +12,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.runtime.CoreException;
@@ -21,7 +22,6 @@
 import org.eclipse.photran.internal.core.FortranCorePlugin;
 import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.IncludeLoaderCallback;
 import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.PreprocessingFreeFormLexerPhase1;
-import org.eclipse.photran.internal.core.vpg.PhotranVPG;
 
 /**
  * Contains constants enumerating the various Fortran source forms (fixed form, free form, free
@@ -63,13 +63,13 @@
      *     </ol>
      * </ol>
      */
-    public abstract IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException;
+    public abstract IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException;
     
     public abstract String getDescription(String filename);
     
     public static final SourceForm UNPREPROCESSED_FREE_FORM = new SourceForm()
     {
-        @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
+        @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
         {
             return new LexerPhase3(new FreeFormLexerPhase2(new FreeFormLexerPhase1(in, file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext)));
         }
@@ -79,7 +79,7 @@
     
     public static final SourceForm FIXED_FORM = new SourceForm()
     {
-        @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
+        @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
         {
             return new LexerPhase3(new FixedFormLexerPhase2(in, file, filename, ASTTokenFactory.getInstance()));
         }
@@ -96,7 +96,7 @@
     {
         return new SourceForm()
         {
-            @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
+            @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
             {
                 SourceForm sf = findPreferredSourceForm(filename);
                 if (sf != null)
Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java,v
retrieving revision 1.3
diff -u -r1.3 FortranPreprocessor.java
--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java	13 Oct 2009 20:25:21 -0000	1.3
+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/FortranPreprocessor.java	12 May 2010 02:34:36 -0000
@@ -12,7 +12,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.Stack;
@@ -21,15 +21,16 @@
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.photran.internal.core.lexer.FreeFormLexerPhase1;
-import org.eclipse.photran.internal.core.lexer.LineInputStream;
+import org.eclipse.photran.internal.core.lexer.LineReader;
+import org.eclipse.photran.internal.core.lexer.SingleCharReader;
 
 /**
- * An <code>InputStream</code> that recognizes and processes Fortran INCLUDE lines.
+ * An <code>Reader</code> that recognizes and processes Fortran INCLUDE lines.
  * <p> 
- * This class implements an {@link InputStream} the provides the contents of a
+ * This class implements an {@link Reader} the provides the contents of a
  * Fortran file <i>after</i> INCLUDE lines have been processed.  Photran's
  * machine-constructed, free-form lexer ({@link FreeFormLexerPhase1}) tokenizes
- * this InputStream, oblivious to the fact that it is actually tokenizing
+ * this Reader, oblivious to the fact that it is actually tokenizing
  * preprocessed text.  However, a special subclass
  * ({@link PreprocessingFreeFormLexerPhase1}) takes care of mapping the
  * preprocessed tokens back to their images and locations in the original files.
@@ -41,10 +42,10 @@
  * 
  * @author Jeff Overbey
  */
-public final class FortranPreprocessor extends InputStream
+public final class FortranPreprocessor extends SingleCharReader
 {
     /**
-     * Utility class.  Originally, we simply used a <pre>Stack<LineInputStream></pre>,
+     * Utility class.  Originally, we simply used a <pre>Stack<LineReader></pre>,
      * but a profile revealed that Stack#peek was consuming a large amount of time
      * due to repeated invocations.  Now, the #topStream field in this class is
      * accessed instead, resulting in a significant performance improvement
@@ -52,13 +53,13 @@
      */
     private static final class StreamStack
     {
-        private Stack<LineInputStream> streamStack = new Stack<LineInputStream>();
-        private LineInputStream topStream = null;
+        private Stack<LineReader> streamStack = new Stack<LineReader>();
+        private LineReader topStream = null;
         
-        public void push(LineInputStream lineInputStream)
+        public void push(LineReader lineReader)
         {
-            streamStack.push(lineInputStream);
-            topStream = lineInputStream;
+            streamStack.push(lineReader);
+            topStream = lineReader;
         }
 
         public int size()
@@ -93,13 +94,13 @@
     private ArrayList<Integer> fileStartLines;
     private ArrayList<Integer> fileStartLineAdjustments;
     
-    public FortranPreprocessor(InputStream readFrom, IFile file, String filename, IncludeLoaderCallback callback) throws IOException
+    public FortranPreprocessor(Reader readFrom, IFile file, String filename, IncludeLoaderCallback callback) throws IOException
     {
         this.topLevelFile = file;
         this.callback = callback;
         
         streamStack = new StreamStack();
-        streamStack.push(new LineInputStream(readFrom, filename));
+        streamStack.push(new LineReader(readFrom, filename));
         
         directivesInTopLevelFile = new LinkedList<FortranIncludeDirective>();
         directivesInTopLevelFile.add(null);
@@ -167,9 +168,10 @@
         throw new IllegalArgumentException();
     }
     
+    @Override
     public int read() throws IOException
     {
-        LineInputStream currentStream = streamStack.topStream;
+        LineReader currentStream = streamStack.topStream;
         if (currentStream.atBOL()) checkForInclude();
         currentStream = streamStack.topStream;
         if (currentStream.atEOF()) finishInclude();
@@ -192,11 +194,11 @@
         Matcher m = INCLUDE_LINE.matcher(streamStack.topStream);
         if (m.matches())
         {
-            LineInputStream origStream = streamStack.topStream;
+            LineReader origStream = streamStack.topStream;
             String includeLine = origStream.currentLine();
             String fileToInclude = m.group(INCLUDE_LINE_CAPTURING_GROUP_OF_FILENAME);
             
-            InputStream newStream = findIncludedFile(fileToInclude);
+            Reader newStream = findIncludedFile(fileToInclude);
             if (newStream != null)
             {
                 if (inTopLevelFile())
@@ -208,7 +210,7 @@
                 origStream.setRestartOffset(offset + includeLine.length());
                 origStream.setRestartLine(line + 1);
                 
-                streamStack.push(new LineInputStream(newStream, fileToInclude, offset, line));
+                streamStack.push(new LineReader(newStream, fileToInclude, offset, line));
                 
                 fileNames.add(fileToInclude);
                 fileStartOffsets.add(new Integer(offset));
@@ -222,7 +224,7 @@
         }
     }
 
-    protected InputStream findIncludedFile(String fileToInclude) throws IOException
+    protected Reader findIncludedFile(String fileToInclude) throws IOException
     {
         try
         {
@@ -282,4 +284,10 @@
     {
         return topLevelFile;
     }
+
+    @Override
+    public void close() throws IOException
+    {
+        streamStack.topStream.close();
+    }
 }
Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java,v
retrieving revision 1.2
diff -u -r1.2 IncludeLoaderCallback.java
--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java	11 Mar 2010 21:41:10 -0000	1.2
+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/IncludeLoaderCallback.java	12 May 2010 02:34:36 -0000
@@ -10,8 +10,11 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include;
 
+import java.io.BufferedReader;
 import java.io.FileNotFoundException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
 
 import org.eclipse.core.resources.IContainer;
 import org.eclipse.core.resources.IFile;
@@ -44,11 +47,19 @@
      * @return <code>InputStream</code>, not null
      * @throws FileNotFoundException if the file cannot be found
      */
-    public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
+    public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
     {
         try
         {
-        	return getIncludedFile(fileToInclude).getContents(true);
+        	IFile file = getIncludedFile(fileToInclude);
+            try
+            {
+                return new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
+            }
+            catch (UnsupportedEncodingException e)
+            {
+                return new BufferedReader(new InputStreamReader(file.getContents(true)));
+            }
         }
         catch (CoreException e)
         {
Index: parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java,v
retrieving revision 1.2
diff -u -r1.2 PreprocessingFreeFormLexerPhase1.java
--- parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java	13 Oct 2009 20:25:21 -0000	1.2
+++ parser/org/eclipse/photran/internal/core/lexer/preprocessor/fortran_include/PreprocessingFreeFormLexerPhase1.java	12 May 2010 02:34:36 -0000
@@ -12,12 +12,13 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.photran.internal.core.lexer.ASTTokenFactory;
 import org.eclipse.photran.internal.core.lexer.FreeFormLexerPhase1;
 import org.eclipse.photran.internal.core.lexer.LexerException;
-import org.eclipse.photran.internal.core.lexer.LineAppendingInputStream;
+import org.eclipse.photran.internal.core.lexer.LineAppendingReader;
 import org.eclipse.photran.internal.core.lexer.Token;
 import org.eclipse.photran.internal.core.lexer.TokenFactory;
 
@@ -39,9 +40,9 @@
     private FortranPreprocessor preprocessor;
     private FortranIncludeDirective lastDirective;
 
-    public PreprocessingFreeFormLexerPhase1(InputStream in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
+    public PreprocessingFreeFormLexerPhase1(Reader in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
     {
-        this(new FortranPreprocessor(new LineAppendingInputStream(in), file, filename, callback), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
+        this(new FortranPreprocessor(new LineAppendingReader(in), file, filename, callback), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
     }
     
     // This would not be here if we could assign the preprocessor to a variable in the above ctor (grrr)
Index: src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java,v
retrieving revision 1.2
diff -u -r1.2 FortranResourceRefactoring.java
--- src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java	8 May 2010 08:02:37 -0000	1.2
+++ src/org/eclipse/photran/internal/core/refactoring/infrastructure/FortranResourceRefactoring.java	12 May 2010 02:34:39 -0000
@@ -11,6 +11,8 @@
 package org.eclipse.photran.internal.core.refactoring.infrastructure;
 
 import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -271,8 +273,9 @@
     {
         try
         {
+            final String charset = Charset.defaultCharset().name();
             IAccumulatingLexer lexer = LexerFactory.createLexer(
-                new ByteArrayInputStream(string.getBytes()), null, "(none)",
+                new InputStreamReader(new ByteArrayInputStream(string.getBytes(charset)), charset), null, "(none)",
                 SourceForm.UNPREPROCESSED_FREE_FORM, true);
             Parser parser = new Parser();
 
Index: src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg/src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java,v
retrieving revision 1.14
diff -u -r1.14 PhotranVPGBuilder.java
--- src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java	27 Apr 2010 16:32:01 -0000	1.14
+++ src/org/eclipse/photran/internal/core/vpg/PhotranVPGBuilder.java	12 May 2010 02:34:39 -0000
@@ -10,11 +10,15 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.core.vpg;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.FileNotFoundException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.PrintStream;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
 import java.util.List;
 
 import org.eclipse.core.resources.IFile;
@@ -26,9 +30,9 @@
 import org.eclipse.photran.internal.core.SyntaxException;
 import org.eclipse.photran.internal.core.analysis.binding.Binder;
 import org.eclipse.photran.internal.core.analysis.binding.Definition;
+import org.eclipse.photran.internal.core.analysis.binding.Definition.Visibility;
 import org.eclipse.photran.internal.core.analysis.binding.ImplicitSpec;
 import org.eclipse.photran.internal.core.analysis.binding.ScopingNode;
-import org.eclipse.photran.internal.core.analysis.binding.Definition.Visibility;
 import org.eclipse.photran.internal.core.lexer.IAccumulatingLexer;
 import org.eclipse.photran.internal.core.lexer.LexerException;
 import org.eclipse.photran.internal.core.lexer.LexerFactory;
@@ -213,7 +217,7 @@
             sourceForm = SourceForm.preprocessedFreeForm(new IncludeLoaderCallback(file.getProject())
             {
                 @Override
-                public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
+                public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
                 {
                     // When we encounter an INCLUDE directive, set up a file dependency in the VPG
 
@@ -303,11 +307,19 @@
         if (ast == null)
             throw new IllegalArgumentException(filename + " returned null AST");
         
-        ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
-        ast.getRoot().printOn(new PrintStream(out), null);
-        ast = parse(filename, new ByteArrayInputStream(out.toByteArray()));
+        try
+        {
+            final String charset = Charset.defaultCharset().name(); // Platform's default charset
+            ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
+            ast.getRoot().printOn(new PrintStream(out, false, charset), null);
+            ast = parse(filename, new InputStreamReader(new ByteArrayInputStream(out.toByteArray()), charset));
 
-        computeEdgesAndAnnotations(filename, ast);
+            computeEdgesAndAnnotations(filename, ast);
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new Error(e);
+        }
     }
 
     @Override public String getSourceCodeFromAST(IFortranAST ast)
@@ -321,7 +333,7 @@
         return parse(filename, null);
     }
 
-    private IFortranAST parse(final String filename, InputStream stream)
+    private IFortranAST parse(final String filename, Reader stream)
     {
         if (filename == null || isVirtualFile(filename)) return null;
 
@@ -331,7 +343,7 @@
             SourceForm sourceForm = determineSourceForm(filename);
             try
             {
-                if (stream == null) stream = file.getContents(true);
+                if (stream == null) stream = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
                 IAccumulatingLexer lexer = LexerFactory.createLexer(stream, file, filename, sourceForm, true);
                 long start = System.currentTimeMillis();
                 ASTExecutableProgramNode ast = parser.parse(lexer);
#P org.eclipse.photran.core.vpg.preprocessor.c
Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java,v
retrieving revision 1.2
diff -u -r1.2 CPreprocessedSourceForm.java
--- src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java	12 Mar 2010 16:47:09 -0000	1.2
+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessedSourceForm.java	12 May 2010 02:34:46 -0000
@@ -11,7 +11,7 @@
 package org.eclipse.photran.internal.core.lexer;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.photran.internal.core.lexer.preprocessor.fortran_include.IncludeLoaderCallback;
@@ -30,7 +30,7 @@
     	this.callback = callback;
     }
 
-    @Override public IAccumulatingLexer createLexer(InputStream in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
+    @Override public IAccumulatingLexer createLexer(Reader in, IFile file, String filename, boolean accumulateWhitetext) throws IOException
     {
         return new LexerPhase3(new FreeFormLexerPhase2(new CPreprocessingFreeFormLexerPhase1(in, file, filename, callback, accumulateWhitetext)));
     }
Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java,v
retrieving revision 1.1
diff -u -r1.1 CPreprocessingFreeFormLexerPhase1.java
--- src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java	2 Mar 2010 22:01:25 -0000	1.1
+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessingFreeFormLexerPhase1.java	12 May 2010 02:34:46 -0000
@@ -10,8 +10,12 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.core.lexer;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 
 import org.eclipse.core.resources.IFile;
@@ -20,7 +24,7 @@
 /**
  * Phase 1 lexer that handles C preprocessor directives.
  * <p>
- * This class feeds {@link FreeFormLexerPhase1} with a {@link CPreprocessingInputStream},
+ * This class feeds {@link FreeFormLexerPhase1} with a {@link CPreprocessingReader},
  * an {@link InputStream} that interprets C preprocessor directives.
  * {@link FreeFormLexerPhase1} tokenizes the preprocessed stream.  However, this
  * class subclasses {@link FreeFormLexerPhase1} and overrides its tokenization
@@ -49,13 +53,13 @@
 	
 	private Token lastReadInFile = null;
 
-    public CPreprocessingFreeFormLexerPhase1(InputStream in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
+    public CPreprocessingFreeFormLexerPhase1(Reader in, IFile file, String filename, IncludeLoaderCallback callback, boolean accumulateWhitetext) throws IOException
     {
-        this(new CPreprocessingInputStream(file, filename, new LineAppendingInputStream(in)), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
+        this(new CPreprocessingReader(file, filename, new LineAppendingReader(in)), file, filename, ASTTokenFactory.getInstance(), accumulateWhitetext);
     }
 
     // This would not be here if we could assign the preprocessor to a variable in the above ctor (grrr)
-    private CPreprocessingFreeFormLexerPhase1(CPreprocessingInputStream cpp, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
+    private CPreprocessingFreeFormLexerPhase1(CPreprocessingReader cpp, IFile file, String filename, TokenFactory tokenFactory, boolean accumulateWhitetext)
     {
         super(cpp, file, filename, tokenFactory, accumulateWhitetext);
         this.producerMap = cpp.getProducerMap();
Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
===================================================================
RCS file: src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
diff -N src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java
--- src/org/eclipse/photran/internal/core/lexer/CPreprocessingInputStream.java	2 Mar 2010 22:01:25 -0000	1.1
+++ /dev/null	1 Jan 1970 00:00:00 -0000
@@ -1,86 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- *     UIUC - Initial API and implementation
- *******************************************************************************/
-package org.eclipse.photran.internal.core.lexer;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.eclipse.core.resources.IFile;
-import org.eclipse.photran.internal.core.preprocessor.c.CppHelper;
-import org.eclipse.photran.internal.core.preprocessor.c.IToken;
-import org.eclipse.photran.internal.core.preprocessor.c.OffsetLimitReachedException;
-
-/**
- * An InputStream to read tokens obtained from the CPP. This class will
- * also make a ProducerMap when constructed that can be used to find
- * directives and macros.
- * 
- * @author Matthew Michelotti
- */
-public class CPreprocessingInputStream extends InputStream
-{
-	/**ProducerMap constructed from the CPP tokens*/
-	private ProducerMap producerMap;
-	
-	private IncludeMap includeMap;
-
-	/**current CPP token being read*/
-	private IToken curToken;
-	/**full image of curToken*/
-	private String curTokenImage;
-	/**the offset in curTokenImage to be read next*/
-	private int curTokenOffset = 0;
-	
-	/**
-	 * Basic constructor. Parses the whole file in the constructor
-	 * and turns the result into CPP tokens. A ProducerMap is made
-	 * from these tokens. The tokens are remembered, so that they
-	 * can be read as an input stream.
-	 * @param filename - name of file that inputStream corresponds to
-	 * @param inputStream - InputStream to read file data from
-	 * @throws IOException
-	 */
-	public CPreprocessingInputStream(IFile file, String filename, InputStream inputStream) throws IOException
-	{
-		try
-		{
-			CppHelper cpp = new CppHelper(file, filename, inputStream);
-			curToken = cpp.getRemainingTokens();
-			curTokenImage = CppHelper.getFullImage(curToken);
-			producerMap = new ProducerMap(curToken);
-			includeMap = new IncludeMap(curToken, file);
-		}catch(OffsetLimitReachedException e) {throw new Error(e);}
-	}
-	
-	@Override public int read() throws IOException
-	{
-		if(curToken == null) return -1;
-		while(curTokenOffset >= curTokenImage.length()) {
-			curToken = curToken.getNext();
-			if(curToken == null) return -1;
-			curTokenImage = CppHelper.getFullImage(curToken);
-			curTokenOffset = 0;
-		}
-		char result = curTokenImage.charAt(curTokenOffset);
-		curTokenOffset++;
-		
-		return result;
-	}
-	
-	/**@return the ProducerMap object made at construction*/
-	public ProducerMap getProducerMap() {
-		return producerMap;
-	}
-	
-	public IncludeMap getIncludeMap() {
-		return includeMap;
-	}
-}
Index: src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
===================================================================
RCS file: src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
diff -N src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/org/eclipse/photran/internal/core/lexer/CPreprocessingReader.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,91 @@
+/*******************************************************************************
+ * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     UIUC - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.photran.internal.core.preprocessor.c.CppHelper;
+import org.eclipse.photran.internal.core.preprocessor.c.IToken;
+import org.eclipse.photran.internal.core.preprocessor.c.OffsetLimitReachedException;
+
+/**
+ * An InputStream to read tokens obtained from the CPP. This class will
+ * also make a ProducerMap when constructed that can be used to find
+ * directives and macros.
+ * 
+ * @author Matthew Michelotti
+ */
+public class CPreprocessingReader extends SingleCharReader
+{
+	/**ProducerMap constructed from the CPP tokens*/
+	private ProducerMap producerMap;
+	
+	private IncludeMap includeMap;
+
+	/**current CPP token being read*/
+	private IToken curToken;
+	/**full image of curToken*/
+	private String curTokenImage;
+	/**the offset in curTokenImage to be read next*/
+	private int curTokenOffset = 0;
+	
+	/**
+	 * Basic constructor. Parses the whole file in the constructor
+	 * and turns the result into CPP tokens. A ProducerMap is made
+	 * from these tokens. The tokens are remembered, so that they
+	 * can be read as an input stream.
+	 * @param filename - name of file that inputStream corresponds to
+	 * @param reader - Reader to read file data from
+	 * @throws IOException
+	 */
+	public CPreprocessingReader(IFile file, String filename, Reader reader) throws IOException
+	{
+		try
+		{
+			CppHelper cpp = new CppHelper(file, filename, reader);
+			curToken = cpp.getRemainingTokens();
+			curTokenImage = CppHelper.getFullImage(curToken);
+			producerMap = new ProducerMap(curToken);
+			includeMap = new IncludeMap(curToken, file);
+		}catch(OffsetLimitReachedException e) {throw new Error(e);}
+	}
+	
+	@Override public int read() throws IOException
+	{
+		if(curToken == null) return -1;
+		while(curTokenOffset >= curTokenImage.length()) {
+			curToken = curToken.getNext();
+			if(curToken == null) return -1;
+			curTokenImage = CppHelper.getFullImage(curToken);
+			curTokenOffset = 0;
+		}
+		char result = curTokenImage.charAt(curTokenOffset);
+		curTokenOffset++;
+		
+		return result;
+	}
+	
+	/**@return the ProducerMap object made at construction*/
+	public ProducerMap getProducerMap() {
+		return producerMap;
+	}
+	
+	public IncludeMap getIncludeMap() {
+		return includeMap;
+	}
+
+    @Override
+    public void close() throws IOException
+    {
+    }
+}
Index: src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c/src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java,v
retrieving revision 1.2
diff -u -r1.2 CppHelper.java
--- src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java	4 Mar 2010 08:01:44 -0000	1.2
+++ src/org/eclipse/photran/internal/core/preprocessor/c/CppHelper.java	12 May 2010 02:34:51 -0000
@@ -12,9 +12,8 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
 import java.util.Arrays;
-import java.util.Map;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
@@ -24,7 +23,6 @@
 import org.eclipse.cdt.core.parser.NullLogService;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ScannerInfo;
-import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.IResource;
 
@@ -64,11 +62,11 @@
      * @param includeSearchPaths - file paths to search in for included files
      * @throws IOException
      */
-    public CppHelper(String filename, InputStream in, String[] includeSearchPaths)
+    public CppHelper(String filename, Reader in, String[] includeSearchPaths)
     		throws IOException
     {
     	//NOTE: it is important that the CodeReader gets an ABSOLUTE file path
-        cpp = new CPreprocessor(new InputStreamCodeReader(filename, in),
+        cpp = new CPreprocessor(new ReaderBasedCodeReader(filename, in),
             new ScannerInfo(null, includeSearchPaths), ParserLanguage.C,
             new NullLogService(), new GCCScannerExtensionConfiguration(),
             FileCodeReaderFactory.getInstance());
@@ -85,7 +83,7 @@
      *      Also, @param in could be different than @param file, if for example there
      *      are un-saved changes in the editor.
      */
-    public CppHelper(IResource infoProvider, String filename, InputStream in) throws IOException {
+    public CppHelper(IResource infoProvider, String filename, Reader in) throws IOException {
         //NOTE: it is important that the CodeReader gets an ABSOLUTE file path
         if (filename == null) 
             filename = "";
@@ -104,7 +102,7 @@
         }
 
         //Map<String, String> temp = scanInfo.getDefinedSymbols();
-        cpp = new CPreprocessor(new InputStreamCodeReader(filename, in),
+        cpp = new CPreprocessor(new ReaderBasedCodeReader(filename, in),
             new ScannerInfo(scanInfo.getDefinedSymbols(), scanInfo.getIncludePaths()), ParserLanguage.C, new NullLogService(), 
             new GCCScannerExtensionConfiguration(), FileCodeReaderFactory.getInstance());
     }
Index: src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
===================================================================
RCS file: src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
diff -N src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java
--- src/org/eclipse/photran/internal/core/preprocessor/c/InputStreamCodeReader.java	2 Mar 2010 22:01:24 -0000	1.1
+++ /dev/null	1 Jan 1970 00:00:00 -0000
@@ -1,63 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- *    UIUC - Initial API and implementation
- *******************************************************************************/
-package org.eclipse.photran.internal.core.preprocessor.c;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.eclipse.cdt.core.parser.CodeReader;
-
-public class InputStreamCodeReader extends CodeReader 
-{
-
-	public InputStreamCodeReader(String filename, InputStream stream)
-			throws IOException 
-	{
-		super(filename, convertToCharBuffer(stream));
-		
-		// TODO Auto-generated constructor stub
-	}
-	
-	public InputStreamCodeReader(String filename, String charSet, InputStream stream)
-			throws IOException 
-	{
-		//We are dropping out the charSet variable because it is used internally by 
-		// CodeReader to read from the FileStream. Since we want to read from the 
-		// InputStream, and we do it byte-wise, we don't really care for encoding
-		super(filename, convertToCharBuffer(stream));
-		// TODO Auto-generated constructor stub
-	}
-	
-	private static char[] convertToCharBuffer(InputStream stream)
-	{
-		int b = 0;
-		StringBuffer aggregate = new StringBuffer();
-		while(b != -1)
-		{
-			try 
-			{
-				b = stream.read();
-			} 
-			catch (IOException e) 
-			{
-				e.printStackTrace();
-				b = -1;
-				break;
-			}
-			if(b != -1)
-			{
-				aggregate.append((char)b);
-			}
-		}
-		return aggregate.toString().toCharArray();
-	}
-
-}
Index: src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
===================================================================
RCS file: src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
diff -N src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/org/eclipse/photran/internal/core/preprocessor/c/ReaderBasedCodeReader.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,59 @@
+/*******************************************************************************
+ * Copyright (c) 2009 University of Illinois at Urbana-Champaign and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    UIUC - Initial API and implementation
+ *           (Jeff Overbey) Modified to use Reader rather than InputStream
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.preprocessor.c;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.eclipse.cdt.core.parser.CodeReader;
+
+public class ReaderBasedCodeReader extends CodeReader 
+{
+
+	public ReaderBasedCodeReader(String filename, Reader stream)
+			throws IOException 
+	{
+		super(filename, convertToCharBuffer(stream));
+	}
+	
+	private static char[] convertToCharBuffer(Reader stream)
+	{
+		int b = 0;
+		StringBuffer aggregate = new StringBuffer();
+		while(b != -1)
+		{
+			try 
+			{
+				b = stream.read();
+			} 
+			catch (IOException e) 
+			{
+				e.printStackTrace();
+				b = -1;
+				break;
+			}
+			if(b != -1)
+			{
+				aggregate.append((char)b);
+			}
+		}
+		try
+        {
+            stream.close();
+        }
+        catch (IOException e)
+        {
+            e.printStackTrace();
+        }
+		return aggregate.toString().toCharArray();
+	}
+}
#P org.eclipse.photran.core.vpg.preprocessor.c.tests
Index: src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.preprocessor.c.tests/src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java,v
retrieving revision 1.1
diff -u -r1.1 CppIntegratedSourceReproductionTestCase.java
--- src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java	2 Mar 2010 22:06:58 -0000	1.1
+++ src/org/eclipse/photran/internal/core/tests/preprocessor/c/CppIntegratedSourceReproductionTestCase.java	12 May 2010 02:34:54 -0000
@@ -15,7 +15,7 @@
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
 
 import org.eclipse.photran.internal.core.lexer.CPreprocessedSourceForm;
 import org.eclipse.photran.internal.core.lexer.SourceForm;
@@ -35,7 +35,7 @@
     {
         return new CPreprocessedSourceForm(new IncludeLoaderCallback(null)
         {
-            @Override public InputStream getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
+            @Override public Reader getIncludedFileAsStream(String fileToInclude) throws FileNotFoundException
             {
                 throw new FileNotFoundException("#include lines not supported in JUnit tests");
             }
#P org.eclipse.photran.core.vpg.tests
Index: src/org/eclipse/photran/internal/tests/BaseTestCase.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/BaseTestCase.java,v
retrieving revision 1.3
diff -u -r1.3 BaseTestCase.java
--- src/org/eclipse/photran/internal/tests/BaseTestCase.java	25 Feb 2010 18:11:29 -0000	1.3
+++ src/org/eclipse/photran/internal/tests/BaseTestCase.java	12 May 2010 02:34:59 -0000
@@ -10,9 +10,8 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.tests;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.StringReader;
 
 import junit.framework.TestCase;
 
@@ -63,8 +62,7 @@
     protected <T extends IBodyConstruct> T parseStmt(String stmt) throws IOException, LexerException, SyntaxException
     {
         String program = stmt + "\nend program\n";
-        InputStream in = new ByteArrayInputStream(program.getBytes());
-        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(in, null, null, SourceForm.UNPREPROCESSED_FREE_FORM, true));
+        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(program), null, null, SourceForm.UNPREPROCESSED_FREE_FORM, true));
         assertTrue(ast != null);
         LoopReplacer.replaceAllLoopsIn(ast);
         return (T)((ASTMainProgramNode)ast.getProgramUnitList().get(0)).getBody().get(0);
Index: src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java,v
retrieving revision 1.1
diff -u -r1.1 TypeCheckerTestCase.java
--- src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java	11 Nov 2009 17:48:12 -0000	1.1
+++ src/org/eclipse/photran/internal/tests/analysis/types/TypeCheckerTestCase.java	12 May 2010 02:35:00 -0000
@@ -10,7 +10,7 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.tests.analysis.types;
 
-import java.io.ByteArrayInputStream;
+import java.io.StringReader;
 
 import junit.framework.TestCase;
 
@@ -39,7 +39,7 @@
         // a parseable program, then extract the expression from the program's AST
 
         String program = "a = " + expression + "\nend";
-        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new ByteArrayInputStream(program.getBytes()), null, "<literal text>", SourceForm.UNPREPROCESSED_FREE_FORM, true /*false*/));
+        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(program), null, "<literal text>", SourceForm.UNPREPROCESSED_FREE_FORM, true /*false*/));
         assertNotNull(ast);
         ASTMainProgramNode mainProg = (ASTMainProgramNode)ast.getProgramUnitList().get(0);
         ASTAssignmentStmtNode assignmentStmt = (ASTAssignmentStmtNode)mainProg.getBody().get(0);
Index: src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java,v
retrieving revision 1.3
diff -u -r1.3 TestOpenMPComments.java
--- src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java	4 May 2010 19:58:05 -0000	1.3
+++ src/org/eclipse/photran/internal/tests/parser/TestOpenMPComments.java	12 May 2010 02:35:00 -0000
@@ -10,8 +10,8 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.tests.parser;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -129,8 +129,7 @@
     
     private ASTExecutableProgramNode parse(SourceForm sourceForm, String string) throws IOException, LexerException, SyntaxException
     {
-        ByteArrayInputStream in = new ByteArrayInputStream(string.getBytes());
-        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(in, null, "<stdin>", sourceForm, true));
+        ASTExecutableProgramNode ast = new Parser().parse(LexerFactory.createLexer(new StringReader(string), null, "<stdin>", sourceForm, true));
         assertTrue(ast != null);
         return ast;
     }
#P org.eclipse.photran.ui.vpg
Index: src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.ui.vpg/src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java,v
retrieving revision 1.14
diff -u -r1.14 FortranEditorTasks.java
--- src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java	11 Mar 2010 20:19:30 -0000	1.14
+++ src/org/eclipse/photran/internal/ui/editor_vpg/FortranEditorTasks.java	12 May 2010 02:35:04 -0000
@@ -1,6 +1,6 @@
 package org.eclipse.photran.internal.ui.editor_vpg;
 
-import java.io.ByteArrayInputStream;
+import java.io.StringReader;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -179,7 +179,7 @@
                         String editorContents = editor.getDocumentProvider().getDocument(editor.getEditorInput()).get();
 
                         SourceForm sourceForm = determineSourceForm();
-                        IAccumulatingLexer lexer = LexerFactory.createLexer(new ByteArrayInputStream(editorContents.getBytes()),
+                        IAccumulatingLexer lexer = LexerFactory.createLexer(new StringReader(editorContents),
                                                                             editor.getIFile(),
                                                                             editor.getIFile().getName(),
                                                                             sourceForm,
#P org.eclipse.photran.ui.vpg.preprocessor.c
Index: src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.photran.ui.vpg.preprocessor.c/src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java,v
retrieving revision 1.1
diff -u -r1.1 DisplayPreprocessedSource.java
--- src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java	2 Mar 2010 22:07:38 -0000	1.1
+++ src/org/eclipse/photran/internal/ui/actions/DisplayPreprocessedSource.java	12 May 2010 02:35:09 -0000
@@ -13,15 +13,15 @@
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.InputStream;
 import java.io.PrintStream;
-import java.io.StringBufferInputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.lang.reflect.InvocationTargetException;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.jface.dialogs.MessageDialog;
 import org.eclipse.jface.text.IDocument;
-import org.eclipse.photran.internal.core.lexer.CPreprocessingInputStream;
+import org.eclipse.photran.internal.core.lexer.CPreprocessingReader;
 import org.eclipse.photran.internal.ui.editor.FixedFormFortranEditor;
 import org.eclipse.photran.internal.ui.editor.FreeFormFortranEditor;
 import org.eclipse.ui.ide.IDE;
@@ -32,7 +32,7 @@
  *
  * @author Jeff Overbey
  */
-@SuppressWarnings({ "deprecation", "restriction" })
+@SuppressWarnings({ "restriction" })
 public class DisplayPreprocessedSource extends FortranEditorActionDelegate
 {
     public void run(IProgressMonitor progressMonitor) throws InvocationTargetException, InterruptedException
@@ -42,8 +42,8 @@
         	boolean isFixedForm = getFortranEditor().isFixedForm();
         	
             IDocument doc = getFortranEditor().getIDocument();
-            InputStream in = new StringBufferInputStream(doc.get());
-            InputStream cppIn = new CPreprocessingInputStream(getFortranEditor().getIFile(), null, in);
+            Reader in = new StringReader(doc.get());
+            Reader cppIn = new CPreprocessingReader(getFortranEditor().getIFile(), null, in);
             
             File tempFile = File.createTempFile(
             	"tmp",
#P org.eclipse.rephraserengine.core
Index: src/org/eclipse/rephraserengine/core/util/StringUtil.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.core/src/org/eclipse/rephraserengine/core/util/StringUtil.java,v
retrieving revision 1.4
diff -u -r1.4 StringUtil.java
--- src/org/eclipse/rephraserengine/core/util/StringUtil.java	25 Feb 2010 14:06:01 -0000	1.4
+++ src/org/eclipse/rephraserengine/core/util/StringUtil.java	12 May 2010 02:35:12 -0000
@@ -67,7 +67,7 @@
     
     public static String read(IFile file) throws IOException, CoreException
     {
-        return read(file.getContents(true));
+        return read(new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset())));
     }
 
     public static String read(File file) throws IOException
#P org.eclipse.rephraserengine.core.vpg
Index: src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.core.vpg/src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java,v
retrieving revision 1.2
diff -u -r1.2 VPGRefactoring.java
--- src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java	11 Mar 2010 21:41:07 -0000	1.2
+++ src/org/eclipse/rephraserengine/core/vpg/refactoring/VPGRefactoring.java	12 May 2010 02:35:16 -0000
@@ -1,7 +1,9 @@
 package org.eclipse.rephraserengine.core.vpg.refactoring;
 
+import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.runtime.CoreException;
@@ -237,9 +239,10 @@
         return ast.toString();
     }
 
-    private int getSizeOf(IFile file) throws CoreException, IOException {
+    private int getSizeOf(IFile file) throws CoreException, IOException
+    {
         int size = 0;
-        InputStream in = file.getContents(true);
+        Reader in = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
         while (in.read() > -1)
             size++;
         in.close();
#P org.eclipse.rephraserengine.examples.text
Index: src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.examples.text/src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java,v
retrieving revision 1.1
diff -u -r1.1 ConvertTabsToSpacesRefactoring.java
--- src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java	3 Nov 2009 17:00:37 -0000	1.1
+++ src/org/eclipse/rephraserengine/internal/examples/text/ConvertTabsToSpacesRefactoring.java	12 May 2010 02:35:20 -0000
@@ -10,8 +10,10 @@
  *******************************************************************************/
 package org.eclipse.rephraserengine.internal.examples.text;
 
-import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.List;
 
 import org.eclipse.core.resources.IFile;
@@ -127,7 +129,7 @@
         TextFileChange change = new TextFileChange("Replace tabs in " + file.getName(), file);
         change.setEdit(new MultiTextEdit());
 
-        BufferedInputStream in = new BufferedInputStream(file.getContents(true));
+        Reader in = new BufferedReader(new InputStreamReader(file.getContents(true), file.getCharset()));
         int offset = 0;
         int column = 0;
         int numberOfTabsReplaced = 0;
#P org.eclipse.rephraserengine.ui.vpg
Index: src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java
===================================================================
RCS file: /cvsroot/tools/org.eclipse.ptp/photran/org.eclipse.rephraserengine.ui.vpg/src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java,v
retrieving revision 1.4
diff -u -r1.4 DisplayModelAction.java
--- src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java	4 Mar 2010 18:19:14 -0000	1.4
+++ src/org/eclipse/rephraserengine/internal/ui/actions/DisplayModelAction.java	12 May 2010 02:35:24 -0000
@@ -12,9 +12,9 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
+import java.io.Reader;
 import java.util.ArrayList;
 
 import org.eclipse.core.runtime.NullProgressMonitor;
@@ -42,7 +42,8 @@
                 if (filename != null)
                 {
                     ArrayList<Integer> lineMap = new ArrayList<Integer>();
-                    String fileContents = readStream(lineMap, info.getFileInEditor().getContents(true));
+                    String fileContents = readStream(lineMap,
+                        new BufferedReader(new InputStreamReader(info.getFileInEditor().getContents(true), info.getFileInEditor().getCharset())));
                     ps.println(filename);
                     ps.println();
                     Model model = new Model("edge model", new NullProgressMonitor(), 0, vpg, filename);
@@ -60,10 +61,9 @@
         }
     }
 
-    protected String readStream(ArrayList<Integer> lineMap, InputStream inputStream) throws IOException
+    protected String readStream(ArrayList<Integer> lineMap, Reader in) throws IOException
     {
         StringBuffer sb = new StringBuffer(4096);
-        BufferedReader in = new BufferedReader(new InputStreamReader(inputStream));
         for (int offset = 0, ch = in.read(); ch >= 0; ch = in.read())
         {
             sb.append((char)ch);