Skip to content

Commit

Permalink
Merge pull request #23 from GGG-KILLER/refactoring/sourcetext-reader
Browse files Browse the repository at this point in the history 
Implement SourceTextReader
  • Loading branch information
@GGG-KILLER
GGG-KILLER authored Apr 11, 2021
2 parents 2874e51 + e62f765 commit c7946a8
Show file tree
Hide file tree
Showing 8 changed files with 310 additions and 127 deletions.
1 change: 0 additions & 1 deletion src/Compilers/Lua/Portable/Loretta.CodeAnalysis.Lua.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
<ItemGroup>
<PackageReference Include="System.Collections.Immutable" Version="5.0.0" />
<PackageReference Include="Tsu" Version="2.1.1" />
<PackageReference Include="GParse" Version="5.0.0-alpha08" />
</ItemGroup>

<ItemGroup>
Expand Down
7 changes: 3 additions & 4 deletions src/Compilers/Lua/Portable/Parser/AbstractLexer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using System.Collections.Generic;
using System.Text;
using GParse.IO;
using Loretta.CodeAnalysis.Text;
using Loretta.Utilities;

Expand All @@ -9,7 +8,7 @@ namespace Loretta.CodeAnalysis.Lua.Syntax.InternalSyntax
internal class AbstractLexer
{
protected readonly SourceText _text;
protected readonly ICodeReader _reader;
protected readonly SourceTextReader _reader;
private readonly StringTable _strings;

private List<SyntaxDiagnosticInfo>? _errors;
Expand All @@ -21,7 +20,7 @@ public AbstractLexer(SourceText text)
_text = text;
// TODO: Either make an SourceTextCodeReader or reimplement the
// Lexer without the ICodeReader.
_reader = new StringCodeReader(text.ToString());
_reader = new SourceTextReader(text);
_strings = new StringTable();
}

Expand All @@ -35,7 +34,7 @@ public AbstractLexer(SourceText text)

protected int LexemeLength => Position - LexemeStart;

public void Restore(int position) => _reader.Restore(position);
public void Restore(int position) => _reader.Position = position;

protected void Start()
{
Expand Down
Empty file added src/Compilers/Lua/Portable/Parser/ICodeReader.UsedMethods.txt
View file
Empty file.
12 changes: 6 additions & 6 deletions src/Compilers/Lua/Portable/Parser/Lexer.Numbers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,23 @@ private void SkipDecimalDigits(StringBuilder builder)
{
if (digit != '_')
builder.Append(digit);
_reader.Advance(1);
_reader.Position += 1;
}
}

private double ParseBinaryNumber()
{
// Skip leading 0s
while (_reader.Peek().GetValueOrDefault() == '0')
_reader.Advance(1);
_reader.Position += 1;

var num = 0L;
var digits = 0;
var hasUnderscores = false;
char digit;
while (CharUtils.IsInRange('0', digit = _reader.Peek().GetValueOrDefault(), '1') || digit == '_')
{
_reader.Advance(1);
_reader.Position += 1;
if (digit == '_')
{
hasUnderscores = true;
Expand Down Expand Up @@ -62,15 +62,15 @@ private double ParseOctalNumber()
{
// Skip leading 0s
while (_reader.Peek().GetValueOrDefault() == '0')
_reader.Advance(1);
_reader.Position += 1;

var num = 0L;
var digits = 0;
var hasUnderscores = false;
char digit;
while (CharUtils.IsInRange('0', digit = _reader.Peek().GetValueOrDefault(), '7') || digit == '_')
{
_reader.Advance(1);
_reader.Position += 1;
if (digit == '_')
{
hasUnderscores = true;
Expand Down Expand Up @@ -173,7 +173,7 @@ void skipHexDigits()
{
if (digit != '_')
_builder.Append(digit);
_reader.Advance(1);
_reader.Position += 1;
}
}
}
Expand Down
57 changes: 31 additions & 26 deletions src/Compilers/Lua/Portable/Parser/Lexer.ShortString.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,78 +22,83 @@ private string ParseShortString()
case '\\':
{
var escapeStart = _reader.Position;
_reader.Advance(1);
_reader.Position += 1;

switch (_reader.Peek())
{
case '\r':
if (_reader.IsAt('\n', 1))
if (_reader.IsAt(1, '\n'))
{
_reader.Advance(2);
_reader.Position += 2;
_builder.Append("\r\n");
}
else
{
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\r');
}
break;

case 'a':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\a');
break;

case 'b':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\b');
break;

case 'f':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\f');
break;

case 'n':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\n');
break;

case 'r':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\r');
break;

case 't':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\t');
break;

case 'v':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\v');
break;

case '\\':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\\');
break;

case '\n':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\n');
break;

case '\'':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\'');
break;

case '"':
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('"');
break;

case 'z':
_reader.Position += 1;
_reader.SkipWhile(c => c is '\f' or '\n' or '\r' or '\t' or '\v' or ' ');
break;

case '0':
case '1':
case '2':
Expand All @@ -113,7 +118,7 @@ private string ParseShortString()

case 'x':
{
_reader.Advance(1);
_reader.Position += 1;
var parsedCharInteger = parseHexadecimalInteger(escapeStart);
if (parsedCharInteger != char.MaxValue)
_builder.Append(parsedCharInteger);
Expand All @@ -126,7 +131,7 @@ private string ParseShortString()
default:
{
// Skip the character after the escape.
_reader.Advance(1);
_reader.Position += 1;
AddError(escapeStart, _reader.Position - escapeStart, ErrorCode.ERR_InvalidStringEscape);
}
break;
Expand All @@ -138,14 +143,14 @@ private string ParseShortString()

case '\r':
{
if (_reader.IsAt('\n', 1))
if (_reader.IsAt(1, '\n'))
{
_reader.Advance(2);
_reader.Position += 2;
_builder.Append("\r\n");
}
else
{
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\r');
}

Expand All @@ -155,23 +160,23 @@ private string ParseShortString()

case '\n':
{
_reader.Advance(1);
_reader.Position += 1;
_builder.Append('\n');

AddError(charStart, _reader.Position - charStart, ErrorCode.ERR_UnescapedLineBreakInString);
}
break;

default:
_reader.Advance(1);
_reader.Position += 1;
_builder.Append(peek);
break;
}
}

if (_reader.IsNext(delim))
{
_reader.Advance(1);
_reader.Position += 1;
}
else
{
Expand All @@ -187,7 +192,7 @@ char parseDecimalInteger(int start)
char ch;
while (readChars < 3 && CharUtils.IsDecimal(ch = _reader.Peek().GetValueOrDefault()))
{
_reader.Advance(1);
_reader.Position += 1;
num = (num * 10) + (ch - '0');
readChars++;
}
Expand All @@ -210,12 +215,12 @@ char parseHexadecimalInteger(int start)
var peek = _reader.Peek().GetValueOrDefault();
if (CharUtils.IsDecimal(peek))
{
_reader.Advance(1);
_reader.Position += 1;
num = (byte) ((num << 4) | (peek - '0'));
}
else if (CharUtils.IsHexadecimal(peek))
{
_reader.Advance(1);
_reader.Position += 1;
num = (byte) ((num << 4) | (10 + CharUtils.AsciiLowerCase(peek) - 'a'));
}
else
Expand Down
Loading

0 comments on commit c7946a8

Please sign in to comment.