Use long for locations. Fixes #161

Pidgin/ParseState.ComputeSourcePos.cs

@@ -8,7 +8,7 @@ namespace Pidgin;
 
 public partial struct ParseState<TToken>
 {
-    private SourcePosDelta ComputeSourcePosDeltaAt(int location)
+    private SourcePosDelta ComputeSourcePosDeltaAt(long location)
     {
         if (location < _lastSourcePosDeltaLocation)
         {
@@ -27,26 +27,37 @@ private SourcePosDelta ComputeSourcePosDeltaAt(int location)
         }
         else if (ReferenceEquals(_sourcePosCalculator, DefaultConfiguration<TToken>.Instance.SourcePosCalculator))
         {
+            // expect delta to be small enough to fit in an int since we're subtracting
+            var delta = (int)(location - _lastSourcePosDeltaLocation);
+
             // _sourcePosCalculator just increments the col
-            return new SourcePosDelta(_lastSourcePosDelta.Lines, _lastSourcePosDelta.Cols + location - _lastSourcePosDeltaLocation);
+            return new SourcePosDelta(_lastSourcePosDelta.Lines, _lastSourcePosDelta.Cols + delta);
         }
 
         var pos = _lastSourcePosDelta;
-        for (var i = _lastSourcePosDeltaLocation - _bufferStartLocation; i < location - _bufferStartLocation; i++)
+
+        // expect start and end to be small enough to fit in an int since we're subtracting
+        var start = (int)(_lastSourcePosDeltaLocation - _bufferStartLocation);
+        var end = (int)(location - _bufferStartLocation);
+        for (var i = start; i < end; i++)
         {
             pos += _sourcePosCalculator(_span[i]);
         }
 
         return pos;
     }
 
-    private SourcePosDelta ComputeSourcePosAt_CharDefault(int location)
+    private SourcePosDelta ComputeSourcePosAt_CharDefault(long location)
     {
+        // expect start and end to be small enough to fit in an int since we're subtracting
+        var start = (int)(_lastSourcePosDeltaLocation - _bufferStartLocation);
+        var end = (int)(location - _lastSourcePosDeltaLocation);
+
         // coerce _span to Span<char>
         var input = MemoryMarshal.CreateSpan(
             ref Unsafe.As<TToken, char>(ref MemoryMarshal.GetReference(_span)),
             _span.Length
-        ).Slice(_lastSourcePosDeltaLocation - _bufferStartLocation, location - _lastSourcePosDeltaLocation);
+        ).Slice(start, end);
 
         var lines = 0;
         var cols = 0;

Pidgin/ParseState.cs

@@ -34,16 +34,16 @@ public ref partial struct ParseState<TToken>
     private readonly ITokenStream<TToken>? _stream;
     private readonly int _bufferChunkSize;
 
-    private TToken[]? _buffer;
+    private TToken[]? _buffer;  // to return to the pool
     private ReadOnlySpan<TToken> _span;
-    private int _keepFromLocation;  // leftmost bookmark which hasn't been discarded
-    private int _bufferStartLocation;  // how many tokens had been consumed up to the start of the buffer?
-    private int _currentIndex;
+    private long _keepFromLocation;  // leftmost bookmark which hasn't been discarded
+    private long _bufferStartLocation;  // how many tokens had been consumed up to the start of the buffer?
+    private int _currentIndex;  // index into the _span
     private int _bufferedCount;
 
     private int _numberOfBookmarks;
 
-    private int _lastSourcePosDeltaLocation;
+    private long _lastSourcePosDeltaLocation;
     private SourcePosDelta _lastSourcePosDelta;
 
     internal ParseState(IConfiguration<TToken> configuration, ReadOnlySpan<TToken> span)
@@ -104,7 +104,7 @@ internal ParseState(IConfiguration<TToken> configuration, ITokenStream<TToken> s
     /// Returns the total number of tokens which have been consumed.
     /// In other words, the current absolute offset of the input stream.
     /// </summary>
-    public int Location
+    public long Location
     {
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         get
@@ -190,28 +190,29 @@ private void Buffer(int readAhead)
         if (readAheadTo >= _bufferedCount && _stream != null)
         {
             // we're about to read past the end of the current chunk. Pull a new chunk from the stream
-            var keepSeenLength = _keepFromLocation >= 0
-                ? Location - _keepFromLocation
-                : 0;
-            var keepFrom = _currentIndex - keepSeenLength;
-            var keepLength = _bufferedCount - keepFrom;
-            var amountToRead = Math.Max(_bufferChunkSize, readAheadTo - _bufferedCount);
-            var newBufferLength = keepLength + amountToRead;
 
             /*                  _currentIndex
             *                        |
             *                        | _bufferedCount
             *              keepFrom  |      |
             *                 |      |      | readAheadTo
             *                 |      |      |    |
-            *              abcdefghijklmnopqrstuvwxyz
+            * input:       abcdefghijklmnopqrstuvwxyz
             *       readAhead        |-----------|
             *  keepSeenLength |------|
             *      keepLength |-------------|
             *    amountToRead               |----|
             * newBufferLength |------------------|
             */
 
+            // expect keepSeenLength to be small enough to fit in an int since we're subtracting
+            var keepSeenLength = _keepFromLocation >= 0
+                ? (int)(Location - _keepFromLocation)
+                : 0;
+            var keepFrom = _currentIndex - keepSeenLength;
+            var keepLength = _bufferedCount - keepFrom;
+            var amountToRead = Math.Max(_bufferChunkSize, readAheadTo - _bufferedCount);
+            var newBufferLength = keepLength + amountToRead;
             UpdateLastSourcePosDelta();
 
             if (newBufferLength > _buffer!.Length)
@@ -243,7 +244,7 @@ private void Buffer(int readAhead)
 
     /// <summary>Start buffering the input.</summary>
     /// <returns>The location of the bookmark.</returns>
-    public int Bookmark()
+    public long Bookmark()
     {
         if (_keepFromLocation < 0)
         {
@@ -257,7 +258,7 @@ public int Bookmark()
 
     /// <summary>Stop buffering the input.</summary>
     /// <param name="bookmark">The location of the bookmark.</param>
-    public void DiscardBookmark(int bookmark)
+    public void DiscardBookmark(long bookmark)
     {
         if (bookmark < _keepFromLocation || bookmark > Location || _numberOfBookmarks <= 0)
         {
@@ -274,9 +275,10 @@ public void DiscardBookmark(int bookmark)
 
     /// <summary>Return to a bookmark previously obtained from <see cref="Bookmark"/> and discard it.</summary>
     /// <param name="bookmark">The location of the bookmark.</param>
-    public void Rewind(int bookmark)
+    public void Rewind(long bookmark)
     {
-        var delta = Location - bookmark;
+        // expect delta to be small enough to fit in an int since we're subtracting
+        var delta = (int)(Location - bookmark);
 
         if (delta > _currentIndex)
         {

Pidgin/Parser.CurrentOffset.cs

@@ -5,13 +5,19 @@ public static partial class Parser<TToken>
     /// <summary>
     /// A parser which returns the number of input tokens which have been consumed.
     /// </summary>
-    public static Parser<TToken, int> CurrentOffset { get; }
+    public static Parser<TToken, long> CurrentOffsetLong { get; }
         = new CurrentOffsetParser<TToken>();
+
+    /// <summary>
+    /// A parser which returns the number of input tokens which have been consumed.
+    /// </summary>
+    public static Parser<TToken, int> CurrentOffset { get; }
+        = CurrentOffsetLong.Select(x => (int)x);
 }
 
-internal sealed class CurrentOffsetParser<TToken> : Parser<TToken, int>
+internal sealed class CurrentOffsetParser<TToken> : Parser<TToken, long>
 {
-    public sealed override bool TryParse(ref ParseState<TToken> state, ref PooledList<Expected<TToken>> expecteds, out int result)
+    public sealed override bool TryParse(ref ParseState<TToken> state, ref PooledList<Expected<TToken>> expecteds, out long result)
     {
         result = state.Location;
         return true;