Document linemaps in the grimoire

grimoire.scrbl

@@ -18,3 +18,4 @@ programmatically on anything found here.
 @include-section[(lib "resyntax/grimoire/syntax-property-bundle.scrbl")]
 @include-section[(lib "resyntax/grimoire/expansion-analyzers.scrbl")]
 @include-section[(lib "resyntax/grimoire/string-replacement.scrbl")]
+@include-section[(lib "resyntax/grimoire/linemap.scrbl")]

private/linemap.rkt → grimoire/linemap.rkt

@@ -8,13 +8,11 @@
  (contract-out
   [string-linemap (-> string? linemap?)]
   [linemap? (-> any/c boolean?)]
-  [linemap-lines (-> linemap? (vectorof (and/c string? immutable?) #:immutable #true #:flat? #true))]
-  [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)]
-  [linemap-line-start-position (-> linemap? exact-positive-integer? exact-positive-integer?)]
-  [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)]
-  [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)]
-  [syntax-start-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)]
-  [syntax-end-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)]
+  [linemap-position-to-line (-> linemap? exact-nonnegative-integer? exact-positive-integer?)]
+  [linemap-position-to-start-of-line
+   (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)]
+  [linemap-position-to-end-of-line
+   (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)]
   [syntax-line-range (-> syntax? #:linemap linemap? range?)]))
 
 
@@ -45,15 +43,15 @@
   (let loop ([line-number 1] [line-start-index 0] [index 0])
     (cond
       [(= index char-count)
-       (define last-line (substring str line-start-index index))
+       (define last-line (string->immutable-string (substring str line-start-index index)))
        (vector-builder-add lines last-line)
-       (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number)
-       (vector-builder-add start-positions-by-line-number (add1 line-start-index))]
+       (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number)
+       (vector-builder-add start-positions-by-line-number line-start-index)]
       [(equal? (string-ref str index) #\newline)
-       (define next-line (substring str line-start-index index))
+       (define next-line (string->immutable-string (substring str line-start-index index)))
        (vector-builder-add lines next-line)
-       (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number)
-       (vector-builder-add start-positions-by-line-number (add1 line-start-index))
+       (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number)
+       (vector-builder-add start-positions-by-line-number line-start-index)
        (define next-index (add1 index))
        (loop (add1 line-number) next-index next-index)]
       [else (loop line-number line-start-index (add1 index))]))
@@ -88,17 +86,11 @@
   (linemap-line-end-position map (linemap-position-to-line map position)))
 
 
-(define (syntax-start-line-position stx #:linemap map)
-  (linemap-position-to-start-of-line map (syntax-position stx)))
-
-
-(define (syntax-end-line-position stx #:linemap map)
-  (linemap-position-to-end-of-line map (+ (syntax-position stx) (syntax-span stx))))
-
-
 (define (syntax-line-range stx #:linemap map)
   (define first-line (syntax-line stx))
-  (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx))))
+  ;; Syntax object positions are one-indexed, unlike linemap positions.
+  (define last-line
+    (linemap-position-to-line map (+ (sub1 (syntax-position stx)) (syntax-span stx))))
   (unless (<= first-line last-line)
     (raise-arguments-error 'syntax-line-range
                            "syntax object's last line number is before its first line number"
@@ -117,118 +109,119 @@
     (define (nat-map . args)
       (apply sorted-map #:key-comparator natural<=> args))
 
-    (check-equal? (string-linemap "") (linemap #("") (nat-map 1 1) #(1)))
-    (check-equal? (string-linemap "a") (linemap #("a") (nat-map 1 1) #(1)))
-    (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 1 1) #(1)))
-    (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 1 1 3 2) #(1 3)))
-    (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 1 1 3 2) #(1 3)))
-    (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 1 1 5 2) #(1 5)))
-    (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 1 1 5 2) #(1 5)))
+    (check-equal? (string-linemap "") (linemap #("") (nat-map 0 1) #(0)))
+    (check-equal? (string-linemap "a") (linemap #("a") (nat-map 0 1) #(0)))
+    (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 0 1) #(0)))
+    (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 0 1 2 2) #(0 2)))
+    (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 0 1 2 2) #(0 2)))
+    (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 0 1 4 2) #(0 4)))
+    (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 0 1 4 2) #(0 4)))
     (check-equal? (string-linemap "aaa\nbbb\n")
-                  (linemap #("aaa" "bbb" "") (nat-map 1 1 5 2 9 3) #(1 5 9)))
+                  (linemap #("aaa" "bbb" "") (nat-map 0 1 4 2 8 3) #(0 4 8)))
     (check-equal? (string-linemap "a\n\n\nb")
-                  (linemap #("a" "" "" "b") (nat-map 1 1 3 2 4 3 5 4) #(1 3 4 5))))
+                  (linemap #("a" "" "" "b") (nat-map 0 1 2 2 3 3 4 4) #(0 2 3 4))))
 
   (test-case (name-string linemap-position-to-line)
 
     (test-case "two-line string with ending newline"
       (define map (string-linemap "hello\nworld\n"))
+      (check-equal? (linemap-position-to-line map 0) 1)
       (check-equal? (linemap-position-to-line map 1) 1)
       (check-equal? (linemap-position-to-line map 2) 1)
       (check-equal? (linemap-position-to-line map 3) 1)
       (check-equal? (linemap-position-to-line map 4) 1)
       (check-equal? (linemap-position-to-line map 5) 1)
-      (check-equal? (linemap-position-to-line map 6) 1)
+      (check-equal? (linemap-position-to-line map 6) 2)
       (check-equal? (linemap-position-to-line map 7) 2)
       (check-equal? (linemap-position-to-line map 8) 2)
       (check-equal? (linemap-position-to-line map 9) 2)
       (check-equal? (linemap-position-to-line map 10) 2)
       (check-equal? (linemap-position-to-line map 11) 2)
-      (check-equal? (linemap-position-to-line map 12) 2)
-      (check-equal? (linemap-position-to-line map 13) 3))
+      (check-equal? (linemap-position-to-line map 12) 3))
 
     (test-case "multiple blank lines"
       (define map (string-linemap "a\n\nb"))
+      (check-equal? (linemap-position-to-line map 0) 1)
       (check-equal? (linemap-position-to-line map 1) 1)
-      (check-equal? (linemap-position-to-line map 2) 1)
-      (check-equal? (linemap-position-to-line map 3) 2)
-      (check-equal? (linemap-position-to-line map 4) 3)
-      (check-equal? (linemap-position-to-line map 5) 3)))
+      (check-equal? (linemap-position-to-line map 2) 2)
+      (check-equal? (linemap-position-to-line map 3) 3)
+      (check-equal? (linemap-position-to-line map 4) 3)))
 
   (test-case (name-string linemap-line-start-position)
 
     (test-case "two-line string with ending newline"
       (define map (string-linemap "hello\nworld\n"))
-      (check-equal? (linemap-line-start-position map 1) 1)
-      (check-equal? (linemap-line-start-position map 2) 7)
-      (check-equal? (linemap-line-start-position map 3) 13))
+      (check-equal? (linemap-line-start-position map 1) 0)
+      (check-equal? (linemap-line-start-position map 2) 6)
+      (check-equal? (linemap-line-start-position map 3) 12))
 
     (test-case "multiple blank lines"
       (define map (string-linemap "a\n\nb"))
-      (check-equal? (linemap-line-start-position map 1) 1)
-      (check-equal? (linemap-line-start-position map 2) 3)
-      (check-equal? (linemap-line-start-position map 3) 4)))
+      (check-equal? (linemap-line-start-position map 1) 0)
+      (check-equal? (linemap-line-start-position map 2) 2)
+      (check-equal? (linemap-line-start-position map 3) 3)))
 
   (test-case (name-string linemap-line-end-position)
 
     (test-case "two-line string with ending newline"
       (define map (string-linemap "hello\nworld\n"))
-      (check-equal? (linemap-line-end-position map 1) 6)
-      (check-equal? (linemap-line-end-position map 2) 12))
+      (check-equal? (linemap-line-end-position map 1) 5)
+      (check-equal? (linemap-line-end-position map 2) 11))
 
     (test-case "multiple blank lines"
       (define map (string-linemap "a\n\nb"))
-      (check-equal? (linemap-line-end-position map 1) 2)
-      (check-equal? (linemap-line-end-position map 2) 3)
-      (check-equal? (linemap-line-end-position map 3) 5)))
+      (check-equal? (linemap-line-end-position map 1) 1)
+      (check-equal? (linemap-line-end-position map 2) 2)
+      (check-equal? (linemap-line-end-position map 3) 4)))
 
   (test-case (name-string linemap-position-to-start-of-line)
 
     (test-case "two-line string with ending newline"
       (define map (string-linemap "hello\nworld\n"))
-      (check-equal? (linemap-position-to-start-of-line map 1) 1)
-      (check-equal? (linemap-position-to-start-of-line map 2) 1)
-      (check-equal? (linemap-position-to-start-of-line map 3) 1)
-      (check-equal? (linemap-position-to-start-of-line map 4) 1)
-      (check-equal? (linemap-position-to-start-of-line map 5) 1)
-      (check-equal? (linemap-position-to-start-of-line map 6) 1)
-      (check-equal? (linemap-position-to-start-of-line map 7) 7)
-      (check-equal? (linemap-position-to-start-of-line map 8) 7)
-      (check-equal? (linemap-position-to-start-of-line map 9) 7)
-      (check-equal? (linemap-position-to-start-of-line map 10) 7)
-      (check-equal? (linemap-position-to-start-of-line map 11) 7)
-      (check-equal? (linemap-position-to-start-of-line map 12) 7)
-      (check-equal? (linemap-position-to-start-of-line map 13) 13))
+      (check-equal? (linemap-position-to-start-of-line map 0) 0)
+      (check-equal? (linemap-position-to-start-of-line map 1) 0)
+      (check-equal? (linemap-position-to-start-of-line map 2) 0)
+      (check-equal? (linemap-position-to-start-of-line map 3) 0)
+      (check-equal? (linemap-position-to-start-of-line map 4) 0)
+      (check-equal? (linemap-position-to-start-of-line map 5) 0)
+      (check-equal? (linemap-position-to-start-of-line map 6) 6)
+      (check-equal? (linemap-position-to-start-of-line map 7) 6)
+      (check-equal? (linemap-position-to-start-of-line map 8) 6)
+      (check-equal? (linemap-position-to-start-of-line map 9) 6)
+      (check-equal? (linemap-position-to-start-of-line map 10) 6)
+      (check-equal? (linemap-position-to-start-of-line map 11) 6)
+      (check-equal? (linemap-position-to-start-of-line map 12) 12))
 
     (test-case "multiple blank lines"
       (define map (string-linemap "a\n\nb"))
-      (check-equal? (linemap-position-to-start-of-line map 1) 1)
-      (check-equal? (linemap-position-to-start-of-line map 2) 1)
+      (check-equal? (linemap-position-to-start-of-line map 0) 0)
+      (check-equal? (linemap-position-to-start-of-line map 1) 0)
+      (check-equal? (linemap-position-to-start-of-line map 2) 2)
       (check-equal? (linemap-position-to-start-of-line map 3) 3)
-      (check-equal? (linemap-position-to-start-of-line map 4) 4)
-      (check-equal? (linemap-position-to-start-of-line map 5) 4)))
+      (check-equal? (linemap-position-to-start-of-line map 4) 3)))
 
   (test-case (name-string linemap-position-to-end-of-line)
 
     (test-case "two-line string with ending newline"
       (define map (string-linemap "hello\nworld\n"))
-      (check-equal? (linemap-position-to-end-of-line map 1) 6)
-      (check-equal? (linemap-position-to-end-of-line map 2) 6)
-      (check-equal? (linemap-position-to-end-of-line map 3) 6)
-      (check-equal? (linemap-position-to-end-of-line map 4) 6)
-      (check-equal? (linemap-position-to-end-of-line map 5) 6)
-      (check-equal? (linemap-position-to-end-of-line map 6) 6)
-      (check-equal? (linemap-position-to-end-of-line map 7) 12)
-      (check-equal? (linemap-position-to-end-of-line map 8) 12)
-      (check-equal? (linemap-position-to-end-of-line map 9) 12)
-      (check-equal? (linemap-position-to-end-of-line map 10) 12)
-      (check-equal? (linemap-position-to-end-of-line map 11) 12)
+      (check-equal? (linemap-position-to-end-of-line map 0) 5)
+      (check-equal? (linemap-position-to-end-of-line map 1) 5)
+      (check-equal? (linemap-position-to-end-of-line map 2) 5)
+      (check-equal? (linemap-position-to-end-of-line map 3) 5)
+      (check-equal? (linemap-position-to-end-of-line map 4) 5)
+      (check-equal? (linemap-position-to-end-of-line map 5) 5)
+      (check-equal? (linemap-position-to-end-of-line map 6) 11)
+      (check-equal? (linemap-position-to-end-of-line map 7) 11)
+      (check-equal? (linemap-position-to-end-of-line map 8) 11)
+      (check-equal? (linemap-position-to-end-of-line map 9) 11)
+      (check-equal? (linemap-position-to-end-of-line map 10) 11)
+      (check-equal? (linemap-position-to-end-of-line map 11) 11)
       (check-equal? (linemap-position-to-end-of-line map 12) 12))
 
     (test-case "multiple blank lines"
       (define map (string-linemap "a\n\nb"))
-      (check-equal? (linemap-position-to-end-of-line map 1) 2)
+      (check-equal? (linemap-position-to-end-of-line map 0) 1)
+      (check-equal? (linemap-position-to-end-of-line map 1) 1)
       (check-equal? (linemap-position-to-end-of-line map 2) 2)
-      (check-equal? (linemap-position-to-end-of-line map 3) 3)
-      (check-equal? (linemap-position-to-end-of-line map 4) 5)
-      (check-equal? (linemap-position-to-end-of-line map 5) 5))))
+      (check-equal? (linemap-position-to-end-of-line map 3) 4)
+      (check-equal? (linemap-position-to-end-of-line map 4) 4))))

grimoire/linemap.scrbl

@@ -0,0 +1,89 @@
+#lang scribble/manual
+
+
+@(require (for-label racket/base
+                     racket/contract/base
+                     rebellion/base/comparator
+                     rebellion/base/range
+                     resyntax/grimoire/linemap
+                     resyntax/grimoire/source))
+
+
+@title[#:tag "linemap"]{Linemaps}
+@defmodule[resyntax/grimoire/linemap]
+
+A @deftech{linemap} is a precomputed index of a string's line structure that supports converting
+between character positions and line numbers. Source code is frequently viewed from one of two
+perspectives:
+
+@itemlist[
+ @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of
+  lines and columns.}
+
+ @item{A @emph{machine} perspective, that looks at code as one linear sequence of characters (or
+  perhaps even just plain bytes).}]
+
+Tools that serve as a human-machine interface for code often have to juggle these two perspectives.
+Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns
+related to viewing and describing source code and source edits come up, such as displaying or
+consuming line-based diffs in the command-line interface. See @secref["cli"] for further details on
+that matter.
+
+@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions
+are character indices into the string. This follows the same convention as Racket's string operations
+such as @racket[string-ref], as well as Resyntax's conventions for @tech{string replacements}. Line
+numbers, however, follow the conventions outlined in
+@secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]: source files begin at line
+@racket[1]. This matches @racket[syntax-line] and the conventions of code editors --- line numbers
+are almost exclusively useful in user interfaces, where one-based numbering is expected.
+
+@bold{Beware that @racket[syntax-position] and file port positions are @emph{one-based}, unlike
+ linemap positions.} The @racket[syntax-line-range] operation performs that conversion itself, but
+positions obtained from syntax objects by other means must be converted before use with a linemap.
+
+The lines of a string are the segments separated by newline characters. The terminating newline is
+not part of a line's contents, but positions of newline characters belong to the lines they
+terminate. A string that ends with a newline has a final empty line after it, and the empty string
+consists of a single empty line. Note that there are multiple distinct byte sequences that Racket
+treats as a newline when reading source code --- for further details on how Resyntax handles these
+cases, see the notes in @racket[with-input-from-source].
+
+
+@defproc[(linemap? [v any/c]) boolean?]{
+ A predicate that recognizes @tech{linemaps}.}
+
+
+@defproc[(string-linemap [str string?]) linemap?]{
+ Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are
+ treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not
+ understood. This never arises in practice, because Resyntax normalizes all newlines to
+ @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for
+ details on that normalization and why it matters.}
+
+
+@defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?])
+         exact-positive-integer?]{
+ Returns the line number of the line containing @racket[position]. The position of a newline
+ character is considered contained by the line that the newline terminates. Positions beyond the
+ end of the string do not raise an error; they are all treated as belonging to the last line.}
+
+
+@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?])
+         exact-nonnegative-integer?]{
+ Returns the position of the first character of the line containing @racket[position]. If the
+ string ends with a newline and @racket[position] is on the final, empty line after it, that
+ line's start position is equal to the length of the string.}
+
+
+@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?])
+         exact-nonnegative-integer?]{
+ Returns the position just past the last character of the contents of the line containing
+ @racket[position] --- that is, the position of the line's terminating newline, or the length of
+ the string if the line is the last one.}
+
+
+@defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{
+ Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers
+ of every line that @racket[stx] spans, from the line on which it begins to the line on which it
+ ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of
+ @racket[stx] must refer to positions within the string that @racket[map] was built from.}