Better stringing.

src/examples/forbble.pbl

@@ -10,7 +10,7 @@
 )))
 
 (def pop (fn ()
-  "Remove the top of the stack"
+  "Remove and print the top of the stack"
   (if (> (len stack) 0) (
     (def top (at 0 stack))
     (set stack (rest stack))
@@ -18,6 +18,13 @@
   ) (prnl "pop: STACK EMPTY"))
 ))
 
+(def drop (fn ()
+  "Remove the top of the stack"
+  (if (> (len stack) 0) (
+    (set stack (rest stack))
+  ) (prnl "pop: STACK EMPTY"))
+))
+
 (def twop (fn (op)
   "Apply the given operation to the top two stack elements" (
   (if (< (len stack) 2) (prnl "stack too small!") (
@@ -92,6 +99,7 @@
   (h-insert operations name op)
 )))
 (add-op "cls" '(sys "clear"))
+(add-op "drop" '(drop))
 (add-op "over" '(over))
 (add-op "rot" '(rot))
 (add-op "dup" '(dup))

src/object.c

@@ -739,22 +739,20 @@ inline Object nullTerminated(const char* string)
 inline Object stringFromSlice(const char* string, int len)
 {
 #ifdef STANDALONE
+    //eprintf("ay: %s\n", string);
     Object o = withLen(len, TYPE_STRING);
     int c = 0;
     for (int i = 0; i < len; i++) {
         if (string[i] == '\\') {
-            switch (string[i + 1]) {
+            i++;
-                case '"':
+            switch (string[i]) {
-                    o.string[c] = '"';
+                case 'n':
-                    break;
+                    o.string[c] = '\n';
-                case '\\':
-                    o.string[c] = '\\';
                     break;
                 default:
-                    o.string[c] = string[i + 1];
+                    o.string[c] = string[i];
                     break;
             }
-            i++;
         } else {
             o.string[c] = string[i];
         }

src/tests.sh

@@ -49,7 +49,7 @@ pass() {
 
 fail() {
     echo -n "X"
-    FAIL_OUTPUT="$FAIL_OUTPUT  $1 FAILED\n    When running \"$2\""
+    FAIL_OUTPUT="$FAIL_OUTPUT  $1 FAILED\n    When running $2"
     ((FAILS++))
     ((TOTAL_FAILS++))
 }
@@ -115,6 +115,9 @@ check "String Inequality" '(= "Beans" "Bean" )' "F"
 check "Null Inequality" '(= "Beans" "" )' "F"
 
 title "Complex Strings"
+check "Escaping Quotes" '"\"Hello\""' '"Hello"'
+check "Escaping Backslash" '"\\Hello\\"' '\Hello\'
+check "Escaping Many" '"\\\"\\\\\\Hello\\\"\\\\\\"' '\"\\\Hello\"\\\'
 check "Triple Quoting" '"""Hello"""' 'Hello'
 check "Triple Quoting With Quotes" '"""My name is "yoink"."""' 'My name is "yoink".'
 

src/tokens.c

@@ -43,6 +43,11 @@ void buildErrFromInput(struct Error* err, enum errorCode code, int i, const char
     strncpy(err->context, &input[start], ERR_LEN);
 }
 
+void collectSymbol(const char* input, int* i, int* length);
+
+void processString(const char* input, struct Error* err, struct Slice* slices, int slice, int* i, int* lineNumber,
+                   int* length);
+
 struct Slice* nf_tokenize(const char* input, struct Error* err)
 {
     if (!input) {
@@ -94,39 +99,13 @@ struct Slice* nf_tokenize(const char* input, struct Error* err)
             }
             continue;
         } else if (input[i] == '"') {
-            if (input[i + 1] == '"' && input[i + 2] == '"') {
+            processString(input, err, slices, slice, &i, &lineNumber, &length);
-                // Triple-quoted block
-                i += 2;
-                slices[slice].text += 2;
-                for (;;) {
-                    i++;
-                    if (input[i] == '"' && input[i + 1] == '"' && input[i + 2] == '"') {
-                        break;
-                    }
-                    if (input[i] == '\n') {
-                        lineNumber++;
-                    }
-                    length++;
-                    if (input[i] == '\0' || input[i + 1] == '\0' || input[i + 2] == '\0') {
-                        buildErrFromInput(err, UNEXPECTED_EOF, i, input);
-                        //free(slices);
-                        return NULL;
-                    }
-                }
-            } else {
-                // Simple string
-                while ((input[++i] != '"' || input[i - 1] == '\\') && input[i] != '\0') {
-                    if (input[i] == '\n') {
-                        lineNumber++;
-                    }
-                    length++;
-                }
-            }
-            i++;
         } else {
-            while (!isWhitespace(input[++i]) && !isSingle(input[i]) && input[i] != '"' && input[i] != '\0') {
+            collectSymbol(input, &i, &length);
-                length++;
+        }
-            }
+
+        if (err->context != NULL) {
+            return NULL;
         }
 
         slices[slice].length = length;
@@ -147,3 +126,69 @@ struct Slice* nf_tokenize(const char* input, struct Error* err)
 
     return allocated;
 }
+
+void singleQuotedString(const char* input, int* i, int* lineNumber, int* length);
+
+void tripleQuotedString(const char* input, struct Error* err, struct Slice* slice, int* i, int* lineNumber,
+                        int* length);
+
+void processString(const char* input, struct Error* err, struct Slice* slices, int slice, int* i, int* lineNumber,
+                   int* length)
+{
+    if (input[(*i) + 1] == '"' && input[(*i) + 2] == '"') {
+        tripleQuotedString(input, err, &slices[slice], i, lineNumber, length);
+    } else {
+        singleQuotedString(input, i, lineNumber, length);
+    }
+    (*i)++;
+}
+
+void collectSymbol(const char* input, int* i, int* length)
+{
+    while (!isWhitespace(input[++(*i)]) && !isSingle(input[(*i)]) && input[(*i)] != '"' && input[(*i)] != '\0') {
+        (*length)++;
+    }
+}
+
+void tripleQuotedString(const char* input, struct Error* err, struct Slice* slice, int* i, int* lineNumber, int* length)
+{
+    // Skip past the extra opening quotes
+    (*i) += 2;
+    slice->text += 2;
+
+    for (;;) {
+        (*i)++;
+        const int c = *i;
+        if (input[c] == '"' && input[c + 1] == '"' && input[c + 2] == '"') {
+            break;
+        }
+        if (input[c] == '\n') {
+            (*lineNumber)++;
+        }
+        (*length)++;
+        if (input[c] == '\0' || input[c + 1] == '\0' || input[c + 2] == '\0') {
+            buildErrFromInput(err, UNEXPECTED_EOF, c, input);
+            return;
+        }
+    }
+}
+
+void singleQuotedString(const char* input, int* i, int* lineNumber, int* length)
+{
+    while (input[++(*i)] != '\0') {
+        if (input[(*i)] == '"') {
+            int backslashes = 0;
+            while (input[((*i) - 1) - backslashes] == '\\') {
+                backslashes++;
+            }
+            // \\\" => Odd number of backslashes, quote IS escaped
+            // \\" => Even number of backslashes, quote is NOT escaped
+            if (backslashes % 2 == 0) {
+                break;
+            }
+        } else if (input[(*i)] == '\n') {
+            (*lineNumber)++;
+        }
+        (*length)++;
+    }
+}