Updated documentation, added tree-sitter experiment

experiments/package.json

@@ -0,0 +1,6 @@
+{
+  "devDependencies": {
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-javascript": "^0.25.0"
+  }
+}

experiments/tree-sitter-1.mjs

@@ -0,0 +1,52 @@
+import Parser from 'tree-sitter';
+import JavaScript from 'tree-sitter-javascript';
+import { readFileSync } from 'node:fs';
+import { inspect } from 'node:util';
+
+// NOTE: 	once upon a time
+//			there was some sort of
+//
+//			example comment that we wanted to investigate
+
+const parser = new Parser();
+parser.setLanguage(JavaScript);
+
+class example {
+
+	#private = 123
+
+	/*
+		This other comment
+		is of the style of a
+		block comment of course
+	*/
+
+	stuff() {
+
+	}
+
+}
+
+const source = readFileSync('./tree-sitter-1.mjs', 'utf-8');
+const tree = parser.parse(source);
+
+function* iter_nodes(node) {
+    yield node;
+    for (let i = 0; i < node.childCount; i++) {
+        yield* iter_nodes(node.child(i));
+    }
+}
+
+
+function *iter_child_nodes(node) {
+    for (let i = 0; i < node.childCount; i++) {
+        yield node.child(i);
+    }
+}
+
+
+for (const node of iter_child_nodes(tree.rootNode)) {
+    //console.log({type: node.type, text: node.text?.slice(0, 40)});
+    console.log(inspect(node));
+    console.log({text: node.text});
+}

planning/dsl.md

@@ -0,0 +1,25 @@
+
+## Example of indented block
+
+```
+§ block
+	this content
+	is in the block body
+```
+
+
+## Inline
+
+`«inline expression»`
+
+## Escaping
+
+
+```
+«§» block
+
+	Here is an «la»inline expression«ra»
+	Or possibly «outer escape: inline expression»
+
+```
+

planning/to-document.md

@@ -0,0 +1,19 @@
+# Readme.md
+
+In `readme.md` we should explain that `npm install` can be used both with and without development dependencies.
+
+We are likely to want some development dependencies for the following purposes:
+
+- Template processing
+- Ecmascript parsing for inline rich semantic documentation
+
+
+## Installing tree-sitter-javascript
+
+- requires `node-gyp`
+
+```sh
+CXXFLAGS="-std=c++20" npm i -D tree-sitter tree-sitter-javascript
+```
+
+This section should be somewhat expanded - especially regarding node-gyp

source/errors.mjs

@@ -1,5 +1,17 @@
 import { inspect } from 'node:util';
 
+// § GROUP: Regexp tokenization
+
+
+
+export class Tokenization_Error extends Error {
+	constructor(data) {
+		const { parser, value, index, end_index } = data;
+		super(`Tokenization_Error`);	//TODO: Format message
+		this.data = data;
+	}
+}
+
 // § GROUP: Configuration field errors
 
 export class Data_Validation_Failed extends Error {

source/parsing/regexp-dispatch.mjs

@@ -1,4 +1,6 @@
 import * as RE from '@efforting.tech/text/regexp';
+import { Tokenization_Error } from '@efforting.tech/errors';
+
 
 // NOTE: 	There are some open questions about this implementation and API which may change as the library matures.
 //			Check out the example at experiments/regexp-tokenizer.mjs for more information on how to use this in its current state.
@@ -104,7 +106,7 @@ export class RegExp_Tokenizer {
 	_handle_default_match(value, index, end_index=null) {
 		const { default_action } = this;
 		if (!default_action) {
-			throw new Parsing_Error({ parser: this, value, index, end_index });
+			throw new Tokenization_Error({ parser: this, value, index, end_index });
 		}
 		return new Default_Match(value, index, end_index, default_action);
 	}