WIP - reduction system

Makefile

@@ -2,12 +2,15 @@
 
 build/packages:
 	mkdir -p $@
-	node tools/stage-for-pnpm.mjs package-manifest.yaml source $@
+	node tools/stage-for-npm.mjs package-manifest.yaml source $@
 
 publish:
 	cd build/packages && ./publish-all.sh
 
+dev:
+	build/packages/local-install.sh
+
 clean:
 	rm -rf build
 
-.PHONY: clean build/packages publish
+.PHONY: clean build/packages publish dev

experiments/config1.mjs

@@ -1,4 +1,4 @@
-import { Schema, Field_Configuration  } from '@efforting.tech/data/field-configuration';
+import { Schema, Field_Configuration  } from '@efforting.tech/schema/field-configuration';
 
 
 function mandatory_anything(value) {

experiments/config2.mjs

@@ -1,4 +1,4 @@
-import * as F from '@efforting.tech/data/field-configuration-factories';
+import * as F from '@efforting.tech/schema/field-configuration-factories';
 
 const s = new F.Schema({
 	foo: F.value(123, 'The value'),

experiments/feature-stub.mjs

@@ -0,0 +1,30 @@
+
+import { Stub } from '@efforting.tech/feature/stub';
+/*
+
+export function Stub(meta, name, description, module_name, function_name) {
+	return function stub() {
+		throw new Error(`The feature "${name}" of "${meta.url}" is not enabled. Enable it by calling "${function_name}(${this.name})" imported from "${module_name}"`);	//TODO - specific error
+	}
+}
+
+*/
+
+class Thing {
+
+	static from_stuff = Stub(import.meta, 'stuff-loader', 'Creates Thing from stuff', '@efforting.tech/stuff/loader', 'enable_stuff_loader');
+
+}
+
+Thing.from_stuff()
+
+/*
+
+Error: The feature "stuff-loader" of "file:///srv/Projekt/efforting.tech/nodejs.esm-library/experiments/generic-parser-2.mjs" is not enabled. Enable it by calling "enable_stuff_loader(Thing)" imported from "@efforting.tech/stuff/loader"
+    at Thing.stub [as from_stuff] (file:///srv/Projekt/efforting.tech/nodejs.esm-library/build/packages/feature/stub.mjs:4:9)
+    at file:///srv/Projekt/efforting.tech/nodejs.esm-library/experiments/generic-parser-2.mjs:10:7
+    at ModuleJob.run (node:internal/modules/esm/module_job:430:25)
+    at async node:internal/modules/esm/loader:639:26
+    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)
+
+*/

experiments/generic-parser-1.mjs

@@ -0,0 +1,40 @@
+import { RegExp_Tokenizer } from '@efforting.tech/parsing/regexp-dispatch';
+import { RegExp_Token_Parsing_Rule, Parser } from '@efforting.tech/parsing/generic-parsing';
+
+import { inspect } from 'node:util';
+
+const text = 'Hello World (how are you (doing)) I may ask';
+
+const rt = new RegExp_Tokenizer();
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\w+/, (tokenizer, match) => tokenizer.push_token(match.value), 'word'));
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\s+/, null, 'space'));
+rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape('('), (tokenizer, match) => tokenizer.enter_sub_tokenizer(undefined, (tokenizer, value) => tokenizer.push_token({kind: 'sub expression', value}) ), 'lpar'));
+rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(')'), (tokenizer, match) => tokenizer.leave_sub_tokenizer(), 'rpar'));
+
+const p = new Parser(text, { tokenizer: rt });
+
+console.log(inspect(p.parse((tokenizer, value) => tokenizer.replace_value({kind: 'parsing result', value})), { colors: true, depth: null }));
+
+/*
+
+{
+  kind: 'parsing result',
+  value: [
+    'Hello',
+    'World',
+    {
+      kind: 'sub expression',
+      value: [
+        'how',
+        'are',
+        'you',
+        { kind: 'sub expression', value: [ 'doing' ] }
+      ]
+    },
+    'I',
+    'may',
+    'ask'
+  ]
+}
+
+*/

experiments/generic-parser-2.mjs

@@ -0,0 +1,146 @@
+import { Row_Based_Table } from '@efforting.tech/table';
+import { load_raster_table } from '@efforting.tech/table/raster-table';
+import { RegExp_Tokenizer } from '@efforting.tech/parsing/regexp-dispatch';
+import { RegExp_Token_Parsing_Rule, Parser } from '@efforting.tech/parsing/generic-parsing';
+
+
+function load_table(raster) {
+	const table = load_raster_table(raster, Row_Based_Table);
+	table.replace_all_cells(({cell}) => cell.trim());
+	return table;
+}
+
+const logic_ops = load_table(`
+
+	name		symbol
+	----		------
+	AND			∧
+	OR			∨
+	XOR			⊕
+	NAND		↑
+	NOR			↓
+	XNOR		⊙
+	IMPLIES		→
+	IFF			↔
+	NOT			¬
+
+`);
+
+const generic_ops = load_table(`
+
+	name		symbol
+	----		------
+	PLUS		+
+	HYPHEN		-
+	DOT			·
+	ASTERISK	*
+	CROSS		×
+	SLASH		/
+	CARET		^
+	UNDERSCORE	_
+	PERCENT		%
+
+`);
+
+const punctuation = load_table(`
+
+	name		symbol
+	----		------
+	COMMA		,
+	SEMI_COLON	;
+	COLON		:
+	PERIOD		.
+
+`);
+
+const grouping = load_table(`
+
+	name					left	right
+	----					----	-----
+	PARENTESIS				(		)
+	SQUARE_BRACKET			[		]
+	CURLY_BRACE				{		}
+	ANGLE_BRACKET			⟨		⟩
+	DOUBLE_ARROW_BRACKET	«		»
+
+`);
+
+const greek_chars = load_table(`
+
+	name		lower	upper
+	----		-----	-----
+	ALPHA		α		Α
+	BETA		β		Β
+	GAMMA		γ		Γ
+	DELTA		δ		Δ
+	EPSILON		ε		Ε
+	ZETA		ζ		Ζ
+	ETA			η		Η
+	THETA		θ		Θ
+	IOTA		ι		Ι
+	KAPPA		κ		Κ
+	LAMBDA		λ		Λ
+	MU			μ		Μ
+	NU			ν		Ν
+	XI			ξ		Ξ
+	OMICRON		ο		Ο
+	PI			π		Π
+	RHO			ρ		Ρ
+	SIGMA		σ		Σ
+	TAU			τ		Τ
+	UPSILON		υ		Υ
+	PHI			φ		Φ
+	CHI			χ		Χ
+	PSI			ψ		Ψ
+	OMEGA		ω		Ω
+
+`);
+
+
+
+
+
+const rt = new RegExp_Tokenizer();
+
+for (const { name, left, right } of grouping.iter_objects()) {
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(left),
+		(tokenizer, ingress_match) => tokenizer.enter_sub_tokenizer(undefined,
+			(tokenizer, value, egress_match) => tokenizer.push_token(
+				{kind: 'EXPR', name, value, ingress_match, egress_match}
+			)
+		), `LEFT_${name}`
+	));
+
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(right),
+		(tokenizer, match) => tokenizer.leave_sub_tokenizer(match), `RIGHT_${name}`)
+	);
+}
+
+
+for (const table of [logic_ops, generic_ops, punctuation]) {
+	for (const { name, symbol } of table.iter_objects()) {
+		rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(symbol),
+			(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), name)
+		);
+	}
+}
+
+for (const { name, lower, upper } of greek_chars.iter_objects()) {
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(lower),
+		(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), `LOWER_${name}`)
+	);
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(upper),
+		(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), `UPPER_${name}`)
+	);
+}
+
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\w+/, (tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), 'WORD'));
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\s+/, null, 'WHITESPACE'));
+
+
+const text = 'Hello World (how are you (doing)) I may ask';
+const p = new Parser(text, { tokenizer: rt });
+
+//console.log(rt.rules.at(-3));
+
+console.log(p.parse())

experiments/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@efforting.tech/experiments",
+  "version": "0.0.1",
+  "private": true,
+  "devDependencies": {
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-javascript": "^0.25.0"
+  }
+}

experiments/reduction-scanner-1.mjs

@@ -0,0 +1,109 @@
+import { Reduction_Scanner, Reduction_Settings } from  '@efforting.tech/rule-processing/reduction-scanner';
+import * as R from  '@efforting.tech/rule-processing/rules';
+
+import { inspect } from 'node:util';
+
+/*
+Here's what needs addressing:
+
+**`perform_reduction` refactor**
+- Remove the `start_index` loop — scanning is the condition's responsibility
+- RULE_MAJOR: iterate rules, call `rule.match(sequence, context)`, apply first that returns a match
+- POSITION_MAJOR: collect matches from all rules, apply the one with lowest `start_index` in result
+
+**`Sequence_Condition.match` implementation**
+- Iterate positions internally
+- Return match result with `start_index`, `end_index`, captures
+- Return null if no match found anywhere
+
+**Rule interface**
+- `rule.match(sequence, context)` → match result or null
+- `rule.action(scanner, sequence, match)` → performs the transformation
+- Decide: forwarding getters, bind in constructor, or scanner calls `rule.condition.match` directly
+
+**Match result shape**
+- `{ rule, sequence, start_index, end_index, captures, ...extra_info }`
+- `captures` lazily evaluated via getter
+
+**Normalization**
+- Decide when rules get normalized/compiled (construction, first transform, explicit `prepare()`)
+- Normalize bare functions to condition objects at that point
+
+**`context` shape**
+- What does the scanner inject into context beyond `start_index`/`end_index`?
+- How does `extra_info` from resolver flow through to condition match?
+*/
+
+
+
+class Rule {	//NOTE: This is somewhat of a place holder because we may want to declare specific transformations later rather than always having an opaque handler function
+	constructor(condition, handler) {
+		Object.assign(this, { condition, handler });
+	}
+
+	get match() {
+		return this.condition.match.bind(this.condition);
+	}
+
+	get action() {
+		return this.handler;
+	}
+
+}
+
+
+
+
+const N = new R.Predicate((i) => typeof i === 'number' || i.type == 'BINOP' );
+const ASTERISK = new R.Strict_Equality('*');
+const PLUS = new R.Strict_Equality('+');
+
+
+const rss = Reduction_Settings.load({
+	// Switching this on or off affects whether add comes before mul or not
+	//reduction_order: 'POSITION_MAJOR',
+});
+const rs = new Reduction_Scanner(rss);
+
+
+rss.rules.push(
+	new Rule(
+		new R.Sequence_Condition([N, ASTERISK, N]),
+		(rs, sequence, match) => {
+			const MS = match.match_start;
+			const ME = match.match_end;
+			sequence.splice(MS, ME - MS + 1, { type: 'BINOP', op: 'MUL', operands: [sequence[MS], sequence[ME]]});
+		}
+	),
+
+	new Rule(
+		new R.Sequence_Condition([N, PLUS, N]),
+		(rs, sequence, match) => {
+			const MS = match.match_start;
+			const ME = match.match_end;
+			sequence.splice(MS, ME - MS + 1, { type: 'BINOP', op: 'ADD', operands: [sequence[MS], sequence[ME]]});
+		},
+	),
+
+);
+
+const arr = [10, '+', 20, '*', 30];
+console.log(inspect(rs.transform(arr), { colors: true, depth: null }));
+
+/* OUTPUT
+
+[
+  {
+    type: 'BINOP',
+    op: 'ADD',
+    operands: [ 10, { type: 'BINOP', op: 'MUL', operands: [ 20, 30 ] } ]
+  }
+]
+
+
+*/
+
+
+// These are for testing conditions without reduction
+// const sc = new R.Sequence_Condition([N, PLUS, N]);
+// console.log(sc.match([10, '+', 20, '*', 30]));

experiments/reduction-scanner-2.mjs

@@ -0,0 +1,76 @@
+import { Reduction_Scanner, Reduction_Settings } from  '@efforting.tech/rule-processing/reduction-scanner';
+import * as R from  '@efforting.tech/rule-processing/rules';
+
+import { inspect } from 'node:util';
+
+
+
+class Rule {	//NOTE: This is somewhat of a place holder because we may want to declare specific transformations later rather than always having an opaque handler function
+	constructor(condition, handler) {
+		Object.assign(this, { condition, handler });
+	}
+
+	get match() {
+		return this.condition.match.bind(this.condition);
+	}
+
+	get action() {
+		return this.handler;
+	}
+
+}
+
+
+
+function sequence_rule(sequence, transform_fn) {
+	return new Rule(
+		new R.Sequence_Condition(sequence),
+		(rs, sequence, match) => {
+			const MS = match.match_start;
+			const ME = match.match_end;
+			sequence.splice(MS, ME - MS + 1, transform_fn(...sequence.slice(MS, ME + 1)));
+		}
+	);
+}
+
+
+const N = new R.Predicate((i) => typeof i === 'number' || i.type == 'BINOP' );
+const ASTERISK = new R.Strict_Equality('*');
+const PLUS = new R.Strict_Equality('+');
+const HAT = new R.Strict_Equality('^');
+
+
+const rss = Reduction_Settings.load({
+	// Switching this on or off affects whether add comes before mul or not
+	//reduction_order: 'POSITION_MAJOR',
+});
+
+const rs = new Reduction_Scanner(rss);
+
+rss.rules.push(
+
+	sequence_rule([N, HAT, N], 		(left, op, right) => ({ type: 'BINOP', op: 'HAT', operands: [left, right]})),
+	sequence_rule([N, ASTERISK, N], (left, op, right) => ({ type: 'BINOP', op: 'ASTERISK', operands: [left, right]})),
+	sequence_rule([N, PLUS, N], 	(left, op, right) => ({ type: 'BINOP', op: 'PLUS', operands: [left, right]})),
+
+);
+
+const arr = [10, '^', 5, '+', 20, '*', 30];
+console.log(inspect(rs.transform(arr), { colors: true, depth: null }));
+
+/* OUTPUT
+
+
+[
+  {
+    type: 'BINOP',
+    op: 'PLUS',
+    operands: [
+      { type: 'BINOP', op: 'HAT', operands: [ 10, 5 ] },
+      { type: 'BINOP', op: 'ASTERISK', operands: [ 20, 30 ] }
+    ]
+  }
+]
+
+
+*/

experiments/reduction-scanner-3.mjs

@@ -0,0 +1,157 @@
+import { Reduction_Scanner, Reduction_Settings } from  '@efforting.tech/rule-processing/reduction-scanner';
+import * as R from  '@efforting.tech/rule-processing/rules';
+
+import { inspect } from 'node:util';
+
+
+class Rule_Match {
+	constructor(rule, match) {
+		Object.assign(this, { rule, match });
+	}
+
+	get action() {
+		return this.rule.handler;
+	}
+
+
+}
+
+
+class Rule {	//NOTE: This is somewhat of a place holder because we may want to declare specific transformations later rather than always having an opaque handler function
+	constructor(condition, handler) {
+		Object.assign(this, { condition, handler });
+	}
+
+	match(...args) {
+		const match = this.condition.match(...args);
+		if (match) {
+			return new Rule_Match(this, match);
+		}
+	}
+
+
+}
+
+class Sub_Scan_Rule_Match {
+	constructor(rule, sub_scan_candidate) {
+		Object.assign(this, { rule, sub_scan_candidate });
+	}
+
+	get action() {
+		return this.sub_scan_candidate.match.action;
+	}
+
+	get match() {
+		return this.sub_scan_candidate.match.match;
+	}
+
+}
+
+class Sub_Scan_Rule {
+	constructor(sub_system) {
+		Object.assign(this, { sub_system });
+	}
+
+	match(...args) {
+		const candidate = this.sub_system.find_reduction_candidate(...args);
+		if (candidate) {
+			return new Sub_Scan_Rule_Match(this, candidate)
+		}
+	}
+
+
+}
+
+
+
+function sequence_rule(sequence, transform_fn) {
+	return new Rule(
+		new R.Sequence_Condition(sequence),
+		({sequence, match}) => {
+			const MS = match.match_start;
+			const ME = match.match_end;
+			sequence.splice(MS, ME - MS + 1, transform_fn(...sequence.slice(MS, ME + 1)));
+		}
+	);
+}
+
+
+
+const N = new R.Predicate((i) => typeof i === 'number' || i.type == 'BINOP' );
+
+const CARET = new R.Strict_Equality('^');
+const CARON = new R.Strict_Equality('ˇ');
+
+const ASTERISK = new R.Strict_Equality('*');
+const SLASH = new R.Strict_Equality('/');
+
+const PLUS = new R.Strict_Equality('+');
+const MINUS = new R.Strict_Equality('-');
+
+// These are the outer settings
+const rss = Reduction_Settings.load();
+
+// These are the inner settings
+const rss_inner = Reduction_Settings.load({
+	reduction_order: 'POSITION_MAJOR',
+});
+
+const rs = new Reduction_Scanner(rss);
+
+// Local factory for sub system
+function sub_system(...rules) {
+	const sub_settings = { ...rss_inner, rules };
+	const scanner = new Reduction_Scanner(sub_settings);
+	return new Sub_Scan_Rule(scanner);
+}
+
+
+
+
+rss.rules.push(
+	sub_system(
+		sequence_rule([N, CARET, N], 		(left, op, right) => ({ type: 'BINOP', op: 'CARET', operands: [left, right]})),
+		sequence_rule([N, CARON, N], 		(left, op, right) => ({ type: 'BINOP', op: 'CARON', operands: [left, right]})),
+	),
+
+	sub_system(
+		sequence_rule([N, ASTERISK, N], (left, op, right) => ({ type: 'BINOP', op: 'ASTERISK', operands: [left, right]})),
+		sequence_rule([N, SLASH, N], (left, op, right) => ({ type: 'BINOP', op: 'SLASH', operands: [left, right]})),
+	),
+
+	sub_system(
+		sequence_rule([N, PLUS, N], 	(left, op, right) => ({ type: 'BINOP', op: 'PLUS', operands: [left, right]})),
+		sequence_rule([N, MINUS, N], 	(left, op, right) => ({ type: 'BINOP', op: 'MINUS', operands: [left, right]})),
+	),
+);
+
+
+const arr = [5, '-', 10, '^', 5, 'ˇ', 2, '+', 20, '*', 30];
+console.log(inspect(rs.transform(arr), { colors: true, depth: null }));
+
+
+/*
+
+[
+  {
+    type: 'BINOP',
+    op: 'MINUS',
+    operands: [
+      5,
+      {
+        type: 'BINOP',
+        op: 'PLUS',
+        operands: [
+          {
+            type: 'BINOP',
+            op: 'CARON',
+            operands: [ { type: 'BINOP', op: 'CARET', operands: [ 10, 5 ] }, 2 ]
+          },
+          { type: 'BINOP', op: 'ASTERISK', operands: [ 20, 30 ] }
+        ]
+      }
+    ]
+  }
+]
+
+*/

experiments/regexp-tokenizer.mjs

@@ -0,0 +1,15 @@
+import { RegExp_Tokenizer, RegExp_Token_Rule } from '@efforting.tech/parsing/regexp-dispatch';
+
+
+const rt = new RegExp_Tokenizer();
+
+rt.add_rules(new RegExp_Token_Rule(/\w+/, 'word'));
+rt.set_default_identifier('random stuff');
+
+//console.log(rt.rules);
+
+//console.log(rt.closest_scanning_match('#Hello World!'));
+
+for (const m of rt.iter_matches('#Hello World!')) {
+	console.log({class: m.constructor.name, identifier: m.identifier, value: m.value, captured: m.captured });
+};

experiments/res1.mjs

@@ -1,6 +1,6 @@
 import { Mapping_Resolver, Chained_Resolver } from '@efforting.tech/rule-processing/resolvers';
 
-
+import { inspect } from 'node:util';
 
 
 const vr = new Mapping_Resolver();
@@ -9,9 +9,55 @@ const tr = new Mapping_Resolver(new Map(), item => typeof item);
 const cr = new Chained_Resolver([vr, tr]);
 
 
-vr.rules.set('HELLO', () => 'WORLD');
-tr.rules.set('string', () => 'World');
+vr.rules.set('HELLO', {handler: () => 'WORLD'});
+tr.rules.set('string', {extra_stuff: 'Yo!', handler: (c) => `World with context: ${inspect(c)}`});
 
 console.log(cr.resolve('HELLO'));
 console.log(cr.resolve('hello'));
 console.log(cr.resolve(123));
+
+
+/* OUTPUT
+
+WORLD
+World with context: {
+  resolver: Chained_Resolver {
+    chain_links: [ [Mapping_Resolver], [Mapping_Resolver] ]
+  },
+  item: 'hello',
+  extra_stuff: 'Yo!',
+  handler: [Function: handler]
+}
+file:///srv/Projekt/efforting.tech/nodejs.esm-library/build/packages/rule-processing/resolv
+                        throw new Item_Unresolvable({ resolver: this, item });
+                              ^
+
+Item_Unresolvable [Error]: Cannot resolve item 123 of type "number" using resolver Chained_
+    at Chained_Resolver.resolve (file:///srv/Projekt/efforting.tech/nodejs.esm-library/buil
+    at file:///srv/Projekt/efforting.tech/nodejs.esm-library/experiments/res1.mjs:17:16
+    at ModuleJob.run (node:internal/modules/esm/module_job:430:25)
+    at async node:internal/modules/esm/loader:639:26
+    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) {
+  data: {
+    resolver: Chained_Resolver {
+      chain_links: [
+        Mapping_Resolver {
+          rules: Map(1) { 'HELLO' => [Object] },
+          key_function: null
+        },
+        Mapping_Resolver {
+          rules: Map(1) { 'string' => [Object] },
+          key_function: [Function (anonymous)]
+        }
+      ]
+    },
+    item: 123
+  }
+}
+
+Node.js v25.8.2
+
+
+
+
+*/

experiments/table-1.mjs

@@ -0,0 +1,23 @@
+import { Row_Based_Table } from '@efforting.tech/table'
+
+
+const t = new Row_Based_Table({ column_names: 'SKU, Quantity, Price' });
+
+//console.log(t.column_names_lut);	/* { SKU: 0, Quantity: 1, Price: 2 } */
+
+t.push_rows(
+	['VOLVO-1', 3, 120_000],
+	['VOLVO-2', 4, 140_000],
+)
+
+const [A] = t.read_rows(0);
+const [B] = t.snapshot_rows(1);
+
+console.log(A.value);
+console.log(B.value);
+A.update({Quantity: 5});
+console.log(A.value);
+
+for (const r of t) {
+	console.log(r.object);
+}

experiments/table-2.mjs

@@ -0,0 +1,34 @@
+import { Row_Based_Table } from '@efforting.tech/table';
+import { load_raster_table } from '@efforting.tech/table/raster-table';
+
+
+const t = load_raster_table(`
+
+	SKU		Quantity		Price
+	---		--------		-----
+	V-1		2				120_000
+	V-2		3				140_000
+
+`, Row_Based_Table);
+
+
+// { row_name, column_name, row_index, column_index, row, cell }
+t.replace_all_cells(
+	({column_name, cell}) => {
+		const translation = {
+			SKU: (s) => s.trim(),
+			Quantity: parseInt,
+			Price: (p) => parseFloat(p.replace(/_/g, '')),
+		}[column_name];
+		return translation ? translation(cell) : cell;
+	}
+);
+
+for (const r of t) {
+	console.log(r.object);
+}
+
+/*
+{ SKU: 'V-1', Quantity: 2, Price: 120000 }
+{ SKU: 'V-2', Quantity: 3, Price: 140000 }
+*/

experiments/text-nodes-dispatch.mjs

@@ -0,0 +1,39 @@
+import * as CF from '@efforting.tech/data/field-configuration-factories';
+
+import { inspect } from 'node:util';
+import { Text_Tree_Node, Text_Tree_Settings } from  '@efforting.tech/text/basic-tree';
+import { RegExp_Resolver } from  '@efforting.tech/rule-processing/resolvers';
+import { parse_csv } from  '@efforting.tech/data/string-utilities';
+
+
+
+
+
+const ts = Text_Tree_Settings.load({
+	text: {
+		indention_mode: 'TABULATORS',
+	},
+	trim_lines: true,
+});
+
+const example_string =
+`
+animals:	dog, cat
+trees:		birch, pine
+
+`;
+
+
+const root = Text_Tree_Node.from_string(ts, example_string);
+
+
+const d = new RegExp_Resolver([
+	[/^animals:\s*(.*)$/, (c) => console.log("ANIMAL", c)],
+	[/^trees:\s*(.*)$/, ({node, predicate_result}) => console.log(`TREE of node at line ${node.line_no}:`, parse_csv(predicate_result[1]))],
+]);
+
+for (const child of root.children) {
+	if (child.has_line) {
+		console.log(child.line, d.resolve(child.line, { node: child }));
+	}
+}

experiments/text-nodes.mjs

@@ -1,51 +1,13 @@
 import * as CF from '@efforting.tech/data/field-configuration-factories';
 
-import { readFileSync } from 'node:fs';
 import { inspect } from 'node:util';
-
-import { Indention_Mode, Text_Settings } from '@efforting.tech/text/basic-tree';
-
-// TODO: Move into string helper module
-function string_has_contents(str) {
-	return /\S/.test(str);
-}
-
-function *indented_line_iterator(settings, text) {
-
-	let line_no = settings.first_line;
-	let index = 0;
-	const { indention_tabulator_width } = settings;
-
-	switch (settings.indention_mode) {
-		case Indention_Mode.symbols.TABULATORS: {
-			for (const line of text.matchAll(/^(\t*)(.*)$/gm)) {
-				const [raw, tabs, remaining] = line;
-				yield { raw, indent: tabs.length, line: remaining, line_no: line_no++, index: index++};
-			}
-			break;
-		}
-		case Indention_Mode.symbols.SPACES: {
-			for (const line of text.matchAll(/^([ ]*)(.*)$/gm)) {
-				const [raw, spaces, remaining] = line;
-
-				if ((spaces.length % indention_tabulator_width) !== 0) {
-					throw new Error('Unaligned indention');	//TODO - proper error
-				}
-				yield { raw, indent: Math.floor(spaces.length / indention_tabulator_width), line: remaining, line_no: line_no++, index: index++};
-			}
-			break;
-		}
-		default:
-			throw new Error('Unsupported indention mode');	//TODO - proper error
-
-	}
+import { Text_Tree_Node, Text_Tree_Settings } from  '@efforting.tech/text/basic-tree';
 
 
-}
-
-
-const ts = Text_Settings.load({
-	indention_mode: 'TABULATORS',
+const ts = Text_Tree_Settings.load({
+	text: {
+		indention_mode: 'TABULATORS',
+	},
 });
 
 const example_string =
@@ -68,61 +30,9 @@ branch3
 `;
 
 
-class Text_Node {
-	constructor(text_settings, line=undefined, indent=0, line_no=undefined, index=undefined, raw=undefined, parent=undefined) {
-		Object.assign(this, { text_settings, line, indent, line_no, index, raw, parent, children: [] });
-	}
-}
+const root = Text_Tree_Node.from_string(ts, example_string);
 
 
-const root = new Text_Node(ts);
-
-// NOTE: This first Text_Node is not added to the tree, it serves as an initial cursor only.
-let current = new Text_Node(root.text_settings, undefined, 0, undefined, undefined, undefined, root);
-
-
-for (const line_info of indented_line_iterator(ts, example_string)) {
-
-	// TODO: Implement other variants than inherit-from-previous
-	const indent = string_has_contents(line_info.line) ? line_info.indent : current.indent;
-
-	const delta_indent = indent - current.indent;
-
-	if (delta_indent == 0) {
-		const pending = new Text_Node(current.text_settings, undefined, current.indent, undefined, undefined, undefined, current.parent);	// Partial insertion - same level
-		if (current.parent) {
-			current.parent.children.push(pending);
-		}
-		current = pending;
-	} else if (delta_indent > 0) {
-		for (let i=0; i<delta_indent; i++) {
-			const pending = new Text_Node(current.text_settings, undefined, current.indent + 1, undefined, undefined, undefined, current);	// Partial insertion
-			current.children.push(pending);
-			current = pending;
-		}
-	} else {
-		for (let i=0; i>delta_indent; i--) {
-			current = current.parent;
-		}
-
-		const pending = new Text_Node(current.text_settings, undefined, current.indent, undefined, undefined, undefined, current.parent);	// Partial insertion - same level
-		if (current.parent) {
-			current.parent.children.push(pending);
-		}
-		current = pending;
-	}
-
-	// Fill in partial insertion
-	Object.assign(current, {
-		line: line_info.line,
-		line_no: line_info.line_no,
-		index: line_info.index,
-		raw: line_info.raw,
-	});
-
-
-}
-
 
 function debug_dump(node, level=0) {
 	console.log(`${"  ".repeat(level)}[${node.line_no ?? '-'}] ${inspect(node.line)}`);
@@ -132,6 +42,8 @@ function debug_dump(node, level=0) {
 }
 
 debug_dump(root);
+
+
 /*
 
 [-] undefined

experiments/tree-sitter-1.mjs

@@ -0,0 +1,52 @@
+import Parser from 'tree-sitter';
+import JavaScript from 'tree-sitter-javascript';
+import { readFileSync } from 'node:fs';
+import { inspect } from 'node:util';
+
+// NOTE: 	once upon a time
+//			there was some sort of
+//
+//			example comment that we wanted to investigate
+
+const parser = new Parser();
+parser.setLanguage(JavaScript);
+
+class example {
+
+	#private = 123
+
+	/*
+		This other comment
+		is of the style of a
+		block comment of course
+	*/
+
+	stuff() {
+
+	}
+
+}
+
+const source = readFileSync('./tree-sitter-1.mjs', 'utf-8');
+const tree = parser.parse(source);
+
+function* iter_nodes(node) {
+    yield node;
+    for (let i = 0; i < node.childCount; i++) {
+        yield* iter_nodes(node.child(i));
+    }
+}
+
+
+function *iter_child_nodes(node) {
+    for (let i = 0; i < node.childCount; i++) {
+        yield node.child(i);
+    }
+}
+
+
+for (const node of iter_child_nodes(tree.rootNode)) {
+    //console.log({type: node.type, text: node.text?.slice(0, 40)});
+    console.log(inspect(node));
+    console.log({text: node.text});
+}

package-manifest.yaml

@@ -1,6 +1,7 @@
+# TODO: Add a way to help making sure we keep internal-dependencies up to date
 scope: '@efforting.tech'
 registry: 'https://npm.efforting.tech/'
-version: 0.2.8
+version: 0.2.9
 
 author:
   name: 'Mikael Lövqvist'
@@ -25,6 +26,24 @@ packages:
     path: source/data
     documentation: documentation/data
     description: Data management
+    internal-dependencies:
+      - schema
+
+  table:
+    path: source/table
+    description: Table management
+    internal-dependencies:
+      - schema
+      - text
+
+  feature:
+    path: source/feature
+    description: Feature management
+
+  schema:
+    path: source/schema
+    #documentation: documentation/schema
+    description: Schema system
     internal-dependencies:
       - errors
 
@@ -35,6 +54,14 @@ packages:
     internal-dependencies:
       - data
 
+  parsing:
+    path: source/parsing
+    #documentation: documentation/text
+    description: Generic string parsing
+    internal-dependencies:
+      - errors
+      - text
+
 wip-packages:
   object-graph-storage:
     path: source/object-graph-storage

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "@efforting.tech/experiments": "file:experiments"
+  }
+}

planning/dsl.md

@@ -0,0 +1,25 @@
+
+## Example of indented block
+
+```
+§ block
+	this content
+	is in the block body
+```
+
+
+## Inline
+
+`«inline expression»`
+
+## Escaping
+
+
+```
+«§» block
+
+	Here is an «la»inline expression«ra»
+	Or possibly «outer escape: inline expression»
+
+```
+

planning/schema-vs-data.md

@@ -0,0 +1,6 @@
+# Rationale
+
+Schema is a quite low level feature that is used all over the place while the data package is more of a high level construct.
+It makes sense to move schemas into their own package.
+
+Schema uses a lot of informal conditions and will not depend on rule-processing/conditions but one could use those in higher levels if wanted.

planning/to-document.md

@@ -0,0 +1,19 @@
+# Readme.md
+
+In `readme.md` we should explain that `npm install` can be used both with and without development dependencies.
+
+We are likely to want some development dependencies for the following purposes:
+
+- Template processing
+- Ecmascript parsing for inline rich semantic documentation
+
+
+## Installing tree-sitter-javascript
+
+- requires `node-gyp`
+
+```sh
+CXXFLAGS="-std=c++20" npm i -D tree-sitter tree-sitter-javascript
+```
+
+This section should be somewhat expanded - especially regarding node-gyp

source/data/iteration-utilities.mjs

@@ -0,0 +1,31 @@
+export class Switchable_Iterator {
+	constructor(iterator=null, stack=[]) {
+		Object.assign(this, { iterator, stack });
+	}
+
+	push(iterator) {
+		this.stack.push(this.iterator);
+		this.switch_to(iterator);
+	}
+
+	pop(iterator) {
+		this.switch_to(this.stack.pop());
+	}
+
+	switch_to(iterator) {
+		this.iterator = iterator;
+	}
+
+	next() {
+		return this.iterator.next();
+	}
+
+	peek() {
+		return this.iterator.peek();
+	}
+
+	[Symbol.iterator]() {
+		return this;
+	}
+
+}

source/data/object-utilities.mjs

@@ -0,0 +1,9 @@
+
+export function assign_defined(target, source) {
+	Object.assign(target, Object.fromEntries(Object.entries(source).filter(([k ,v]) => v !== undefined )));
+}
+
+
+export function assign_using_predicate(target, source, kv_predicate) {
+	Object.assign(target, Object.fromEntries(Object.entries(source).filter(kv_predicate)));	// Call predicate with ([k, v])
+}

source/data/stack.mjs

@@ -0,0 +1,54 @@
+export const DELETE_PROPERTY = Symbol('DELETE_PROPERTY');
+
+
+export class String_Keyed_Stack {
+	constructor(target={}, stack=[]) {
+		Object.assign(this, { target, stack });
+	}
+
+	push(updates={}) {
+		const frame = {}
+		this.stack.push(frame);
+		for (const [key, value] of Object.entries(updates)) {
+
+			if (key in this.target) {
+				frame[key] = this.target[key];
+			} else {
+				frame[key] = DELETE_PROPERTY;
+			}
+
+			if (value === DELETE_PROPERTY) {
+				delete this.target[key];
+			} else {
+				this.target[key] = value;
+			}
+		}
+		return frame;
+	}
+
+	push_defined(updates={}) {
+		this.push(Object.fromEntries(Object.entries(updates).filter(([k ,v]) => v !== undefined )));
+	}
+
+	pop(copy_previous_state=false) {
+		const frame = this.stack.pop();
+		const { target } = this;
+
+		const return_value = copy_previous_state ? { ...target } : null;
+
+		for (const [key, value] of Object.entries(frame)) {
+			if (value === DELETE_PROPERTY) {
+				delete target[key];
+			} else {
+				target[key] = value;
+			}
+		}
+
+		return return_value;
+	}
+
+	get top_reverse_delta() {
+		return this.stack.at(-1);
+	}
+
+}

source/data/string-utilities.mjs

@@ -0,0 +1,56 @@
+import * as CF from '@efforting.tech/schema/field-configuration-factories';
+
+
+export const Indention_Mode = new CF.symbol_set({
+	AUTO: 'Automatic detection of indention mode',
+	SPACES: 'Indention is based on spaces',
+	TABULATORS: 'Indention is based on tabulators',
+}, 'Indention mode');
+
+
+// BUG: Current implementation of CF.symbol_set doesn't support default value
+export const Text_Settings = new CF.Schema({
+	indention_mode: Indention_Mode,
+	indention_tabulator_width: CF.cardinal_value(4, 'Width of a tabulator in spaces'),
+	first_line: CF.natural_value(1, 'First line number'),
+}, 'Text settings');
+
+
+export function string_has_contents(str) {
+	return /\S/.test(str);
+}
+
+export function parse_csv(str) {
+	// NOTE: This is for simple comma separated values, a future RFC-4180 compatible version would have to be in a different module in this library (or be a third party thing)
+	return str.split(',').map(element => element.trim());
+}
+
+
+export function *indented_line_iterator(settings, text) {
+	let line_no = settings.first_line;
+	let index = 0;
+	const { indention_tabulator_width } = settings;
+
+	switch (settings.indention_mode) {
+		case Indention_Mode.symbols.TABULATORS: {
+			for (const line of text.matchAll(/^(\t*)(.*)$/gm)) {
+				const [raw, tabs, remaining] = line;
+				yield { raw, indent: tabs.length, line: remaining, line_no: line_no++, index: index++};
+			}
+			break;
+		}
+		case Indention_Mode.symbols.SPACES: {
+			for (const line of text.matchAll(/^([ ]*)(.*)$/gm)) {
+				const [raw, spaces, remaining] = line;
+
+				if ((spaces.length % indention_tabulator_width) !== 0) {
+					throw new Error('Unaligned indention');	//TODO - proper error
+				}
+				yield { raw, indent: Math.floor(spaces.length / indention_tabulator_width), line: remaining, line_no: line_no++, index: index++};
+			}
+			break;
+		}
+		default:
+			throw new Error(`Unsupported indention mode: ${settings.indention_mode}`);	//TODO - proper error
+	}
+}

source/errors.mjs

@@ -1,5 +1,17 @@
 import { inspect } from 'node:util';
 
+// § GROUP: Regexp tokenization
+
+
+
+export class Tokenization_Error extends Error {
+	constructor(data) {
+		const { parser, value, index, end_index } = data;
+		super(`Tokenization_Error`);	//TODO: Format message
+		this.data = data;
+	}
+}
+
 // § GROUP: Configuration field errors
 
 export class Data_Validation_Failed extends Error {
@@ -50,7 +62,7 @@ export class Item_Unresolvable extends Error {
 	constructor(data) {
 		const { resolver, item } = data;
 		const type = item === null ? 'null' : typeof item;
-		super(`Cannot resolve item ${inspect(item)} of type "${type}" using resolver ${resolver}`);
+		super(`Cannot resolve item ${inspect(item)} of type "${type}" using resolver ${resolver.constructor.name}`);
 		this.data = data;
 	}
 }

source/feature/stub.mjs

@@ -0,0 +1,7 @@
+
+export function Stub(meta, name, description, module_name, function_name) {
+	return function stub() {
+		throw new Error(`The feature "${name}" of "${meta.url}" is not enabled. Enable it by calling "${function_name}(${this.name})" imported from "${module_name}"`);	//TODO - specific error
+	}
+}
+

source/parsing/generic-parsing.mjs

@@ -0,0 +1,100 @@
+import { RegExp_Token_Rule } from '@efforting.tech/parsing/regexp-dispatch';
+import { Switchable_Iterator } from '@efforting.tech/data/iteration-utilities';
+import { String_Keyed_Stack } from '@efforting.tech/data/stack';
+import { assign_defined } from '@efforting.tech/data/object-utilities';
+import * as F from '@efforting.tech/schema/field-configuration-factories';
+
+export class RegExp_Token_Parsing_Rule extends RegExp_Token_Rule {
+	constructor(pattern, action, identifier=undefined) {
+		super(pattern, identifier);
+		Object.assign(this, { action });
+	}
+}
+
+
+export const Parser_State = new F.Schema({
+
+	position: F.value(0, 'Pending position in source'),
+	value: F.factory(() => [], 'Pending value to return'),
+	sub_tokenizer_handlers: F.factory(() => [], 'Pending sub tokenizer handlers'),
+	tokenizer: F.value(null, 'Current tokenizer'),
+	context: F.value(null, 'User supplied context'),
+
+}, 'Parser state');
+
+
+
+export class Parser {
+	constructor(source, state=undefined) {
+		state = Parser_State.load(state);
+		const token_generator = new Switchable_Iterator();
+		const stack = new String_Keyed_Stack(state);
+		Object.assign(this, { source, state, stack, token_generator });
+		this.switch_to();
+	}
+
+
+	switch_to(tokenizer=undefined, position=undefined) {
+		assign_defined(this.state, { tokenizer, position });
+		this.token_generator.switch_to(this.state.tokenizer.iter_matches(this.source, this.state.position));
+	}
+
+	parse(handler=undefined) {
+
+		for (const match of this.token_generator) {
+			const { action } = match.rule;
+			if (!action) { continue; }
+
+			if (typeof action !== 'function') {	//TODO - proper error (possibly a warning, the warning is nice when you are developing, have to think about this one)
+				console.log('NOT IMPLEMENTED', match.rule.action);
+				continue;
+			}
+
+			this.state.position = match.pending_index;
+			this.state.match = match;
+			action(this, match);
+
+		}
+
+		if (handler) {
+			this.state.match = null;	//TODO: Decide if we should reset match here or not
+			handler(this, this.state.value);
+			return this.state.value;
+		} else {
+			return this.state.value;
+		}
+
+
+	}
+
+	push_token(...tokens) {
+		this.state.value.push(...tokens);
+	}
+
+	replace_value(value) {
+		this.state.value = value;
+	}
+
+	enter_sub_tokenizer(tokenizer=undefined, handler=undefined) {
+		this.stack.push_defined({ tokenizer, value: [] });
+		if (handler) {
+			this.state.sub_tokenizer_handlers.push(handler);
+		}
+		this.switch_to(tokenizer);
+	}
+
+	leave_sub_tokenizer(egress_match=null) {
+		const frame = this.stack.pop(true);
+		const { sub_tokenizer_handlers } = this.state;
+
+		if (sub_tokenizer_handlers.length) {
+			const handler = sub_tokenizer_handlers.pop();
+			this.state.match = null;	//TODO: Decide if we should reset match here or not
+			handler(this, frame.value, egress_match);
+		} else {
+			this.push_token(frame.value);
+		}
+		this.switch_to();
+	}
+
+}

source/parsing/regexp-dispatch.mjs

@@ -0,0 +1,185 @@
+import * as RE from '@efforting.tech/text/regexp';
+import { Tokenization_Error } from '@efforting.tech/errors';
+
+
+// NOTE: 	There are some open questions about this implementation and API which may change as the library matures.
+//			Check out the example at experiments/regexp-tokenizer.mjs for more information on how to use this in its current state.
+//
+//			Specifically it is not currently decided where the boundary between rule/action/capture should be
+
+export class Pattern_Match {
+	constructor(match, rule) {
+		Object.assign(this, { match, rule });
+	}
+
+	get identifier() {
+		return this.rule. identifier;
+	}
+
+	get value() {
+		return this.match[0];
+	}
+
+	get captured() {
+		return this.match.slice(1);
+	}
+
+	get pending_index() {
+		return this.match.index + this.match[0].length;
+	}
+
+}
+
+export class Default_Match {
+	constructor(text, index, end_index, action) {
+		const identifier = action(this);
+		Object.assign(this, { text, index, end_index, action, identifier });
+	}
+
+	get value() {
+		return this.text;
+	}
+
+
+	get pending_index() {
+		if (this.end_index === null) {
+			return null;
+		} else {
+			return this.end_index;
+		}
+	}
+}
+
+
+export class Abstract_RegExp_Token_Rule {
+
+	constructor(pattern) {
+
+		const pattern_source = RE.get_source(pattern);
+		const pattern_flags = RE.get_flags(pattern);
+
+		const immediate_flags = String.prototype.concat(...(new Set([...pattern_flags, 'y'])));
+		const scanning_flags =  String.prototype.concat(...(new Set([...pattern_flags, 'g'])));
+
+		const immediate_pattern = new RegExp(pattern_source, immediate_flags);
+		const scanning_pattern = new RegExp(pattern_source, scanning_flags);
+
+		Object.assign(this, { pattern, immediate_pattern, scanning_pattern });
+	}
+
+}
+
+export class RegExp_Token_Rule extends Abstract_RegExp_Token_Rule {
+	constructor(pattern, identifier=undefined) {
+		super(pattern);
+		Object.assign(this, { identifier: identifier ?? this });
+	}
+}
+
+// Note: There is no clean built in way to set an end position of a RegExp pattern, the only generic way is to slice the string we match before.
+//		 We may at some point implement support for this (and it would only be done if end position was given)
+
+export class RegExp_Tokenizer {
+	constructor(rules=[], default_action=undefined) {
+		Object.assign(this, { rules, default_action });
+	}
+
+	set_default_identifier(identifier) {
+		this.default_action = (
+			() => identifier
+		);
+	}
+
+	add_rules(...rules_to_add) {
+		this.rules.push(...rules_to_add);
+	}
+
+	immediate_match(text, position=0) {
+		for (const rule of this.rules) {
+			const pattern = rule.immediate_pattern;
+			pattern.lastIndex = position;
+			const match = pattern.exec(text);
+			if (match) {
+				return new Pattern_Match(match, rule);
+			}
+		}
+	}
+
+
+	_handle_default_match(value, index, end_index=null) {
+		const { default_action } = this;
+		if (!default_action) {
+			throw new Tokenization_Error({ parser: this, value, index, end_index });
+		}
+		return new Default_Match(value, index, end_index, default_action);
+	}
+
+
+	closest_scanning_match(text, position=0) {
+		const immediate_match = this.immediate_match(text, position);
+		if (immediate_match) {
+			return immediate_match;
+		}
+
+		let best_candidate;
+		for (const candidate of this.iter_scanning_rule_candidates(text, position)) {
+			if ((best_candidate === undefined) || (best_candidate.match.index > candidate.match.index)) {
+				best_candidate = candidate;
+			}
+		}
+
+		// There was no match, just get the tail
+		if (!best_candidate) {
+			const tail = text.slice(position);
+			if (tail.length) {
+				return this._handle_default_match(tail, position);
+			}
+		}
+
+		// There was a match, check the head
+		if (best_candidate) {
+			const head = text.slice(position, best_candidate.match.index);
+			if (head.length) {
+				return this._handle_default_match(head, position, best_candidate.match.index);
+			}
+		}
+
+		return best_candidate;
+
+	}
+
+
+	*iter_scanning_rule_candidates(text, position=0) {
+		// Iterates over all rules and yields any matches found anywhere (but only once per rule)
+
+		for (const rule of this.rules) {
+			const pattern = rule.scanning_pattern;
+			pattern.lastIndex = position;
+			const match = pattern.exec(text);
+
+			if (match) {
+				yield new Pattern_Match(match, rule);
+			}
+		}
+
+	}
+
+
+	*iter_matches(text, position=0) {
+		while (true) {
+			const pending = this.closest_scanning_match(text, position);
+			if (pending) {
+				yield pending;
+			}
+			if (!pending || pending.pending_index === null) {
+				break;
+			}
+			position = pending.pending_index;
+
+		}
+	}
+
+
+
+}
+

source/rule-processing/contracts.mjs

@@ -0,0 +1,74 @@
+import { FPR_State } from './state.mjs';
+
+export class Abstract_Reduction_Contract {
+	on_create_scanner(scanner) {}
+	on_start_transform(scanner, sequence) {}
+	on_end_transform(scanner, sequence) {}
+	on_reduction_attempted(scanner, sequence) {}
+	on_reduction_made(scanner, sequence) {}
+
+	check_if_done(scanner, sequence) {
+		return false;
+	}
+
+	assert_readiness(scanner, sequence) {}
+}
+
+export class Reduction_Contract extends Abstract_Reduction_Contract {
+	check_if_done(scanner, sequence) {
+		return scanner.last_reduction_status === false;
+	}
+}
+
+export class FPR_Contract extends Reduction_Contract {
+	constructor(end_condition=null) {
+		super();
+		Object.assign(this, { end_condition });
+	}
+
+	check_if_done(scanner, sequence) {
+		if (this.end_condition) {
+			return this.end_condition(scanner, sequence);
+		} else {
+			return super.check_if_done(scanner, sequence);
+		}
+	}
+
+	on_create_scanner(scanner) {
+		Object.assign(scanner, {
+			cycle_detected: false,
+			state_manager: new FPR_State(),
+		});
+	}
+
+	on_start_transform(scanner, sequence) {
+		super.on_start_transform(scanner, sequence);
+		const state_manager = scanner.state_manager;
+		state_manager.clear_journal();
+		const state = state_manager.gather_state(sequence);	//Log initial state
+		state_manager.log_state(state);
+
+		scanner.cycle_detected = false;
+	}
+
+	on_reduction_made(scanner, sequence) {
+		const state_manager = scanner.state_manager;
+		const state = state_manager.gather_state(sequence);
+
+		const pre_count = state_manager.length;
+		state_manager.log_state(state);
+		const post_count = state_manager.length;
+
+		if (pre_count === post_count) {
+			scanner.cycle_detected = true;
+		}
+
+	}
+
+	assert_readiness(scanner, sequence) {
+		if (scanner.cycle_detected) {
+			throw new Error('Cycle detected');
+		}
+	}
+
+}

source/rule-processing/reduction-scanner.mjs

@@ -0,0 +1,123 @@
+import { Reduction_Contract, FPR_Contract } from './contracts.mjs';
+import * as CF from '@efforting.tech/schema/field-configuration-factories';
+
+
+export const Reduction_Order = new CF.symbol_set({
+	RULE_MAJOR: 'For each rule, scan the full sequence',
+	POSITION_MAJOR: 'For each item in the sequence, try all rules',
+}, 'Reduction ordering');
+
+
+export const Reduction_Settings = new CF.Schema({
+	reduction_order: CF.symbol(Reduction_Order, 'RULE_MAJOR'),
+	reduction_contract: CF.instance(Reduction_Contract, 'The reduction contract defines the implementation of the reduction scanner'),
+	rules: CF.factory(() => []),
+}, 'Reduction settings');
+
+//TODO: Consider whether we can implement some hierarchical sub schema, like classes - we could of course reuse things by defining and object and spread it in here
+export const FP_Reduction_Settings = new CF.Schema({
+	reduction_order: CF.symbol(Reduction_Order, 'RULE_MAJOR'),
+	reduction_contract: CF.instance(FPR_Contract, 'The reduction contract defines the implementation of the reduction scanner'),
+	rules: CF.factory(() => []),
+}, 'Fixed point reduction settings');
+
+
+
+//TODO - we should probably have a pre-defined record shape as argument for actions and such rather than using an ever growing list of positionals or an anonymous Object()
+
+export class Reduction_Scanner {
+
+	static settings_schema = Reduction_Settings;
+
+	constructor(settings, use_lifecycle_cb=true) {
+		settings = this.constructor.settings_schema.load(settings);
+		Object.assign(this, { settings });
+		this.clear_transform_state();
+		if (use_lifecycle_cb) {
+			settings.reduction_contract.on_create_scanner(this);
+		}
+	}
+
+	find_reduction_candidate(sequence) {
+		const { settings } = this;
+		switch (settings.reduction_order) {
+
+			case Reduction_Order.symbols.RULE_MAJOR:
+				for (const rule of settings.rules) {
+					const match = rule.match(sequence);
+					if (match) {
+						return { sequence, rule, match };
+					}
+				}
+				return;
+
+			case Reduction_Order.symbols.POSITION_MAJOR:
+
+				let best_match, best_rule;
+
+				for (const rule of settings.rules) {
+					const match = rule.match(sequence);
+
+					if (match)  {
+						if (!best_match || best_match.match_start > match.match_start) {
+							//TODO - early return if start of sequence
+							best_match = match;
+							best_rule = rule;
+						}
+					}
+				}
+
+				if (best_match) {
+					return { sequence, rule: best_rule, match: best_match };
+				}
+
+
+				return;
+			default:
+				throw new Error(`Unknown reduction order: ${this.reduction_order}`);	//TODO: Force invalid configuration  error
+
+		}
+	}
+
+
+	perform_reduction(sequence) {
+		const candidate = this.find_reduction_candidate(sequence);
+		if (candidate) {
+			const { sequence, rule, match } = candidate;
+			//console.log('ACT', match.match)
+			match.action({ reduction_system: this, rule, sequence, match: match.match });
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	clear_transform_state() {
+		this.last_reduction_status = null;
+		this.reductions_attempted = 0;
+		this.reductions_made = 0;
+	}
+
+	transform(sequence) {
+		const { settings } = this;
+		const reduction_contract = settings.reduction_contract;
+		this.clear_transform_state();
+		reduction_contract.on_start_transform(this, sequence);
+
+		while (!reduction_contract.check_if_done(this, sequence)) {
+			reduction_contract.assert_readiness(this, sequence);
+			this.last_reduction_status = this.perform_reduction(sequence);
+			this.reductions_attempted++;
+			if (this.last_reduction_status) {
+				this.reductions_made++;
+				reduction_contract.on_reduction_made(this, sequence);
+			}
+			reduction_contract.on_reduction_attempted(this, sequence);
+		}
+
+		reduction_contract.on_end_transform(this, sequence);
+		return sequence;
+	}
+
+}
+

source/rule-processing/resolvers.mjs

@@ -1,12 +1,15 @@
 import { Item_Unresolvable } from '@efforting.tech/errors';
 
+//TODO: Should this be integrated with rules.mjs for predicates? Possibly a normalization step during rule insertion.
+
 export class Abstract_Resolver {
-	resolve(item) {
-		const handler = this.resolve_handler(item);
-		if (!handler) {
+	resolve(item, extra_info={}) {
+		const result = this.resolve_handler(item, extra_info);
+		if (!result?.handler) {
 			throw new Item_Unresolvable({ resolver: this, item });
 		}
-		return handler({ resolver: this, item });
+		// TO DOC: Spreading result into the resulting context means there are some reserved keys we need to be mindful of to avoid clobbering them
+		return result.handler({ resolver: this, item, ...extra_info, ...result });
 	}
 }
 
@@ -16,17 +19,47 @@ export class Chained_Resolver extends Abstract_Resolver {
 		Object.assign(this, { chain_links });
 	}
 
-	resolve_handler(item) {
+	resolve_handler(item, extra_info={}) {
 		const { chain_links } = this;
 		for (const link of chain_links) {
-			const handler = link.resolve_handler(item);
-			if (handler) {
-				return handler;
+			const result = link.resolve_handler(item, extra_info);
+			if (result?.handler) {
+				return result;
 			}
 		}
 	}
 }
 
+export class Predicate_Resolver extends Abstract_Resolver {
+	constructor(rules=[]) {
+		// NOTE: Rules should be iterable as [predicate, handler] pairs
+		super();
+		Object.assign(this, { rules });
+	}
+
+	resolve_handler(item, extra_info={}) {
+		const { rules } = this;
+		for (const [predicate, handler] of rules) {
+			const predicate_result = predicate(item, extra_info);
+			// NOTE: to return a falsy predicate_result as a positive hit you must wrap it in something
+			if (predicate_result) {
+				return { handler, predicate_result };
+			}
+		}
+	}
+}
+
+export class RegExp_Resolver extends Predicate_Resolver {
+	constructor(rules=[]) {
+		// NOTE: Rules should be iterable as [predicate, handler] pairs
+		super();
+		Object.assign(this, {
+			rules: rules.map(([pattern, handler]) => [(str, extra_info) => str.match(pattern), handler])
+		});
+	}
+
+
+}
 
 export class Mapping_Resolver extends Abstract_Resolver {
 	constructor(rules=new Map(), key_function=null) {
@@ -34,7 +67,7 @@ export class Mapping_Resolver extends Abstract_Resolver {
 		Object.assign(this, { rules, key_function });
 	}
 
-	resolve_handler(item) {
+	resolve_handler(item, extra_info={}) {
 		const { key_function, rules } = this;
 		const key = key_function ? key_function(item) : item;
 		return rules.get(key);
@@ -42,3 +75,26 @@ export class Mapping_Resolver extends Abstract_Resolver {
 
 }
 
+
+export class Sequential_Resolver extends Abstract_Resolver {
+	constructor(rules=[]) {
+		// NOTE: Rules should be iterable as [Abstract_Sequential_Condition, handler] pairs
+		super();
+		Object.assign(this, { rules });
+	}
+
+
+	resolve_handler(item, extra_info={}) {
+		const { rules } = this;
+
+		for (const [sequential_condition, handler] of rules) {
+			const sequential_condition_result = sequential_condition.match(item, extra_info);
+			// NOTE: to return a falsy predicate_result as a positive hit you must wrap it in something
+			if (sequential_condition_result) {
+				return { handler, sequential_condition_result };
+			}
+		}
+	}
+
+}
+

source/rule-processing/rules.mjs

@@ -0,0 +1,108 @@
+
+const ABORT_SEQUENCE = Symbol('ABORT_SEQUENCE');
+
+export class Abstract_Item_Condition {
+	match(sequence, context={}) {
+		return this.match_value(sequence[context.start_index ?? 0], context);
+	}
+
+}
+
+export class Abstract_Sequence_Condition {
+}
+
+export class Match {
+	constructor(rule, value, context) {
+		Object.assign(this, { rule, value, ...context });
+	}
+}
+
+export class Predicate extends Abstract_Item_Condition {
+	constructor(predicate) {
+		super();
+		Object.assign(this, { predicate });
+	}
+
+	match_value(item, context={}) {
+		if (this.predicate(item, context)) {
+			const si = context.start_index ?? 0;
+			return new Match(this, item, { ...context, match_start: si, match_end: si });
+		};
+	}
+
+}
+
+export class Type_Is extends Abstract_Item_Condition {
+	constructor(type) {
+		super();
+		Object.assign(this, { type });
+	}
+
+	match_value(item, context={}) {
+		if (typeof item === this.type) {
+			const si = context.start_index ?? 0;
+			return new Match(this, item, { ...context, match_start: si, match_end: si });
+		};
+	}
+
+}
+
+export class Strict_Equality extends Abstract_Item_Condition {
+	constructor(value) {
+		super();
+		Object.assign(this, { value });
+	}
+
+	match_value(item, context={}) {
+		if (item === this.value) {
+			const si = context.start_index ?? 0;
+			return new Match(this, item, { ...context, match_start: si, match_end: si });
+		}
+	}
+
+}
+
+
+export class Sequence_Condition extends Abstract_Sequence_Condition {
+	constructor(sequence) {
+		super();
+		Object.assign(this, { sequence });
+	}
+
+	match(sequence, context={}) {
+		//TODO: For anchors we need anchor_start_index and anchor_end_index (compare with regexp ^ and $)
+		const start_index = context.start_index ?? 0;
+		const end_index = context.end_index ?? sequence.length - 1;
+
+		const match_from = (sequence_index, pattern_index=0) => {
+
+			if (pattern_index === this.sequence.length) {
+				return [];
+			}
+
+			//TODO - There are plenty of optimizations to be implemented here but we must be suer they are correct - we will start naively
+			const sub_condition = this.sequence[pattern_index];
+			const sub_match = sub_condition.match(sequence, { ...context, start_index: sequence_index });
+
+			//console.log('match_from', {sequence_index, pattern_index, sub_condition, sub_match})
+
+			if (sub_match) {
+				const remaining = match_from(sub_match.match_end + 1, pattern_index + 1);
+				if (remaining) {
+					return [sub_match, ...remaining];
+				}
+			}
+
+		}
+
+		for (let i=start_index; i<=end_index; i++) {
+			const m = match_from(i);
+			if (m) {
+				// NOTE: If result is empty array which can be a positive match, match_start and match_end will be undefined which is by design
+				return new Match(this, m, { match_start: m.at(0)?.match_start, match_end: m.at(-1)?.match_end, ...context});
+			}
+		}
+
+	}
+
+}

source/rule-processing/state-serializer.mjs

@@ -0,0 +1,134 @@
+import * as CF from '@efforting.tech/schema/field-configuration-factories';
+
+
+export const Object_Serialization_Strategy = new CF.Schema({
+	identity: CF.boolean(true, 'Serialize based on identity'),
+	type: CF.boolean(true, 'Serialize based on constructor'),
+	symbols: CF.boolean(true, 'Include symbol-keyed properties'),
+	keys: CF.boolean(true, 'Include property keys'),
+	values: CF.boolean(true, 'Include property values'),
+}, 'Object serialization strategy');
+
+
+
+export const State_Serialization_Strategy = new CF.Schema({
+	//BUG - there is currently no way (I think) to put defaults into a sub schema - this should be fixed
+	array: CF.schema(Object_Serialization_Strategy, {
+		identity: false,
+		type: true,
+		symbols: false,
+		keys: false,
+		values: true,
+	}, 'State serialization strategy for Array'),
+	object: CF.schema(Object_Serialization_Strategy, {
+		identity: true,
+		type: true,
+		symbols: true,
+		keys: true,
+		values: true,
+	}, 'State serialization strategy for Object'),
+}, 'State serialization strategy');
+
+
+
+
+export class State_Serializer {
+
+	static settings_schema = State_Serialization_Strategy;
+
+	constructor(symbols=new Map(), strategy) {
+		strategy = this.constructor.settings_schema.load(strategy);
+		Object.assign(this, { symbols, strategy });
+	}
+
+	serialize_symbols(...symbols) {
+		const result = [];
+		for (const symbol of symbols) {
+			const existing = this.symbols.get(symbol);
+			if (existing !== undefined) {
+				result.push(existing);
+			} else {
+				const new_symbol_index = this.symbols.size;
+				this.symbols.set(symbol, new_symbol_index);
+				result.push(new_symbol_index);
+			}
+		}
+		return result;
+	}
+
+
+	serialize_object(object) {
+		const state_strategy = this.strategy;
+		const result = [];
+
+		switch (typeof(object)) {
+			case 'object':
+				if (object === null) {
+					return ['N'];
+				}
+
+
+				const is_array = object.constructor === Array;
+				const strategy = is_array ? state_strategy.array : state_strategy.object;
+				result.push(is_array ? 'a' : 'o');
+
+
+				if (strategy.identity) {
+					const existing_instance = this.symbols.get(object);
+					if (existing_instance !== undefined) {
+						return existing_instance;
+					}
+
+					const [instance] = this.serialize_symbols(object);
+					result.push(instance);
+				}
+
+				if (strategy.type) {
+					const [type] = this.serialize_symbols(object.constructor);
+					result.push(type);
+				}
+
+				if (strategy.symbols) {
+					const symbols = this.serialize_symbols(...Object.getOwnPropertySymbols(object));
+					result.push(symbols);
+				}
+
+				if (strategy.keys) {
+					const keys = Object.keys(object).map(item => this.serialize_object(item));
+					result.push(keys);
+				}
+
+				if (strategy.values) {
+					const values = Object.values(object).map(item => this.serialize_object(item));
+					result.push(values);
+				}
+
+				return result;
+
+			case 'string':
+				return ['s', ...this.serialize_symbols(typeof(object)), object];
+
+			case 'number':
+				return ['n', ...this.serialize_symbols(typeof(object)), object];
+
+			case 'boolean':
+				return [object ? 'T' : 'F'];
+
+			case 'function':
+				const [reference_type, reference_id] = this.serialize_symbols(typeof(object), object);
+				return ['f', reference_type, reference_id];
+
+			case 'symbol':
+				return ['S', ...this.serialize_symbols(object)];
+
+
+			case 'undefined':
+				return ['u'];
+
+
+			default:
+				throw new Error(typeof(object));	//TODO - proper error
+		}
+	}
+};
+

source/rule-processing/state.mjs

@@ -0,0 +1,47 @@
+import { State_Serializer } from './state-serializer.mjs';
+import { createHash } from 'node:crypto';
+
+
+export class Abstract_State {
+	constructor(seen = new Set()) {
+		Object.assign(this, { seen });
+	}
+
+	gather_state(sequence) {
+		throw new Error(`gather_state(sequence) not implemented for ${this.constructor.name}`);
+	}
+
+	seen_state(state) {
+		return this.seen.has(state);
+	}
+
+	log_state(state) {
+		this.seen.add(state);
+	}
+
+	discard_state(state) {
+		this.seen.delete(state);
+	}
+
+	get length() {
+		return this.seen.size;
+	}
+
+	clear_journal() {
+		this.seen.clear();
+	}
+}
+
+export class FPR_State extends Abstract_State {
+	constructor(seen = new Set(), serializer = new State_Serializer()) {
+		super(seen);
+		Object.assign(this, { serializer });
+	}
+
+	gather_state(sequence) {
+		const str = JSON.stringify(this.serializer.serialize_object(sequence));
+		const hash = createHash('sha1').update(str).digest('hex');
+		return `${hash.slice(0, 8)}-${str}`;
+	}
+
+}

source/schema/field-configuration-factories.mjs

@@ -11,10 +11,21 @@ export function factory(factory_function, description) {
 	return new Field_Configuration(null, null, factory_function, description);
 }
 
+export function instance(type, description) {
+
+	return new Field_Configuration((i) => i instanceof type, null, () => new type(), description);
+}
+
 export function typed_value(coercion_function, default_value, description) {
 	return new Field_Configuration(null, coercion_function, () => default_value, description);
 }
 
+export function boolean(default_value, description) {
+	//BUG: Text representations such as "false" is still truthy here - we should have a more capable coearcing function
+	return new Field_Configuration(null, Boolean, () => default_value, description);
+}
+
+
 export function typed_factory(coercion_function, factory_function, description) {
 	return new Field_Configuration(null, coercion_function, factory_function, description);
 }
@@ -27,6 +38,14 @@ export function typed_required(coercion_function, description) {
 	return new Field_Configuration((value) => value !== undefined, coercion_function, null, description);
 }
 
+export function schema(field_schema, default_value, description) {
+	return new Field_Configuration(null, (v) => field_schema.load(v), () => field_schema.load(default_value), description);
+}
+
+export function symbol(symbol_set, default_value, description) {
+	return new Field_Configuration(null, (v) => symbol_set.load(v), () => symbol_set.load(default_value), description);
+}
+
 export function symbol_set(description_to_name_mapping, description=null) {
 
 	const symbols_by_name = Object.fromEntries(Object.keys(description_to_name_mapping).map(k => [k, Symbol(k)]));

source/table/raster-table.mjs

@@ -0,0 +1,46 @@
+import { tabs_to_spaces } from '@efforting.tech/text/text-utilities';
+
+const raster_table_pattern = /^((?:[^\S\n]|[\w-])+)\n((?:[^\S\n]|-)+)\n(.+)/ms;
+const column_pattern = /\s*(.+?)(?=\s{2,}|$)/gd;
+const row_pattern = /^(.+)$/mg
+
+// If loader is null we return a raw representation
+export function load_raster_table(raster, loader=null, tab_width=4, null_padding=true) {
+	if (raster.match(/\t/g)) {
+		raster = tabs_to_spaces(raster, tab_width);
+	}
+
+	if (null_padding) {	//TODO - this is just experimental proof of concept
+		raster = raster.replace(/〃/g, '\0〃');
+	}
+
+
+	const m = raster.match(raster_table_pattern);
+	const column_matches = [...m[1].matchAll(column_pattern)];
+	const column_positions = [...column_matches.map(cm => cm.indices[1][0])]
+	const column_names = [...column_matches.map(cm => cm[1])]
+	const rows = [];
+
+	for (const row of m[3].matchAll(row_pattern)) {
+
+		const pending_row = [];
+		for (let ci=0; ci<column_positions.length; ci++) {
+			const cell = row[1].slice(column_positions[ci], column_positions[ci+1]);
+			if (null_padding) {
+				pending_row.push(cell.replace(/\0/, ''));
+			} else {
+				pending_row.push(cell);
+			}
+		}
+		rows.push(pending_row);
+	}
+
+	if (loader) {
+		return new loader({ rows, column_names });
+	} else {
+		return {column_names, column_positions, rows};
+	}
+}
+
+
+//TODO: load_ditto_raster_table

source/table/table.mjs

@@ -0,0 +1,165 @@
+import * as F from '@efforting.tech/schema/field-configuration-factories';
+import { parse_csv } from '@efforting.tech/data/string-utilities';
+
+export const Row_Based_Table_Settings = new F.Schema({
+
+	rows: F.factory(() => [], 'Rows to initialize table with'),
+	row_names: F.typed_factory((v) => typeof v === 'string' ? parse_csv(v) : v, () => [], 'Names of rows'),
+	column_names: F.typed_factory((v) => typeof v === 'string' ? parse_csv(v) : v, () => [], 'Names of columns'),
+
+}, 'Row based table settings');
+
+
+export class Table_Row_Reference {
+
+	constructor(table, index, snapshot=null) {
+		Object.assign(this, { table, index, snapshot });
+	}
+
+	get value() {
+		const { table, index, snapshot } = this;
+		return snapshot ?? table.read_row(index);
+	}
+
+	get object() {
+		const { table, index, snapshot } = this;
+		const { column_names } = table;
+		const value = snapshot ?? table.read_row(index);
+		//TODO - check shape of column_names vs the value
+
+		return Object.fromEntries(value.map((cell, column_index) => [column_names[column_index], cell]));
+	}
+
+
+	update(updates) {
+		const { table, index, snapshot } = this;
+		if (snapshot) {
+			throw new Error('Can not update snapshot references, clear the snapshot to reuse this index');	//TODO - proper error
+		}
+		table.write_row(index, updates);
+	}
+
+}
+
+export class Row_Based_Table {
+
+	constructor(settings) {
+		const { rows, column_names, row_names } = Row_Based_Table_Settings.load(settings);
+		Object.assign(this, { rows });
+		this.set_column_names(...column_names);
+		this.set_row_names(...row_names);
+	}
+
+	// General operations
+
+	get width() {
+		const { rows, column_names } = this;
+		return rows[0]?.length || column_names.length || undefined;
+	}
+
+	get length() {
+		const { rows } = this;
+		return rows.length;
+	}
+
+	get size() {
+		const { width, length } = this;
+		return [width, length];
+	}
+
+	replace_all_cells(replacement_fn) {
+
+		for (const [row_index, row] of this.rows.entries()) {
+			const pending_row = [];
+			for (const [column_index, cell] of row.entries()) {
+				const row_name = this.row_names[row_index];
+				const column_name = this.column_names[column_index];
+				pending_row.push(replacement_fn({ row_name, column_name, row_index, column_index, row, cell }));
+			}
+			this.rows[row_index] = pending_row;
+		}
+
+
+	}
+
+	// TODO: Make sure column and row operations covers the same uses
+	// TODO: Sub table operations
+
+	// Column operations
+
+	set_column_names(...names) {
+		this.column_names = names;
+		this.column_names_lut = Object.fromEntries(names.map((name, index) => [name, index]));
+	}
+
+	// Row operations
+
+	set_row_names(...names) {
+		this.row_names = names;
+		this.row_names_lut = Object.fromEntries(names.map((name, index) => [name, index]));
+	}
+
+	read_rows(...indices) {
+		const result = [];
+		for (const index of indices) {
+			result.push(new Table_Row_Reference(this, index));
+		}
+		return result;
+	}
+
+	snapshot_rows(...indices) {
+		//NOTE: Snapshot doesn't include current layout or other settings, just the contents of the row at the time, and it currently is a shallow copy
+		const result = [];
+		for (const index of indices) {
+			result.push(new Table_Row_Reference(this, index, [...this.rows[index]]));
+		}
+		return result;
+	}
+
+	push_rows(...rows) {
+		//TODO - verify shape
+		this.rows.push(...rows);
+	}
+
+	read_row(index) {
+		return this.rows[index];
+	}
+
+	write_row(index, row_data) {
+		const { rows, column_names_lut } = this;
+		if (Array.isArray(row_data)) {
+			//TODO - shape check
+			rows[index] = row_data;
+		} else {
+			//TODO - possibly allow array of array for using numerical indices
+			const work_row = rows[index];
+			for (const [key, value] of Object.entries(row_data)) {
+				const col_index = column_names_lut[key];
+
+				if (col_index === undefined) {
+					throw new Error(`Unknown column: ${key}`);	//TODO - proper error
+				}
+
+				work_row[col_index] = value;
+			}
+		}
+	}
+
+
+	//TODO: Implement map and other functions expected by collections
+
+	*[Symbol.iterator]() {
+		for (const index of this.rows.keys()) {
+			yield new Table_Row_Reference(this, index);
+		}
+	}
+
+	*iter_objects() {
+		for (const index of this.rows.keys()) {
+			yield new Table_Row_Reference(this, index).object;
+		}
+	}
+
+
+}
+

source/text/basic-tree.mjs

@@ -1,16 +1,86 @@
-import * as CF from '@efforting.tech/data/field-configuration-factories';
+import { Text_Settings, string_has_contents, indented_line_iterator } from  '@efforting.tech/data/string-utilities';
+import * as CF from '@efforting.tech/schema/field-configuration-factories';
+
+export const Text_Tree_Settings = new CF.Schema({
+	//BUG - there is currently no way (I think) to put defaults into a sub schema - this should be fixed
+	text: Text_Settings,
+	trim_head: CF.boolean(false, 'Trim the empty lines from the head of a node'),
+	trim_tail: CF.boolean(false, 'Trim the empty lines from the tail of a node'),
+	trim_lines: CF.boolean(false, 'Trim lines'),
+}, 'Text tree settings');
+
+
+export class Text_Tree_Node {
+	constructor(text_tree_settings, line=undefined, indent=0, line_no=undefined, index=undefined, raw=undefined, parent=undefined) {
+		Object.assign(this, { text_tree_settings, line, indent, line_no, index, raw, parent, children: [] });
+	}
+
+	get has_line() {
+		return string_has_contents(this.line);
+	}
+
+
+	static from_string(text_tree_settings, str) {
+
+		const root = new this(text_tree_settings);
+		const { trim_head, trim_tail, trim_lines } = text_tree_settings;
+
+		// NOTE: This first Text_Node is not added to the tree, it serves as an initial cursor only.
+		let current = new this(root.text_tree_settings, undefined, 0, undefined, undefined, undefined, root);
+
+
+		for (const line_info of indented_line_iterator(text_tree_settings.text, str)) {
+
+			// TODO: Implement other variants than inherit-from-previous
+			const indent = string_has_contents(line_info.line) ? line_info.indent : current.indent;
+
+			const delta_indent = indent - current.indent;
+
+			if (delta_indent == 0) {
+				const pending = new this(current.text_tree_settings, undefined, current.indent, undefined, undefined, undefined, current.parent);	// Partial insertion - same level
+				if (current.parent) {
+					current.parent.children.push(pending);
+				}
+				current = pending;
+			} else if (delta_indent > 0) {
+				for (let i=0; i<delta_indent; i++) {
+					const pending = new this(current.text_tree_settings, undefined, current.indent + 1, undefined, undefined, undefined, current);	// Partial insertion
+					current.children.push(pending);
+					current = pending;
+				}
+			} else {
+				for (let i=0; i>delta_indent; i--) {
+					current = current.parent;
+				}
+
+				const pending = new this(current.text_tree_settings, undefined, current.indent, undefined, undefined, undefined, current.parent);	// Partial insertion - same level
+				if (current.parent) {
+					current.parent.children.push(pending);
+				}
+				current = pending;
+			}
+
+			// Fill in partial insertion
+			Object.assign(current, {
+				line: trim_lines ? line_info.line.trim() : line_info.line,
+				line_no: line_info.line_no,
+				index: line_info.index,
+				raw: line_info.raw,
+			});
+
+
+		}
+
+
+		if (trim_head || trim_tail) {	//TODO: Implement
+			throw new Error('Trimming is not implemented');	//TODO: Proper non implemented error
+		}
+
+		return root;
+
+	}
+}
 
 
-export const Indention_Mode = new CF.symbol_set({
-	AUTO: 'Automatic detection of indention mode',
-	SPACES: 'Indention is based on spaces',
-	TABULATORS: 'Indention is based on tabulators',
-}, 'Indention mode');
 
 
-// BUG: Current implementation of CF.symbol_set doesn't support default value
-export const Text_Settings = new CF.Schema({
-	indention_mode: Indention_Mode,
-	indention_tabulator_width: CF.cardinal_value(4, 'Width of a tabulator in spaces'),
-	first_line: CF.natural_value(1, 'First line number'),
-}, 'Text settings');

source/text/regexp.mjs

@@ -0,0 +1,49 @@
+export function get_flags(pattern) {
+	if (pattern instanceof RegExp) {
+		return new Set(pattern.flags);
+	} else {
+		return new Set();
+	}
+}
+export function get_source(pattern) {
+	if (pattern instanceof RegExp) {
+		return pattern.source;
+	} else {
+		return pattern;
+	}
+}
+
+export function concat(...pattern_list) {
+	let pending_source = '';
+	const pending_flags = new Set();
+
+	for (const pattern of pattern_list) {
+
+		if (pattern instanceof RegExp) {
+			pending_source += pattern.source;
+			for (const flag of pattern.flags) {
+				pending_flags.add(flag);
+			}
+		} else {
+			pending_source += pattern;
+		}
+	}
+
+	return new RegExp(pending_source, String.prototype.concat(...pending_flags));
+}
+
+
+export function join(pattern_list, separator, flags=undefined) {
+	return new RegExp(pattern_list.map(pattern => get_source(pattern)).join(get_source(separator)), flags);
+}
+
+
+export function update_flag(pattern, flag, state) {
+	const pattern_flags = get_flags(pattern);
+	if (state) {
+		pattern_flags.add(flag);
+	} else {
+		pattern_flags.delete(flag);
+	}
+	return new RegExp(pattern.source, String.prototype.concat(...pattern_flags));
+}

source/text/text-utilities.mjs

@@ -0,0 +1,30 @@
+export const char_width_lut = {
+	'〃': 2,
+}
+
+
+//NOTE - this one only support posix newlines, this should be fixed later
+export function tabs_to_spaces(text, tab_width=4) {
+	let pending_column_index=0;
+
+	let result = '';
+	for (const char of text) {
+		if (char == '\t') {
+			const new_column_index = (Math.floor(pending_column_index / tab_width) + 1) * tab_width;
+			result += ' '.repeat(new_column_index - pending_column_index);
+			pending_column_index = new_column_index;
+		} else {
+			if (char.charCodeAt(0) === 10) {
+				result += '\n';
+				pending_column_index = 0;
+/*			} else if (char.charCodeAt(0) < 32) {
+				result += '<27>';
+*/			} else {
+				pending_column_index += (char_width_lut[char] ?? 1)
+				result += char;
+			}
+		}
+	}
+	return result;
+
+}

tools/show-annotations.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+#NOTE: This tool require you to install https://gitea.efforting.tech/mikael-lovqvist/shell-utils but you could remove the last line if you don't want that.
+#Example use: From the source-directory run ../tools/show-annotations.sh
+
+grep --color=always \
+	-rne '//[A-Z ]*:' \
+	--include "*.mjs" \
+	--exclude-dir node_modules \
+	| grep-mtime-sorter | ansi-trunc -l 250
+

tools/stage-for-npm.mjs

@@ -60,9 +60,6 @@ const common_package_data = {
 	author,
 	version,
 	type: 'module',
-	publishConfig: {
-		registry
-	},
 };
 
 const root_package = {
@@ -91,6 +88,9 @@ for (const [package_name, package_data] of Object.entries(manifest.packages)) {
 	const linked_docs = pkg.documentation ? link_tree(pkg.documentation, pkg_dir).map(p => path.relative(pkg_dir, p)) : [];
 	//console.log('DOCS', { linked_docs });
 
+
+	// A file named after its package (e.g. table.mjs in the table package) becomes the root export '.'
+	// All other files become named subpath exports e.g. './raster-table'
 	const exports_map = {};
 	for (const file of linked_sources) {
 		const name = path.basename(file, '.mjs');
@@ -127,9 +127,9 @@ for (const [package_name, package_data] of Object.entries(manifest.packages)) {
 
 
 //const publish_tool = 'pnpm publish --no-git-checks';
-const publish_tool = 'npm publish';
+const publish_tool = `npm publish --registry "${registry}"`;
 const publish_script_lines = published_packages.map(
-	pkg => `${publish_tool} ${pkg}`
+	pkg => `${publish_tool} ./${pkg}`
 );
 
 const dev_stage_script_lines = published_packages.map(
@@ -148,11 +148,15 @@ const publish_script = (
 	`${publish_script_lines.join('\n')}\n`
 );
 
+const repo_root = path.resolve(output_directory, '../..');
+const experiments_link_line = `mkdir -p "${repo_root}/experiments/node_modules"\nln -sf "../../build/packages" "${repo_root}/experiments/node_modules/@efforting.tech"`;
+
 const dev_stage_script = (
 	'#!/usr/bin/env bash\n' +
 	'set -e\n' +
 	`${dev_stage_mkdir_lines.join('\n')}\n` +
-	`${dev_stage_script_lines.join('\n')}\n`
+	`${dev_stage_script_lines.join('\n')}\n` +
+	experiments_link_line
 );