Worked on #3 where I believe the regexp part is now complete but it needs testing

experiments/generic-parser-2.mjs

@@ -0,0 +1,146 @@
+import { Row_Based_Table } from '@efforting.tech/table';
+import { load_raster_table } from '@efforting.tech/table/raster-table';
+import { RegExp_Tokenizer } from '@efforting.tech/parsing/regexp-dispatch';
+import { RegExp_Token_Parsing_Rule, Parser } from '@efforting.tech/parsing/generic-parsing';
+
+
+function load_table(raster) {
+	const table = load_raster_table(raster, Row_Based_Table);
+	table.replace_all_cells(({cell}) => cell.trim());
+	return table;
+}
+
+const logic_ops = load_table(`
+
+	name		symbol
+	----		------
+	AND			∧
+	OR			∨
+	XOR			⊕
+	NAND		↑
+	NOR			↓
+	XNOR		⊙
+	IMPLIES		→
+	IFF			↔
+	NOT			¬
+
+`);
+
+const generic_ops = load_table(`
+
+	name		symbol
+	----		------
+	PLUS		+
+	HYPHEN		-
+	DOT			·
+	ASTERISK	*
+	CROSS		×
+	SLASH		/
+	CARET		^
+	UNDERSCORE	_
+	PERCENT		%
+
+`);
+
+const punctuation = load_table(`
+
+	name		symbol
+	----		------
+	COMMA		,
+	SEMI_COLON	;
+	COLON		:
+	PERIOD		.
+
+`);
+
+const grouping = load_table(`
+
+	name					left	right
+	----					----	-----
+	PARENTESIS				(		)
+	SQUARE_BRACKET			[		]
+	CURLY_BRACE				{		}
+	ANGLE_BRACKET			⟨		⟩
+	DOUBLE_ARROW_BRACKET	«		»
+
+`);
+
+const greek_chars = load_table(`
+
+	name		lower	upper
+	----		-----	-----
+	ALPHA		α		Α
+	BETA		β		Β
+	GAMMA		γ		Γ
+	DELTA		δ		Δ
+	EPSILON		ε		Ε
+	ZETA		ζ		Ζ
+	ETA			η		Η
+	THETA		θ		Θ
+	IOTA		ι		Ι
+	KAPPA		κ		Κ
+	LAMBDA		λ		Λ
+	MU			μ		Μ
+	NU			ν		Ν
+	XI			ξ		Ξ
+	OMICRON		ο		Ο
+	PI			π		Π
+	RHO			ρ		Ρ
+	SIGMA		σ		Σ
+	TAU			τ		Τ
+	UPSILON		υ		Υ
+	PHI			φ		Φ
+	CHI			χ		Χ
+	PSI			ψ		Ψ
+	OMEGA		ω		Ω
+
+`);
+
+
+
+
+
+const rt = new RegExp_Tokenizer();
+
+for (const { name, left, right } of grouping.iter_objects()) {
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(left),
+		(tokenizer, ingress_match) => tokenizer.enter_sub_tokenizer(undefined,
+			(tokenizer, value, egress_match) => tokenizer.push_token(
+				{kind: 'EXPR', name, value, ingress_match, egress_match}
+			)
+		), `LEFT_${name}`
+	));
+
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(right),
+		(tokenizer, match) => tokenizer.leave_sub_tokenizer(match), `RIGHT_${name}`)
+	);
+}
+
+
+for (const table of [logic_ops, generic_ops, punctuation]) {
+	for (const { name, symbol } of table.iter_objects()) {
+		rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(symbol),
+			(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), name)
+		);
+	}
+}
+
+for (const { name, lower, upper } of greek_chars.iter_objects()) {
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(lower),
+		(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), `LOWER_${name}`)
+	);
+	rt.add_rules(new RegExp_Token_Parsing_Rule(RegExp.escape(upper),
+		(tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), `UPPER_${name}`)
+	);
+}
+
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\w+/, (tokenizer, match) => tokenizer.push_token({ kind: 'TOKEN', match }), 'WORD'));
+rt.add_rules(new RegExp_Token_Parsing_Rule(/\s+/, null, 'WHITESPACE'));
+
+
+const text = 'Hello World (how are you (doing)) I may ask';
+const p = new Parser(text, { tokenizer: rt });
+
+//console.log(rt.rules.at(-3));
+
+console.log(p.parse())

experiments/reduction-scanner-2.mjs

@@ -44,8 +44,8 @@ const rss = Reduction_Settings.load({
 	// Switching this on or off affects whether add comes before mul or not
 	//reduction_order: 'POSITION_MAJOR',
 });
-const rs = new Reduction_Scanner(rss);
 
+const rs = new Reduction_Scanner(rss);
 
 rss.rules.push(
 

experiments/reduction-scanner-3.mjs

@@ -0,0 +1,157 @@
+import { Reduction_Scanner, Reduction_Settings } from  '@efforting.tech/rule-processing/reduction-scanner';
+import * as R from  '@efforting.tech/rule-processing/rules';
+
+import { inspect } from 'node:util';
+
+
+class Rule_Match {
+	constructor(rule, match) {
+		Object.assign(this, { rule, match });
+	}
+
+	get action() {
+		return this.rule.handler;
+	}
+
+
+}
+
+
+class Rule {	//NOTE: This is somewhat of a place holder because we may want to declare specific transformations later rather than always having an opaque handler function
+	constructor(condition, handler) {
+		Object.assign(this, { condition, handler });
+	}
+
+	match(...args) {
+		const match = this.condition.match(...args);
+		if (match) {
+			return new Rule_Match(this, match);
+		}
+	}
+
+
+}
+
+class Sub_Scan_Rule_Match {
+	constructor(rule, sub_scan_candidate) {
+		Object.assign(this, { rule, sub_scan_candidate });
+	}
+
+	get action() {
+		return this.sub_scan_candidate.match.action;
+	}
+
+	get match() {
+		return this.sub_scan_candidate.match.match;
+	}
+
+}
+
+class Sub_Scan_Rule {
+	constructor(sub_system) {
+		Object.assign(this, { sub_system });
+	}
+
+	match(...args) {
+		const candidate = this.sub_system.find_reduction_candidate(...args);
+		if (candidate) {
+			return new Sub_Scan_Rule_Match(this, candidate)
+		}
+	}
+
+
+}
+
+
+
+function sequence_rule(sequence, transform_fn) {
+	return new Rule(
+		new R.Sequence_Condition(sequence),
+		({sequence, match}) => {
+			const MS = match.match_start;
+			const ME = match.match_end;
+			sequence.splice(MS, ME - MS + 1, transform_fn(...sequence.slice(MS, ME + 1)));
+		}
+	);
+}
+
+
+
+const N = new R.Predicate((i) => typeof i === 'number' || i.type == 'BINOP' );
+
+const CARET = new R.Strict_Equality('^');
+const CARON = new R.Strict_Equality('ˇ');
+
+const ASTERISK = new R.Strict_Equality('*');
+const SLASH = new R.Strict_Equality('/');
+
+const PLUS = new R.Strict_Equality('+');
+const MINUS = new R.Strict_Equality('-');
+
+// These are the outer settings
+const rss = Reduction_Settings.load();
+
+// These are the inner settings
+const rss_inner = Reduction_Settings.load({
+	reduction_order: 'POSITION_MAJOR',
+});
+
+const rs = new Reduction_Scanner(rss);
+
+// Local factory for sub system
+function sub_system(...rules) {
+	const sub_settings = { ...rss_inner, rules };
+	const scanner = new Reduction_Scanner(sub_settings);
+	return new Sub_Scan_Rule(scanner);
+}
+
+
+
+
+rss.rules.push(
+	sub_system(
+		sequence_rule([N, CARET, N], 		(left, op, right) => ({ type: 'BINOP', op: 'CARET', operands: [left, right]})),
+		sequence_rule([N, CARON, N], 		(left, op, right) => ({ type: 'BINOP', op: 'CARON', operands: [left, right]})),
+	),
+
+	sub_system(
+		sequence_rule([N, ASTERISK, N], (left, op, right) => ({ type: 'BINOP', op: 'ASTERISK', operands: [left, right]})),
+		sequence_rule([N, SLASH, N], (left, op, right) => ({ type: 'BINOP', op: 'SLASH', operands: [left, right]})),
+	),
+
+	sub_system(
+		sequence_rule([N, PLUS, N], 	(left, op, right) => ({ type: 'BINOP', op: 'PLUS', operands: [left, right]})),
+		sequence_rule([N, MINUS, N], 	(left, op, right) => ({ type: 'BINOP', op: 'MINUS', operands: [left, right]})),
+	),
+);
+
+
+const arr = [5, '-', 10, '^', 5, 'ˇ', 2, '+', 20, '*', 30];
+console.log(inspect(rs.transform(arr), { colors: true, depth: null }));
+
+
+/*
+
+[
+  {
+    type: 'BINOP',
+    op: 'MINUS',
+    operands: [
+      5,
+      {
+        type: 'BINOP',
+        op: 'PLUS',
+        operands: [
+          {
+            type: 'BINOP',
+            op: 'CARON',
+            operands: [ { type: 'BINOP', op: 'CARET', operands: [ 10, 5 ] }, 2 ]
+          },
+          { type: 'BINOP', op: 'ASTERISK', operands: [ 20, 30 ] }
+        ]
+      }
+    ]
+  }
+]
+
+*/

experiments/regexp-tokenizer.mjs

@@ -12,4 +12,15 @@ rt.set_default_identifier('random stuff');
 
 for (const m of rt.iter_matches('#Hello World!')) {
 	console.log({class: m.constructor.name, identifier: m.identifier, value: m.value, captured: m.captured });
-};
+};
+
+
+console.log('--=| Slicing |=--')
+
+for (const m of rt.iter_matches('#Hello World!', 3, -3)) {
+	//console.log(m, m.pending_index)
+	console.log({class: m.constructor.name, identifier: m.identifier, value: m.value, captured: m.captured });
+
+
+};
+

planning/math-subsystem.md

@@ -0,0 +1,79 @@
+# Mathematical Expression Subsystem
+
+> [!NOTE]
+> This document is written by Claude by Anthropic using Sonnet 4.6 and has yet to be vetted by Mikael Lövqvist
+
+## Overview
+
+A math-like expression language built on top of the reduction scanner, supporting
+operator notation, matrix literals, subscripts, superscripts, and symbolic operators.
+
+## Operator Notation
+
+Operators are identified by their symbol name rather than semantic meaning, since the
+same symbol can mean different things depending on operand types:
+
+- `*` (ASTERISK) — could be scalar multiplication, Hadamard product, or scale depending on types
+- `·` (DOT) — dot product
+- `×` (CROSS) — cross product
+- `⊕` (OPLUS) — direct sum or XOR
+
+Semantic resolution (e.g. `ASTERISK(matrix, matrix)` → Hadamard) is a separate
+type-inference pass, not part of the structural reduction.
+
+## ASCII Input for Special Symbols
+
+LaTeX-inspired escape sequences for entering special symbols in plain ASCII:
+
+- `\oplus` → ⊕
+- `\times` → ×
+- `\cdot` → ·
+- `\otimes` → ⊗
+
+`^` is reserved for superscript (not XOR), `_` for subscript. `S_12` reads as S₁₂.
+
+## Matrix Literals
+
+Single-line input using nested brackets:
+
+```
+[[1, 0, 0], [0, 1, 0], [0, 0, 1]]
+```
+
+Pretty-printed output using Unicode bracket characters:
+
+```
+⎡1 0 0⎤
+⎢0 1 0⎥
+⎣0 0 1⎦
+```
+
+## 2D Raster Reduction Scanner
+
+For parsing pretty-printed multi-line matrix literals within larger expressions like
+`M + 2 * N` where M and N are written in 2D notation, a raster-based reduction pass
+is needed before the standard 1D reduction pass.
+
+### Approach
+
+1. **Raster pass first** — operate on a 2D grid of characters
+2. Locate matrix corner anchors `⎡⎤⎣⎦` — these are highly selective so candidate
+   detection is cheap
+3. Scan right for `⎤`, down for `⎣`, verify `⎦` at intersection
+4. Use `⎢`/`⎥` to identify row boundaries within the region
+5. Collapse the identified rectangle into a single matrix token
+6. **1D pass second** — the surrounding expression now contains ordinary tokens and
+   the collapsed matrix nodes, reducible by standard rules
+
+### Scope Boundaries
+
+Fraction bars define containment — a matrix appearing in a numerator or denominator
+is only part of that sub-expression. The horizontal extent of the fraction bar bounds
+the operand scan. Containment must be resolved outside-in: find outermost structure
+first, recurse into sub-regions.
+
+### Generalization
+
+A 2D reduction scanner is a natural generalization of the 1D scanner — the "sequence"
+becomes a 2D array and conditions match spatial patterns rather than linear ones.
+The same anchor-point and backtracking concepts apply.

source/errors.mjs

@@ -6,7 +6,7 @@ import { inspect } from 'node:util';
 
 export class Tokenization_Error extends Error {
 	constructor(data) {
-		const { parser, value, index, end_index } = data;
+		const {parser, text, start_position, end_position, match_start, match_end, value} = data;
 		super(`Tokenization_Error`);	//TODO: Format message
 		this.data = data;
 	}

source/parsing/generic-parsing.mjs

@@ -83,14 +83,14 @@ export class Parser {
 		this.switch_to(tokenizer);
 	}
 
-	leave_sub_tokenizer() {
+	leave_sub_tokenizer(egress_match=null) {
 		const frame = this.stack.pop(true);
 		const { sub_tokenizer_handlers } = this.state;
 
 		if (sub_tokenizer_handlers.length) {
 			const handler = sub_tokenizer_handlers.pop();
 			this.state.match = null;	//TODO: Decide if we should reset match here or not
-			handler(this, frame.value);
+			handler(this, frame.value, egress_match);
 		} else {
 			this.push_token(frame.value);
 		}

source/parsing/regexp-dispatch.mjs

@@ -7,13 +7,24 @@ import { Tokenization_Error } from '@efforting.tech/errors';
 //
 //			Specifically it is not currently decided where the boundary between rule/action/capture should be
 
+
+function normalize_bounds(text, start_position, end_position) {
+	const len = text.length;
+	const norm_start = start_position < 0 ? Math.max(0, len + start_position) : start_position;
+	const norm_end = end_position == undefined ? undefined : (end_position < 0 ? Math.max(0, len + end_position) : end_position);
+	return [norm_start, norm_end];
+}
+
+
 export class Pattern_Match {
-	constructor(match, rule) {
-		Object.assign(this, { match, rule });
+	constructor(text, start_position, end_position, match, rule) {
+		// Normalize positions
+		[start_position, end_position] = normalize_bounds(text, start_position, end_position);
+		Object.assign(this, { text, start_position, end_position, match, rule });
 	}
 
 	get identifier() {
-		return this.rule. identifier;
+		return this.rule.identifier;
 	}
 
 	get value() {
@@ -24,28 +35,34 @@ export class Pattern_Match {
 		return this.match.slice(1);
 	}
 
-	get pending_index() {
-		return this.match.index + this.match[0].length;
+	get absolute_start() {
+		return this.match.index + this.start_position;
 	}
 
+	get absolute_end() {
+		return this.match.index + this.start_position + this.match[0].length - 1;
+	}
+
+	get pending_index() {
+		return this.match.index + this.start_position + this.match[0].length;
+	}
 }
 
 export class Default_Match {
-	constructor(text, index, end_index, action) {
-		const identifier = action(this);
-		Object.assign(this, { text, index, end_index, action, identifier });
-	}
+	//TBD: Here we invoke action while creating this object, and assign the identifier but we don't do that on Pattern_Match - this feels a bit sketchy
+	constructor(text, start_position, end_position, match_start, match_end, value, action) {
+		// Normalize positions
+		[start_position, end_position] = normalize_bounds(text, start_position, end_position);
+		[match_start, match_end] = normalize_bounds(text, match_start, match_end);
 
-	get value() {
-		return this.text;
+		const identifier = action(this);	//TODO: action protocol in accordance with issue #5
+		Object.assign(this, { text, start_position, end_position, match_start, match_end, value, action, identifier });
 	}
 
-
 	get pending_index() {
-		if (this.end_index === null) {
-			return null;
-		} else {
-			return this.end_index;
+		// CLARIFICATION: loose inequality ( != ) matches null and undefined but not false/0/'' but we use strict for start_position since it is always a number
+		if (this.match_end != undefined) {
+			return this.match_end + 1;
 		}
 	}
 }
@@ -76,8 +93,6 @@ export class RegExp_Token_Rule extends Abstract_RegExp_Token_Rule {
 	}
 }
 
-// Note: There is no clean built in way to set an end position of a RegExp pattern, the only generic way is to slice the string we match before.
-//		 We may at some point implement support for this (and it would only be done if end position was given)
 
 export class RegExp_Tokenizer {
 	constructor(rules=[], default_action=undefined) {
@@ -94,53 +109,58 @@ export class RegExp_Tokenizer {
 		this.rules.push(...rules_to_add);
 	}
 
-	immediate_match(text, position=0) {
+	immediate_match(text, start_position=0, end_position=undefined) {
+		// CLARIFICATION: loose inequality ( != ) matches null and undefined but not false/0/'' but we use strict for start_position since it is always a number
+		const bounded = start_position !== 0 || end_position != undefined;
+		const text_to_search = bounded ? text.slice(start_position, end_position) : text;
+
 		for (const rule of this.rules) {
 			const pattern = rule.immediate_pattern;
-			pattern.lastIndex = position;
-			const match = pattern.exec(text);
+			pattern.lastIndex = 0;
+			const match = pattern.exec(text_to_search);
 			if (match) {
-				return new Pattern_Match(match, rule);
+				return new Pattern_Match(text, start_position, end_position, match, rule);
 			}
 		}
 	}
 
 
-	_handle_default_match(value, index, end_index=null) {
+	_handle_default_match(text, start_position, end_position, match_start, match_end, value) {
 		const { default_action } = this;
 		if (!default_action) {
-			throw new Tokenization_Error({ parser: this, value, index, end_index });
+			throw new Tokenization_Error({ parser: this, text, start_position, end_position, match_start, match_end, value });
 		}
-		return new Default_Match(value, index, end_index, default_action);
+		return new Default_Match(text, start_position, end_position, match_start, match_end, value, default_action);
 	}
 
 
-	closest_scanning_match(text, position=0) {
-		const immediate_match = this.immediate_match(text, position);
+	closest_scanning_match(text, start_position=0, end_position=undefined) {
+
+		const immediate_match = this.immediate_match(text, start_position, end_position);
 		if (immediate_match) {
 			return immediate_match;
 		}
 
 		let best_candidate;
-		for (const candidate of this.iter_scanning_rule_candidates(text, position)) {
-			if ((best_candidate === undefined) || (best_candidate.match.index > candidate.match.index)) {
+		for (const candidate of this.iter_scanning_rule_candidates(text, start_position, end_position)) {
+			if ((best_candidate === undefined) || (best_candidate.absolute_start > candidate.absolute_start)) {
 				best_candidate = candidate;
 			}
 		}
 
 		// There was no match, just get the tail
 		if (!best_candidate) {
-			const tail = text.slice(position);
+			const tail = text.slice(start_position);
 			if (tail.length) {
-				return this._handle_default_match(tail, position);
+				return this._handle_default_match(text, start_position, end_position, start_position, end_position, tail);
 			}
 		}
 
 		// There was a match, check the head
 		if (best_candidate) {
-			const head = text.slice(position, best_candidate.match.index);
+			const head = text.slice(start_position, best_candidate.absolute_start);
 			if (head.length) {
-				return this._handle_default_match(head, position, best_candidate.match.index);
+				return this._handle_default_match(text, start_position, end_position, start_position, best_candidate.absolute_start - 1, head);
 			}
 		}
 
@@ -149,32 +169,41 @@ export class RegExp_Tokenizer {
 	}
 
 
-	*iter_scanning_rule_candidates(text, position=0) {
-		// Iterates over all rules and yields any matches found anywhere (but only once per rule)
+	*iter_scanning_rule_candidates(text, start_position=0, end_position=undefined) {
+		// CLARIFICATION: loose inequality ( != ) matches null and undefined but not false/0/'' but we use strict for start_position since it is always a number
+		const bounded = start_position !== 0 || end_position != undefined;
+		const text_to_search = bounded ? text.slice(start_position, end_position) : text;
 
+
+		// Iterates over all rules and yields any matches found anywhere (but only once per rule)
 		for (const rule of this.rules) {
 			const pattern = rule.scanning_pattern;
-			pattern.lastIndex = position;
-			const match = pattern.exec(text);
+			pattern.lastIndex = 0;
+			const match = pattern.exec(text_to_search);
 
 			if (match) {
-				yield new Pattern_Match(match, rule);
+				yield new Pattern_Match(text, start_position, end_position, match, rule);
 			}
 		}
 
 	}
 
 
-	*iter_matches(text, position=0) {
+	*iter_matches(text, start_position=0, end_position=undefined) {
+
+		// Normalize positions
+		[start_position, end_position] = normalize_bounds(text, start_position, end_position);
+
 		while (true) {
-			const pending = this.closest_scanning_match(text, position);
+			const pending = this.closest_scanning_match(text, start_position, end_position);
 			if (pending) {
 				yield pending;
 			}
-			if (!pending || pending.pending_index === null) {
+			// CLARIFICATION: loose equality ( == ) matches null and undefined but not false/0/'' but we use strict for start_position since it is always a number
+			if (!pending || pending.pending_index == null || pending.pending_index === end_position ) {
 				break;
 			}
-			position = pending.pending_index;
+			start_position = pending.pending_index;
 
 		}
 	}

source/rule-processing/reduction-scanner.mjs

@@ -23,6 +23,8 @@ export const FP_Reduction_Settings = new CF.Schema({
 
 
 
+//TODO - we should probably have a pre-defined record shape as argument for actions and such rather than using an ever growing list of positionals or an anonymous Object()
+
 export class Reduction_Scanner {
 
 	static settings_schema = Reduction_Settings;
@@ -36,7 +38,7 @@ export class Reduction_Scanner {
 		}
 	}
 
-	perform_reduction(sequence) {
+	find_reduction_candidate(sequence) {
 		const { settings } = this;
 		switch (settings.reduction_order) {
 
@@ -44,12 +46,10 @@ export class Reduction_Scanner {
 				for (const rule of settings.rules) {
 					const match = rule.match(sequence);
 					if (match) {
-						//console.log('RULE_MAJOR', match);
-						rule.action(this, sequence, match);	//TODO: should rule.action be able to add additional checks? Though that probably dillutes responsibility and blurs interfaces
-						return true;
+						return { sequence, rule, match };
 					}
 				}
-				return false;
+				return;
 
 			case Reduction_Order.symbols.POSITION_MAJOR:
 
@@ -68,18 +68,28 @@ export class Reduction_Scanner {
 				}
 
 				if (best_match) {
-					//console.log('POSITION_MAJOR', best_match)
-					best_rule.action(this, sequence, best_match);	//TODO: should rule.action be able to add additional checks? Though that probably dillutes responsibility and blurs interfaces
-					return true;
+					return { sequence, rule: best_rule, match: best_match };
 				}
 
 
-				return false;
+				return;
 			default:
 				throw new Error(`Unknown reduction order: ${this.reduction_order}`);	//TODO: Force invalid configuration  error
 
 		}
+	}
 
+
+	perform_reduction(sequence) {
+		const candidate = this.find_reduction_candidate(sequence);
+		if (candidate) {
+			const { sequence, rule, match } = candidate;
+			//console.log('ACT', match.match)
+			match.action({ reduction_system: this, rule, sequence, match: match.match });
+			return true;
+		} else {
+			return false;
+		}
 	}
 
 	clear_transform_state() {

source/table/table.mjs

@@ -146,12 +146,20 @@ export class Row_Based_Table {
 	}
 
 
+	//TODO: Implement map and other functions expected by collections
+
 	*[Symbol.iterator]() {
 		for (const index of this.rows.keys()) {
 			yield new Table_Row_Reference(this, index);
 		}
 	}
 
+	*iter_objects() {
+		for (const index of this.rows.keys()) {
+			yield new Table_Row_Reference(this, index).object;
+		}
+	}
+
 
 }