Initial commit

data.mjs

@@ -0,0 +1,168 @@
+import fs from 'node:fs';
+
+export function load_ndjson(filePath, show_errors=true) {
+	return fs.readFileSync(filePath, 'utf8').split(/\n/).filter(Boolean).map(line => {
+		try {
+			return JSON.parse(line);
+		} catch (e) {
+			if (show_errors) {
+				console.warn({ error: e, line });
+			}
+		}
+	}).filter(Boolean);
+}
+
+const data = load_ndjson('/srv/project-fs.log', false);
+
+
+
+class Data_Range {
+	constructor(data) {
+		const min = data.at(0);
+		const max = data.at(-1);
+		const median = data.at(Math.floor(data.length / 2));
+		Object.assign(this, { data, min, max, median });
+	}
+
+	get length() {
+		return this.data.length;
+	}
+
+	median_split() {
+		const first_half = [];
+		const second_half = [];
+		const median = this.median[0];
+
+		for (const entry of this.data) {
+			const position = entry[0];
+			if (position <= median) {
+				first_half.push(entry);
+			} else {
+				second_half.push(entry);
+			}
+		}
+
+		return [
+			new Data_Range(first_half),
+			new Data_Range(second_half),
+		];
+
+	}
+
+}
+
+
+
+function predicate_split(range, should_split) {
+
+	if (should_split(range)) {
+		const [left, right] = range.median_split();
+		if ((left.length > 0) && (right.length > 0)) {
+			return [...predicate_split(left, should_split), ...predicate_split(right, should_split)];
+		}
+	}
+	return [range];
+}
+
+const all_data = new Data_Range(data.map(e => [e.ts[0], e]));
+
+export const ranges = predicate_split(new Data_Range(data.map(e => [e.ts[0], e])), r => r.length > 10_000);
+
+export function* entries_after(ranges, query_position) {
+	const range_iter = ranges[Symbol.iterator]();
+
+	for (const range of range_iter) {
+
+		if (range.max[0] >= query_position) {
+			for (const entry of range.data) {
+				const pos = entry[0];
+				if (pos >= query_position) {
+					yield entry;
+				}
+			}
+
+			for (const range of range_iter) {
+				yield* range.data;
+			}
+
+			return;
+		}
+	}
+}
+
+
+const point_in_time = Math.floor(Date.now() / 1000) - 30 * 86400;	// 30 days ago
+
+const events = new Data_Range([...entries_after(ranges, point_in_time)]);
+
+
+class Counter extends Map {
+	increase(key, delta = 1) {
+		this.set(key, (this.get(key) ?? 0) + delta);
+	}
+}
+
+class Counter_Bin_Set extends Map {
+	increase(slot, value, delta = 1) {
+		const existing = this.get(slot);
+		if (existing) {
+			existing.increase(value, delta);
+		} else {
+			const new_counter = new Counter();
+			new_counter.increase(value, delta);
+			this.set(slot, new_counter);
+		}
+	}
+}
+
+
+
+const activity = new Counter_Bin_Set();
+
+
+
+function pos_to_slot(pos) {
+	return Math.floor(pos / 86400) - Math.floor(events.min[0] / 86400);	// Per day
+}
+
+function slot_to_pos(slot) {
+	return (slot + Math.floor(events.min[0] / 86400)) * 86400;	// Per day
+}
+
+const ignore_patterns = [
+	/\/\.git(?:\/.*)?/g,
+	/\.tmp$/ig,
+];
+
+for (const [pos, value] of events.data) {
+
+	for (const name of [value.name, value.old, value.new]) {
+		if (name) {
+			let ignore = false;
+			for (const pattern of ignore_patterns) {
+				if (name.match(pattern)) {
+					ignore = true;
+					break;
+				}
+			}
+			if (!ignore) {
+				activity.increase(pos_to_slot(pos), name);
+			}
+		}
+	}
+
+}
+
+function epoch_s_to_date_time(epoch) {
+	return new Date(epoch*1000).toISOString().slice(0,16).replace('T',' ');
+}
+
+function epoch_s_to_date(epoch) {
+	return new Date(epoch*1000).toISOString().split('T')[0];
+}
+
+
+
+for (const [slot, entry] of activity.entries()) {
+	console.log(epoch_s_to_date(slot_to_pos(slot)), entry.size);
+}