From 957ffc093226cfc013915f464271ee8d90a842f7 Mon Sep 17 00:00:00 2001 From: mikael-lovqvists-claude-agent Date: Sat, 23 May 2026 23:32:01 +0000 Subject: [PATCH] Add ci directory with architecture plan and experiment 01 (rsync publish) --- ci/.gitignore | 1 + ci/experiments/01-rsync-publish/Dockerfile | 3 + .../01-rsync-publish/Dockerfile.checkout | 11 + ci/experiments/01-rsync-publish/config.yml | 27 ++ ci/experiments/01-rsync-publish/index.mjs | 211 +++++++++++++++ ci/experiments/01-rsync-publish/readme.md | 31 +++ .../steps/01-generate-key.mjs | 76 ++++++ ci/package-lock.json | 33 +++ ci/package.json | 9 + ci/planning/architecture.md | 244 ++++++++++++++++++ 10 files changed, 646 insertions(+) create mode 100644 ci/.gitignore create mode 100644 ci/experiments/01-rsync-publish/Dockerfile create mode 100644 ci/experiments/01-rsync-publish/Dockerfile.checkout create mode 100644 ci/experiments/01-rsync-publish/config.yml create mode 100644 ci/experiments/01-rsync-publish/index.mjs create mode 100644 ci/experiments/01-rsync-publish/readme.md create mode 100644 ci/experiments/01-rsync-publish/steps/01-generate-key.mjs create mode 100644 ci/package-lock.json create mode 100644 ci/package.json create mode 100644 ci/planning/architecture.md diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/ci/experiments/01-rsync-publish/Dockerfile b/ci/experiments/01-rsync-publish/Dockerfile new file mode 100644 index 0000000..5deb9eb --- /dev/null +++ b/ci/experiments/01-rsync-publish/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.21 +RUN apk add --no-cache rsync +CMD ["rsync", "-aP", "--delete", "--verbose", "/src/", "/deploy/"] diff --git a/ci/experiments/01-rsync-publish/Dockerfile.checkout b/ci/experiments/01-rsync-publish/Dockerfile.checkout new file mode 100644 index 0000000..7d50989 --- /dev/null +++ b/ci/experiments/01-rsync-publish/Dockerfile.checkout @@ -0,0 +1,11 @@ +FROM alpine:3.21 +RUN apk add --no-cache git openssh-client + +# Expected env vars: +# REPO_URL — git URL to clone +# REVISION — branch, tag, or full sha to check out +# +# The base worktree volume is mounted at /worktree (writable during this step only). + +WORKDIR /worktree +CMD git clone "$REPO_URL" . && git checkout "$REVISION" diff --git a/ci/experiments/01-rsync-publish/config.yml b/ci/experiments/01-rsync-publish/config.yml new file mode 100644 index 0000000..67502e8 --- /dev/null +++ b/ci/experiments/01-rsync-publish/config.yml @@ -0,0 +1,27 @@ +# Experiment 01 — rsync publish configuration + +# Host directory where secrets are stored +secrets_base: /srv/ci-secrets + +# Gitea API base URL — used to verify literal commit shas +gitea_api_base: https://gitea.efforting.tech/api/v1 + +# CI user — scopes secrets and worktree volumes +user: mikael-lovqvist + +# Checkout container — clones one or more repositories +checkout: + repositories: + - repo_url: git@git.efforting.tech:mikael-lovqvist/websperiments.git + revision: main + mutable: true # true = resolve via ls-remote on each run; false = treat revision as a fixed sha + key: websperiments-deploy # references git-ssh//private/ + submodules: false + +# Task container — runs the actual job +task: + image: ci-rsync + # Secrets to mount into the task container (empty for this task — + # rsync deploys to a local bind mount and needs no credentials) + secrets: [] + deploy_target: /srv/sites.efforting.tech diff --git a/ci/experiments/01-rsync-publish/index.mjs b/ci/experiments/01-rsync-publish/index.mjs new file mode 100644 index 0000000..33e89ed --- /dev/null +++ b/ci/experiments/01-rsync-publish/index.mjs @@ -0,0 +1,211 @@ +/** + * Experiment 01 — rsync publish + * + * Walks through the basic lifecycle: + * 1. Prepare a base worktree Docker volume for (owner, repo, revision) + * — runs a checkout container if the volume doesn't exist yet + * 2. Run the rsync container with the worktree volume mounted read-only + * and the deploy target bind-mounted writable + * + * Everything is explicit and manual here — no framework, no abstraction. + * The goal is to understand the mechanics before building on top of them. + */ + +import { execFile as exec_file } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { createHash } from 'node:crypto' +import * as path from 'node:path' +import { promisify } from 'node:util' +import { fileURLToPath } from 'node:url' +import yaml from 'js-yaml' + +const exec = promisify(exec_file) +const HERE = path.dirname(fileURLToPath(import.meta.url)) + +// --------------------------------------------------------------------------- +// Config — loaded from config.yml next to this file +// --------------------------------------------------------------------------- + +const CONFIG = yaml.load(readFileSync(path.join(HERE, 'config.yml'), 'utf8')) + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function run(cmd, args = [], opts = {}) { + console.log(`$ ${cmd} ${args.join(' ')}`) + const { stdout, stderr } = await exec(cmd, args, opts) + if (stdout) { + process.stdout.write(stdout) + } + if (stderr) { + process.stderr.write(stderr) + } + return stdout.trim() +} + +async function docker(...args) { + return run('docker', args) +} + +// Run docker compose with a generated config passed via stdin (-f -) +async function compose(compose_config, args) { + const compose_yaml = yaml.dump(compose_config) + console.log(`$ docker compose -f - ${args.join(' ')}`) + + return new Promise((resolve, reject) => { + const child = exec_file('docker', ['compose', '-f', '-', ...args]) + + child.stdin.write(compose_yaml) + child.stdin.end() + + let stdout = '' + let stderr = '' + child.stdout.on('data', chunk => { process.stdout.write(chunk); stdout += chunk }) + child.stderr.on('data', chunk => { process.stderr.write(chunk); stderr += chunk }) + + child.on('close', code => { + if (code === 0) { + resolve(stdout.trim()) + } else { + reject(new Error(`docker compose exited with code ${code}\n${stderr}`)) + } + }) + }) +} + +// Hash (owner, repo_url, revision) to get an unambiguous volume name. +// Human-readable metadata is stored as Docker volume labels instead. +function worktree_volume_name(owner, repo_url, revision) { + const hash = createHash('sha256') + .update(owner).update('\0') + .update(repo_url).update('\0') + .update(revision) + .digest('hex') + .slice(0, 24) + return `ci-worktree-${hash}` +} + +// --------------------------------------------------------------------------- +// Step 1: Prepare base worktree volume +// --------------------------------------------------------------------------- + +async function prepare_worktree(owner, repo_url, revision, mutable, private_key_path) { + const volume_name = worktree_volume_name(owner, repo_url, revision) + + // Check if the volume already exists + let exists = false + try { + await docker('volume', 'inspect', volume_name) + exists = true + } catch { + exists = false + } + + if (exists && !mutable) { + // Immutable revision — volume is permanent once created + console.log(`Base worktree volume ${volume_name} already exists, reusing.`) + return volume_name + } + + if (!exists) { + await docker('volume', 'create', + '--label', `ci.owner=${owner}`, + '--label', `ci.repo_url=${repo_url}`, + '--label', `ci.revision=${revision}`, + '--label', `ci.mutable=${mutable}`, + volume_name, + ) + } + + // Mutable: always re-checkout (ref may have moved) + // Immutable + not exists: first checkout + console.log(`${mutable ? 'Re-checking out mutable' : 'Checking out immutable'} revision "${revision}"`) + await docker( + 'run', '--rm', + '--volume', `${volume_name}:/worktree`, + '--volume', `${private_key_path}:/git_identity:ro`, + '--network', 'host', + 'ci-checkout', + 'git', '-c', 'core.sshCommand=ssh -i /git_identity -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null', + 'clone', repo_url, '--branch', revision, '.', '/worktree', + ) + + return volume_name +} + +// --------------------------------------------------------------------------- +// Compose config builders +// --------------------------------------------------------------------------- + +function build_rsync_compose_config({ container_name, worktree_volume, deploy_target }) { + return { + name: container_name, + services: { + rsync: { + container_name, + build: { + context: HERE, + dockerfile: 'Dockerfile', + }, + image: 'ci-rsync', + volumes: [ + `${worktree_volume}:/src:ro`, + `${deploy_target}:/deploy`, + ], + restart: 'no', + network_mode: 'none', + environment: { + TERM: 'xterm-256color', + FORCE_COLOR: '1', + CLICOLOR_FORCE: '1', + }, + }, + }, + volumes: { + [worktree_volume]: { external: true }, + }, + } +} + +// --------------------------------------------------------------------------- +// Step 2: Run rsync container +// --------------------------------------------------------------------------- + +async function run_rsync(compose_config) { + // No --detach — blocks until container exits, streaming output + // Stopped container is retained for inspection or restart + await compose(compose_config, ['up', '--build']) +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log('=== Experiment 01: rsync publish ===\n') + + const { secrets_base, user, checkout, task } = CONFIG + const repo = checkout.repositories[0] + const private_key_path = path.join(secrets_base, 'git-ssh', user, 'private', repo.key) + + const worktree_volume = await prepare_worktree(user, repo.repo_url, repo.revision, repo.mutable, private_key_path) + console.log(`\nWorktree volume: ${worktree_volume}\n`) + + const container_name = `ci-rsync-${worktree_volume.slice('ci-worktree-'.length)}` + + const rsync_config = build_rsync_compose_config({ + container_name, + worktree_volume, + deploy_target: task.deploy_target, + }) + + await run_rsync(rsync_config) + + console.log('\n=== Done ===') +} + +main().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/ci/experiments/01-rsync-publish/readme.md b/ci/experiments/01-rsync-publish/readme.md new file mode 100644 index 0000000..cd86a6c --- /dev/null +++ b/ci/experiments/01-rsync-publish/readme.md @@ -0,0 +1,31 @@ +# Experiment 01 — rsync publish + +## Goal + +Demonstrate the basic worktree-to-deployment pipeline using a minimal container (rsync only). This is the simplest possible end-to-end path: check out a git revision, mount it into a container, rsync the output to a bind-mounted deployment target. + +## What this experiment covers + +- Checking out a specific git revision into a worktree layer +- Starting a Docker container with: + - The worktree bind-mounted read-only as the source + - The deployment target bind-mounted writable as the destination +- Running rsync inside the container to publish +- Verifying the result on the host + +## What this experiment does NOT cover + +- Overlayfs (the worktree is a plain bind mount here — overlay comes later) +- Caches +- Container reuse / stop-start lifecycle +- Network isolation + +## Deployment target + +`sites.efforting.tech` — details TBD (path on host, permissions, ownership). + +## Open questions for this experiment + +- What user does the container run as, and how does it get write access to the deployment target? +- Do we rsync into a staging path and then atomically swap, or write directly? +- What is the exact rsync flags we want (delete extraneous files? preserve timestamps?)? diff --git a/ci/experiments/01-rsync-publish/steps/01-generate-key.mjs b/ci/experiments/01-rsync-publish/steps/01-generate-key.mjs new file mode 100644 index 0000000..4495069 --- /dev/null +++ b/ci/experiments/01-rsync-publish/steps/01-generate-key.mjs @@ -0,0 +1,76 @@ +/** + * Step 01 — Generate a named SSH key pair + * + * Generates an ed25519 key pair and stores it in the secrets directory: + * + * /git-ssh//public/ + * /git-ssh//private/ + * + * The private key is written with mode 0600. + * The public key can be registered as a read-only deploy key in Gitea. + */ + +import { execFile as exec_file } from 'node:child_process' +import { mkdirSync, mkdtempSync, chmodSync, existsSync, readFileSync, renameSync, rmSync } from 'node:fs' +import * as path from 'node:path' +import * as os from 'node:os' +import { promisify } from 'node:util' +import { fileURLToPath } from 'node:url' +import yaml from 'js-yaml' + +const exec = promisify(exec_file) +const HERE = path.dirname(fileURLToPath(import.meta.url)) +const CONFIG = yaml.load(readFileSync(path.join(HERE, '..', 'config.yml'), 'utf8')) + +async function run(cmd, args = []) { + console.log(`$ ${cmd} ${args.join(' ')}`) + const { stdout, stderr } = await exec(cmd, args) + if (stdout) { process.stdout.write(stdout) } + if (stderr) { process.stderr.write(stderr) } + return stdout.trim() +} + +async function main() { + const { secrets_base, user, key_name } = CONFIG + + const public_dir = path.join(secrets_base, 'git-ssh', user, 'public') + const private_dir = path.join(secrets_base, 'git-ssh', user, 'private') + const public_path = path.join(public_dir, key_name) + const private_path = path.join(private_dir, key_name) + + if (existsSync(private_path) || existsSync(public_path)) { + console.error(`Key "${key_name}" already exists for user "${user}". Aborting.`) + process.exit(1) + } + + mkdirSync(public_dir, { recursive: true }) + mkdirSync(private_dir, { recursive: true, mode: 0o700 }) + + // Generate into a secure temp directory, then move into place + const tmp_dir = mkdtempSync(path.join(os.tmpdir(), 'ci-keygen-')) + const tmp_path = path.join(tmp_dir, 'key') + + try { + await run('ssh-keygen', [ + '-t', 'ed25519', + '-N', '', // no passphrase + '-C', `ci/${user}/${key_name}`, + '-f', tmp_path, + ]) + + renameSync(`${tmp_path}.pub`, public_path) + renameSync(tmp_path, private_path) + chmodSync(private_path, 0o600) + } finally { + rmSync(tmp_dir, { recursive: true, force: true }) + } + + console.log(`\nKey "${key_name}" generated for user "${user}".`) + console.log(`\nPublic key (add as read-only deploy key in Gitea):\n`) + console.log(readFileSync(public_path, 'utf8')) +} + +main().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/ci/package-lock.json b/ci/package-lock.json new file mode 100644 index 0000000..a86c821 --- /dev/null +++ b/ci/package-lock.json @@ -0,0 +1,33 @@ +{ + "name": "efforting-ci", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "efforting-ci", + "version": "0.1.0", + "dependencies": { + "js-yaml": "^4.1.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + } + } +} diff --git a/ci/package.json b/ci/package.json new file mode 100644 index 0000000..4179c25 --- /dev/null +++ b/ci/package.json @@ -0,0 +1,9 @@ +{ + "name": "efforting-ci", + "version": "0.1.0", + "type": "module", + "private": true, + "dependencies": { + "js-yaml": "^4.1.0" + } +} diff --git a/ci/planning/architecture.md b/ci/planning/architecture.md new file mode 100644 index 0000000..ad4290c --- /dev/null +++ b/ci/planning/architecture.md @@ -0,0 +1,244 @@ +# CI Architecture + +This document captures architectural decisions made for the efforting.tech CI system and flags what is still open. + +--- + +## Core Concepts + +### Image declarations vs task declarations + +Images and tasks are declared separately. A task references an image by name. Multiple tasks may reference the same image — the image is built once and reused, rather than each task re-specifying its full setup. + +**Status: decided.** + +--- + +### Worktree layers (overlayfs) + +Each run mounts a single worktree into the container, but this is composed from two separately managed Docker volumes: + +**Base worktree** — a named Docker volume containing a plain `git checkout` of a specific `(owner, repo, revision)`. Read-only from the task container's perspective. Whether it is reused or re-checked-out on each run depends on whether the revision is mutable (see below). + +**Worktree mutations** — a named Docker volume holding the overlayfs upper layer for a specific run. Unique per run. Contains only what the container wrote or modified — the base worktree is never touched. + +From inside the container these appear as a single mount point (the overlayfs merged view). From outside, the CI system names and manages them separately. + +Using named Docker volumes means the checkout is a straightforward `git clone` or `git checkout` into a volume — no need for the `--git-dir` / `--work-tree` / `GIT_INDEX_FILE` technique. That approach remains useful for direct host-based deployments but is out of scope here. + +#### Two-step sequence + +The worktree is prepared in two distinct container runs: + +1. **Checkout container** — mounts the base worktree volume as writable. User credentials are injected here. Runs `git clone` / `git checkout` for the target revision, then exits. Keeps git operations and credential handling sandboxed away from the host. + +2. **Task container** — mounts base worktree read-only (overlayfs lower) and mutations volume writable (overlayfs upper). Runs the actual CI job. No credentials present. + +The base worktree volume is writable only during step 1. For all subsequent runs it is mounted `:ro` — the kernel enforces this. Any writes from the task container go to the mutations volume via overlayfs and cannot affect the base worktree. + +#### Mutable vs immutable revisions + +Each repository entry in a task declaration is flagged as mutable or immutable: + +- **Immutable** (`mutable: false`) — the revision is a fixed sha. The checkout container runs once; on subsequent runs the existing volume is reused as-is. +- **Mutable** (`mutable: true`) — the revision is a ref name (branch, tag). The checkout container runs on every run to pick up any changes. The user is responsible for asserting correctness — the system trusts the flag. + +There is no automatic resolution of ref names to shas. Attempting to detect whether a string is a sha or a ref name without querying the remote is unreliable and a potential security issue, and querying the remote for sha verification requires credentials and a round trip. The mutable flag keeps this explicit and honest. + +> [!NOTE] +> The checkout container should restore file mtimes from git history after cloning (e.g. via `git restore-mtime`). This matters for deployment chains where downstream steps may rely on mtimes to detect what changed. + +#### Credentials + +Each `(owner, repo)` registration generates a unique ed25519 SSH key pair. The private key is stored in the CI secrets store scoped to that registration. The public key is presented to the user to add as a read-only deploy key on the repo in Gitea. + +At checkout time the private key for that specific repo is mounted into the checkout container — no other keys are present. A compromised container cannot reach any other repo. + +Initially the public key is registered manually by the user. Automating this via the Gitea API (`POST /repos/{owner}/{repo}/keys`) is a future improvement. + +Keys are a first-class resource managed independently from tasks. The user creates and names them via the CI management dashboard, views the public half to register it in Gitea, and deletes them when no longer needed (with a warning if any tasks still reference the key). The private half never leaves the secrets store. + +The CI system enforces no scoping policy — the user decides how broadly or narrowly to scope each key. They may use one key per repo, one key for all their repos, or any grouping that makes sense to them. Tasks reference a key by ID. + +**Status: decided. Secrets store layout decided (see below). Automated key registration via Gitea API is open.** + +#### Service user + +All CI host operations (key generation, secrets storage, mounting volumes into containers) run as a dedicated service user — not a human account, no login shell. This user owns `secrets_base` and the CI server process runs under it. + +Initially the service user is created manually during bring-up. A future Debian package should create it automatically via `adduser --system` if it does not already exist, following standard Debian packaging conventions. The package should also ensure ownership of `secrets_base` is set correctly on install. + +The CI server process running as this user has access to all secrets in `secrets_base`. The checkout container always gets the deploy key for its repo mounted for the duration of the clone only. Whether secrets are also mounted into the task container is a per-task configuration decision — some tasks need no secrets (e.g. rsync to a local bind mount), others require them (e.g. SSH deploy to a remote server, pushing to a registry, publishing a package). A secrets broker would just move the same level of trust up one hop without meaningfully changing what gets mounted where. + +**Status: manual setup for now. Debian packaging is a future goal.** + +--- + +#### Secrets store layout + +``` +/ + git-ssh/ + / + public/ + (public key — safe to display) + private/ + (private key — mode 0600, never leaves the store) +``` + +`secrets_base` is a host directory (e.g. `/srv/ci-secrets`). The `git-ssh/` scope leaves room for future secret types without polluting the root. Key names are chosen by the user and have no enforced format. + +The private key is mounted into the checkout container for the duration of the clone and unmounted when the container exits. + +--- + +#### Access control and worktree scoping + +Base worktrees are scoped to `(owner, repo, revision)` — not just `(repo, revision)`. Sharing across owners would allow one user's job to read a repo checked out under another user's credentials. Even for public repos this special case is not worth the complexity. + +Checkout is performed using credentials of the job owner (or a member of their org). If access is revoked, the owner's base worktree cache is invalidated and future runs re-checkout with fresh credential verification. Runs already in progress are unaffected — mid-run revocation is accepted as a limitation. + +#### Cleanup + +Cleanup of base worktrees and worktree mutations is deferred. It may involve both manual and automatic steps. No eviction policy is defined yet. + +**Status: decided. Implementation details (how overlayfs upper layer is composed with the base worktree volume inside Docker) are still open.** + +--- + +### Container reuse (stop/start vs recreate) + +Containers are not destroyed between runs unless explicitly evicted. A stopped container retains its internal state. This enables: + +- **Debugging**: exec into a stopped container, patch something, restart without rebuilding. +- **Multi-image pipelines**: checkpoint after stage A, retry stage B without re-running A. +- **Faster iteration**: start/stop overhead is lower than image pull + container create. + +Lifecycle states: `created → running → stopped → (restarted | removed)` + +**Status: decided. Eviction policy (when to actually remove containers) is open.** + +--- + +### Caches as bind mounts + +Caches (e.g. Rust crate registry, npm cache, Maven local repo) are bind-mounted into containers at well-known paths. Cache selection is configurable per task. + +#### Ownership model + +| Type | Description | +|---|---| +| **Private** | Owned by one project. Removed when that project releases it. | +| **Shared** | Multiple projects declare ownership. Reference-counted — not removed until all owners release it. | + +#### Quota + +Each cache has a quota. The system must tally usage and enforce limits. Shared caches split responsibility across owners (exact apportionment policy is open). + +**Status: concept decided. The following are open:** +- Quota enforcement mechanism (inotify + du polling? quotafs? btrfs subvolume quota?) +- How "releasing" a cache is triggered (explicit API call, TTL, run completion hook?) +- How shared cache conflicts are handled if owners have incompatible contents (e.g. different versions of a tool populating the same cache path) +- Whether caches can be layered the same way worktrees are (read-only shared base + writable upper) + +--- + +### Multi-image pipelines + +Tasks may be orchestrated in sequence. The output layer of one stage can become an additional lower layer in the overlayfs stack for the next stage. This avoids re-running earlier stages when retrying later ones. + +**Status: directionally decided. Exact mechanism for passing outputs between stages is open.** + +--- + +### Deployment bind mounts + +Certain directories on the host (e.g. `sites.efforting.tech` web root) are bind-mounted into containers as write targets. The task writes its output there directly, which constitutes deployment. + +Access control (which tasks are allowed to write which mounts) is a security consideration that must be addressed before this is used in untrusted contexts. + +**Status: approach decided. Permission model is open.** + +--- + +### Progress reporting + +stdout/stderr from containers is captured by streaming `docker compose up` output directly to the CI server process. The container runs its task and exits naturally — no long-running idle process, no exec. This requires no networking inside the container. + +For richer or asynchronous progress reporting (e.g. from a long-running build that wants to emit status mid-run), a future option is a callback channel from inside the container to the CI server — likely a Unix socket or HTTP POST to a known address. This requires limited networking back to the host only, not full WAN access. + +A dedicated **text stream server** will handle log distribution: + +- Live runs: consumers can subscribe and receive output as it arrives +- Past runs: logs are stored and retrievable by run ID +- The CI server pipes container stdout/stderr into this service rather than handling log storage itself + +This keeps log concerns out of the CI orchestrator and gives a single place to tail, replay, or archive output. + +**Status: stdout capture via `compose up` is the current approach. Text stream server is planned but not yet designed.** + +A **web interface** will consume the text stream server and render output in the browser with full ANSI escape support. We will implement our own renderer — asciinema was considered but its `.cast` format silently replaces non-UTF-8 bytes with U+FFFD and has no binary escape hatch, making it a poor fit for a general log store. + +The set of ANSI escapes seen in real CI output is small and well-defined: + +- SGR colour/style codes (`\e[...m`) — foreground/background colours, bold, dim, reset +- Cursor movement (`\e[A/B/C/D`, `\e[H`, `\e[f`) — used by progress bars +- Erase (`\e[K`, `\e[2J`) — line/screen clearing, also used by progress bars + +A purpose-built renderer targeting only these sequences is straightforward and gives full control over the UI. + +**Status: decided (custom renderer). Design and implementation deferred.** + +To maximise colour output from tools running inside containers, we set a combination of environment variables since there is no standard "enable colour, no interaction" signal — tools each have their own heuristic. The baseline set: + +| Variable | Effect | +|---|---| +| `TERM=xterm-256color` | Tells tools the terminal supports 256 colours | +| `FORCE_COLOR=1` | Respected by Node.js ecosystem tools | +| `CLICOLOR_FORCE=1` | Respected by many Unix tools | +| `NO_COLOR` | Must be *unset* (some tools default to checking this) | + +Individual tools may need additional flags (e.g. `--color=always` for git, cargo). These can be set per image or per task declaration. + +> [!NOTE] +> **Future / low priority:** Allocating a PTY for the container process would make isatty() return true, solving the colour detection problem universally without any env var hacks. The trade-off is added complexity in I/O handling (terminal control sequences, input plumbing). Not worth pursuing until the env var approach proves insufficient. + +--- + +### Network isolation + +Runners need outbound WAN access (to fetch dependencies) but must not be able to reach internal LAN or host-local services. Two approaches under consideration: + +1. **Subnet filtering** — firewall rules block RFC-1918 ranges for container network namespaces while allowing WAN. +2. **Custom egress VPS** — route all runner traffic through a separate VPS with egress filtering, reusable across services. + +For tasks that only operate on local mounts (e.g. the rsync publish experiment), `network_mode: none` is used — no network at all. + +**Status: open. Approach 2 is preferred for reusability but adds infrastructure complexity.** + +--- + +### Cross-platform builds + +Builds targeting non-Linux platforms (Windows, macOS) would require QEMU or separate VPS instances. This is explicitly deferred — it will not be part of the initial architecture. + +**Status: deferred.** + +--- + +## Summary of open questions + +| Topic | What's open | +|---|---| +| Overlayfs + Docker | How to compose overlayfs worktree mounts with Docker's own storage driver | +| Worktree cache invalidation | How and when owner worktree caches are invalidated on access revocation | +| Container eviction | When and how stopped containers are removed | +| Cache quota enforcement | Mechanism for measuring and enforcing per-cache quotas | +| Cache release trigger | How a project signals it no longer needs a cache | +| Shared cache conflicts | How to handle incompatible writes from different owners | +| Layered caches | Whether caches can use the same overlay approach as worktrees | +| Deployment permissions | Which tasks are allowed to write which bind-mounted targets | +| SSH key registration | Automate public key registration via Gitea API (`POST /repos/{owner}/{repo}/keys`) — requires a Gitea user token with repo admin permissions, which is a separate credential. Only worth pursuing if the CI system already holds a user token for other reasons (status checks, repo metadata etc.). | +| Network isolation | Subnet filtering vs egress VPS | +| Progress reporting | In-container callback channel (socket/HTTP) for async mid-run status | +| Multi-stage output passing | Exact format/protocol for stage-to-stage data handoff |