commit 3fee797b7f794eed4193443341830f478c621ede Author: Sckathach Date: Mon Oct 19 14:16:52 2026 -0700 first commit? diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36a3079 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +vendor/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..231ff51 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,11 @@ +i value honesty and criticism: it helps me learn and improve; i am never annoyed: tell me when im wrong or when i make mistakes; im doing research and need to focus on what's most likely to work, not bullshit; try to be concise, and get important stuff clear; i will ask for more details if i need them; avoid sycophancy at all costs; + +i know python, and used to code in other languages (c, java, typescript, lua, ocaml, rust); i know databases and cloud infrastructure (docker, podman, kubernetes); im using my terminal as much as possible, so im fluent in bash and prefer cli tools over gui; + +im a researcher in maths and ai; + +the rules that survive are the ones that raise quality or make verification cheaper; + +If you're not sure, say so. Don't invent things. Make your uncertainty clear. + +Write the smallest amount of code that solves the real problem. Don’t build extra layers for code that’s used only once. Don’t use clever patterns that make the code harder to maintain later. diff --git a/skills/common-tooling/SKILL.md b/skills/common-tooling/SKILL.md new file mode 100644 index 0000000..a94f725 --- /dev/null +++ b/skills/common-tooling/SKILL.md @@ -0,0 +1,38 @@ +--- +name: common-tooling +description: CLI tools that should be present on the machine and that you should use, read this at the beginning of every session. +--- + +## Tools + +Tools that should be present on your machine: + +- Search and edit: `rg`, `fd`, `sd`, `ast-grep`, `tree`, `difftastic` +- Structured data: `dasel` (yaml, toml, json, xml, v3 syntax: `-i -o ` , not the `-f / -r / -w` of v1 that most online examples show), `jq` (json only), `yq` (kislyuk's jq wrapper: jq syntax on yaml, not mikefarah's Go `yq`), `mlr` (csv, tsv, json records), `sqlite3` (knowledge graph) +- HTTP, network: `xh` (prefer that to `curl`), `monolith` (freezing webpage into self-contained HTML), `websocat`, `grpcurl` +- Running things: `watchexec`, `just`, `pre-commit`, `hyperfine` (timing) +- Quality gates: `shellcheck` (use with every shell script), `shfmt`, `eslint`, `tokei` +- System and debugging: `lsof`, `strace`, `procs` (instead of `ps`), `btm` (better `top`), `dust` (what is eating this directory), `duf` -> use `dust` and `duf` when managing heavy artifacts +- Papers and bibtex export: `papis` + +The list lives in `scripts/tools.tsv`. Run `scripts/verify-tools.sh -q` to check they are present, and alert the user if not; `scripts/install-tools.sh` installs the missing ones (pacman or apt, then cargo-binstall / GitHub releases / uv for the rest, needs sudo), so ask before running it. + +## Feedbacks + +Improvement needs feedback, and yours is very important. If anything related to skills, project management, or general work with the user goes wrong (or clearly works), append an entry to `feedbacks.jsonl` at the root of where you're working. Each entry has: + +``` +at: ISO-8601 local timestamp. +kind: "bug", "friction", "idea", "improvement", "worked" +text: The note itself, stripped. +about: Which skill, command or convention it concerns; `""` if unsaid. +repo: Working directory it was written from: feedbacks will be merged automatically, write the name of the project so we know the origin after the merge; don't worry too much about possible conflicts; +``` + +## Markdown + +Do not wrap lines manually please. Neovim handles the wrapping and adapts to the width of the window. + +## General coding + +**Rule of three.** Promote a function to shared only when a _third_ project would copy it unchanged. Until then, copy-paste: duplication is cheaper than the wrong abstraction. diff --git a/skills/common-tooling/scripts/install-tools.sh b/skills/common-tooling/scripts/install-tools.sh new file mode 100755 index 0000000..aa80c79 --- /dev/null +++ b/skills/common-tooling/scripts/install-tools.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Install every tool in tools.tsv that is not already on PATH. +# +# Order per tool: the distro package (pacman, or apt when apt has a candidate), +# then the fallback column: cargo-binstall for Rust tools (prebuilt binaries, +# no compiling), a GitHub release for uv and the Go tools, `uv tool` or +# `npm -g` for the rest. No `curl | sh`: every download is a release asset. +# User-level installs land in ~/.local/bin (binstall's own in ~/.cargo/bin). +# +# Usage: install-tools.sh [--dry-run] +# Arch, Debian (trixie; bookworm lacks more and falls back more), Ubuntu 24.04. +# Needs sudo for the distro half unless run as root. +set -euo pipefail + +here=$(cd -- "$(dirname -- "$(readlink -f -- "$0")")" && pwd) +dry=false +[ "${1:-}" = "--dry-run" ] && dry=true + +run() { + printf ' + %s\n' "$*" + $dry || "$@" +} + +sudo=() +[ "$(id -u)" -eq 0 ] || sudo=(sudo) + +if command -v pacman >/dev/null; then + distro=pacman +elif command -v apt-get >/dev/null; then + distro=apt +else + echo "neither pacman nor apt-get found; only fallbacks will be tried" >&2 + distro=none +fi + +# --- 1. Distro packages, one transaction -------------------------------------- +missing_pkgs=() # parallel arrays: package name, and the binary it provides +missing_bins=() +while read -r bin _ pacman apt _; do + case $bin in '' | '#'*) continue ;; esac + command -v -- "$bin" >/dev/null && continue + case $distro in + pacman) pkg=$pacman ;; + apt) pkg=$apt ;; + *) pkg=- ;; + esac + [ "$pkg" = - ] && continue + missing_pkgs+=("$pkg") + missing_bins+=("$bin") +done <"$here/tools.tsv" + +if [ "$distro" = apt ]; then + run "${sudo[@]}" apt-get update -qq + # Keep only what this release actually has: trixie has xh/tokei/procs, + # bookworm and noble do not, and one unknown name fails the whole install. + pkgs=() bins=() + for i in "${!missing_pkgs[@]}"; do + if apt-cache policy "${missing_pkgs[i]}" 2>/dev/null | grep -q 'Candidate: [^(]'; then + pkgs+=("${missing_pkgs[i]}") + bins+=("${missing_bins[i]}") + fi + done + missing_pkgs=("${pkgs[@]}") missing_bins=("${bins[@]}") +fi +# What the distro step covers is not the fallbacks' business, even in a dry run +# where nothing actually got installed. +covered=" ${missing_bins[*]} " + +if [ "${#missing_pkgs[@]}" -gt 0 ]; then + echo "distro packages:" + case $distro in + pacman) run "${sudo[@]}" pacman -S --needed --noconfirm "${missing_pkgs[@]}" ;; + apt) run "${sudo[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "${missing_pkgs[@]}" ;; + esac +fi + +mkdir -p "$HOME/.local/bin" +# Debian renames fd's binary to fdfind (a name clash with an old package). +if ! command -v fd >/dev/null && command -v fdfind >/dev/null; then + run ln -sf "$(command -v fdfind)" "$HOME/.local/bin/fd" +fi + +# --- 2. Fallbacks, per tool --------------------------------------------------- +# Release assets name the architecture three ways. +case $(uname -m) in +x86_64) arch_rust=x86_64 arch_go=amd64 arch_tar=x86_64 ;; +aarch64 | arm64) arch_rust=aarch64 arch_go=arm64 arch_tar=arm64 ;; +*) arch_rust=unknown arch_go=unknown arch_tar=unknown ;; +esac +latest() { printf 'https://github.com/%s/releases/latest/download/%s' "$1" "$2"; } + +# Installed, or planned earlier in this run (which is all a dry run has). +have() { + command -v -- "$1" >/dev/null && return + case $covered in *" $1 "*) return 0 ;; esac + return 1 +} + +ensure_binstall() { + have cargo-binstall && return + covered="$covered cargo-binstall " + echo "cargo-binstall (fetches prebuilt Rust binaries):" + # musl: static, so it runs whatever glibc the machine has. + run bash -c "curl -fsSL $(latest cargo-bins/cargo-binstall "cargo-binstall-$arch_rust-unknown-linux-musl.tgz") | tar -xz -C $HOME/.local/bin cargo-binstall" +} + +gh_install() { + case $1 in + uv) # the tarball holds uv-/{uv,uvx} + run bash -c "curl -fsSL $(latest astral-sh/uv "uv-$arch_rust-unknown-linux-gnu.tar.gz") | tar -xz --strip-components=1 -C $HOME/.local/bin" + ;; + dasel) + run curl -fsSL -o "$HOME/.local/bin/dasel" "$(latest TomWright/dasel "dasel_linux_$arch_go")" + run chmod +x "$HOME/.local/bin/dasel" + ;; + grpcurl) # the asset name embeds the version, so ask for it + local tag=LATEST + $dry || tag=$(curl -fsSL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest | + sed -n 's/.*"tag_name": *"v\([^"]*\)".*/\1/p') + run bash -c "curl -fsSL https://github.com/fullstorydev/grpcurl/releases/download/v$tag/grpcurl_${tag}_linux_$arch_tar.tar.gz | tar -xz -C $HOME/.local/bin grpcurl" + ;; + *) echo " no GitHub recipe for $1" >&2 ;; + esac +} + +login_path=$PATH +export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH" +while read -r bin need _ _ fallback; do + case $bin in '' | '#'*) continue ;; esac + have "$bin" && continue + [ "$fallback" = - ] && { + echo " $bin: no fallback; install it by hand" >&2 + continue + } + echo "$bin (${fallback%%:*}):" + covered="$covered$bin " + case $fallback in + binstall:*) + ensure_binstall + run cargo-binstall -y --quiet "${fallback#binstall:}" + ;; + gh:*) gh_install "${fallback#gh:}" ;; + uv:*) + if have uv; then + run uv tool install --quiet "${fallback#uv:}" + else + echo " needs uv, which failed to install above" >&2 + fi + ;; + npm:*) + if have npm; then + run "${sudo[@]}" npm install -g --silent "${fallback#npm:}" + else + echo " $bin is $need and needs npm; skipped" >&2 + fi + ;; + esac +done <"$here/tools.tsv" + +$dry && exit 0 +# The verify below sees this script's PATH; a login shell may not. +for dir in "$HOME/.local/bin" "$HOME/.cargo/bin"; do + case ":$login_path:" in *":$dir:"*) ;; *) echo "NOTE: add $dir to PATH in your shell rc" ;; esac +done +echo "verifying:" +exec "$here/verify-tools.sh" -q diff --git a/skills/common-tooling/scripts/tools.tsv b/skills/common-tooling/scripts/tools.tsv new file mode 100644 index 0000000..1143e7b --- /dev/null +++ b/skills/common-tooling/scripts/tools.tsv @@ -0,0 +1,52 @@ +# The one list both install-tools.sh and verify-tools.sh read. Columns, tab or +# space separated; `-` means "not from here": +# +# bin the command verify-tools.sh looks for on PATH +# need core (verify fails without it) | opt (verify only warns) +# pacman Arch package, official repos only (AUR-only tools use the fallback) +# apt Debian/Ubuntu package, used only when apt has a candidate for it +# fallback binstall: | gh: | uv: | npm: +# +# Deliberate `-` in the apt column, checked against the trixie/bookworm/noble +# indexes on 2026-09-25: +# dasel apt ships v2; the skill documents v3 syntax +# eslint apt ships 6.4 (2019) +# typst is in no Debian release (sources.debian.org, 2026-10-10); pyroli +# renders `math: typst` atoms with it +# uv is in no apt release; first row, because the uv: fallbacks need it +# yq is kislyuk's jq wrapper everywhere (pacman `yq`, apt `yq`, PyPI `yq`), +# not mikefarah's Go yq (`go-yq`); jq syntax, and it needs jq +# +# bin need pacman apt fallback +uv core uv - gh:uv +rg core ripgrep ripgrep binstall:ripgrep +fd core fd fd-find binstall:fd-find +sd core sd sd binstall:sd +ast-grep core ast-grep - binstall:ast-grep +tree core tree tree - +difft core difftastic - binstall:difftastic +typst core typst - binstall:typst-cli +dasel core - - gh:dasel +jq core jq jq - +yq core yq yq uv:yq +mlr core miller miller - +sqlite3 core sqlite sqlite3 - +xh core xh xh binstall:xh +monolith core monolith - binstall:monolith +websocat core websocat - binstall:websocat +grpcurl core - - gh:grpcurl +watchexec core watchexec - binstall:watchexec-cli +just core just just binstall:just +pre-commit core pre-commit pre-commit uv:pre-commit +hyperfine core hyperfine hyperfine binstall:hyperfine +shellcheck core shellcheck shellcheck - +shfmt core shfmt shfmt - +eslint opt eslint - npm:eslint +tokei core tokei tokei binstall:tokei +lsof core lsof lsof - +strace core strace strace - +procs core procs procs binstall:procs +btm core bottom - binstall:bottom +dust core dust du-dust binstall:du-dust +duf core duf duf - +papis core - - uv:papis diff --git a/skills/common-tooling/scripts/verify-tools.sh b/skills/common-tooling/scripts/verify-tools.sh new file mode 100755 index 0000000..98d1559 --- /dev/null +++ b/skills/common-tooling/scripts/verify-tools.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Report which tools from tools.tsv are on PATH. Exit 1 if a core one is missing. +# +# Usage: verify-tools.sh [-q] -q prints only what is missing +set -euo pipefail + +here=$(cd -- "$(dirname -- "$(readlink -f -- "$0")")" && pwd) +quiet=false +[ "${1:-}" = "-q" ] && quiet=true + +status=0 +while read -r bin need _; do + case $bin in '' | '#'*) continue ;; esac + if command -v -- "$bin" >/dev/null; then + $quiet || printf ' ok %s\n' "$bin" + elif [ "$need" = core ]; then + printf ' MISSING %s\n' "$bin" + status=1 + else + printf ' missing %s (optional)\n' "$bin" + fi +done <"$here/tools.tsv" + +# Present is not enough for the one tool whose major versions disagree on syntax. +if command -v dasel >/dev/null && ! dasel version 2>/dev/null | grep -q '^v\?3\.'; then + printf ' WRONG dasel is not v3 (%s); the skill documents v3 syntax\n' \ + "$(dasel --version 2>/dev/null || dasel version 2>/dev/null)" + status=1 +fi + +[ "$status" = 0 ] || printf 'install the missing ones with %s/install-tools.sh\n' "$here" +exit "$status" diff --git a/skills/knowledge-and-memory/SKILL.md b/skills/knowledge-and-memory/SKILL.md new file mode 100644 index 0000000..80153fb --- /dev/null +++ b/skills/knowledge-and-memory/SKILL.md @@ -0,0 +1,74 @@ +--- +name: knowledge-and-memory +description: Knowledge and memory system that you should follow at every session, unless explicitly discarded by the user +--- + +Rationale: Files like `MEMORY.md`, `ARCHITECTURE.md` will rot. Memory is composed of project-wise entries, as well as general entries worth keeping for later projects. Knowledge is relational rather than linear and context is precious. + +Solution: atomic (mostly markdown) files with yaml frontheader and links: only retrieve what you need. Plus, regular sessions will be aimed at cleaning this knowledge database, update or remove atoms, and promote general atoms to a global knowledge database -> you don't need to worry about this process, it's user based. + +## Knowledge atoms + +One file, one thing: `knowledge//.md`, YAML frontmatter plus ~40 lines of body. Use more lines only if necessary to describe ONE thing. The directory _is_ the type, the filename stem _is_ the id, links are `[[wikilinks]]`, and the ontology is `knowledge/schema.yaml`. If it is missing, copy the one from the `research` or `swe` skill (`references/schema.yaml`) and edit it from there: the repo's copy is the one in force. + +```yaml +--- +title: The vocabulary-common direction is invisible to the softmax # The title is the claim: it's the thing we'll retrieve with fd, make it understandable, "The moment split - M = v v^T + Sigma" is a much better title than "Moment split notes". +type: gotcha # the directory this file is in; see the schema.yaml +status: stable # legal values depend on the type +tags: [jlens, centring] # assembly keys, not truth — see below +evidence: artifacts/rfm.safetensors # optional: path to the artifact that supports this atom +confidence: high # optional: low | medium | high +importance: 4 # optional: 1-5, for ordering a work queue +math: typst # optional: write the math in Typst syntax, rendered by pyroli with the `typst` binary; default is LaTeX (KaTeX) +aliases: [softmax-blind] # optional: former slugs that still resolve, use this if you ever need to change the slug +edges: # optional: add CLEAN edges if it makes sense: the whole knowledge depends on qualitative relations. Edges are one-directional; use only relations defined in schema.yaml, never their inverses + supports: [rfm-is-layer-local] + depends_on: [h3-mean-covariance-split] +--- + +**One-sentence claim, bolded.** This is the line that gets grepped and the line a report links to. + +Then why. Derivation, mechanism, or measurement. Inline links to [[effective-rank]] and [[haar-random-null]] instead of re-explaining them. + + +## Examples + +Examples that can be project related: a math definition with examples from this project for instance. +``` + +Design background for all of this, when a question about the format comes up: `references/sources.md`. + +Tags are assembly keys, not a retrieval index. They are used to pack atoms together. + +Other frontmatter keys are allowed (`verify:`, `provenance:`...): the schema's `requires:` can name them. + +## Rules + +1. **Anything derivable from the code is generated, never written.** A hand-written module table is exactly the one that rots. Invariants are not derivable, which is why they are written down and why they do not. Use bash scripts, and reference scripts / commands inside markdown instead of writing the output. +2. Math in atoms is LaTeX (rendered by KaTeX), or Typst when the frontmatter says `math: typst`: follow the atom's flag, never mix the two in one atom. Use ASCII transcription when talking to the user in the terminal. +3. Atoms cite sources, sources never link to atoms (we can always go back with the graph) + +## `pyroli`: the wiki and the checks + +`pyroli serve` renders `knowledge/` as a linked wiki (KaTeX, backlinks, full-text search) with a margin: the user selects text and annotates it, and notes append to `annotations.jsonl`, never to an atom. Read the margin notes on an atom before editing it. Install or update with `scripts/install-pyroli.sh`. + +`pyroli check` runs, and must pass before committing atoms: + +1. every `[[link]]` and edge target resolves, aliases included — **including links in `reports/*.py` and `experiments/*.py`**, which is where a rename actually hurts +2. type declared, status legal for it, required fields present +3. every edge is a declared relation and respects its domain and range +4. the file sits in the directory its type names +5. every `evidence:` script exists **in git**, not merely on disk +6. no settled atom `depends_on` a dead one +7. an atom whose evidence script was **committed** later than it → warn + +Checks 2, 3 and 6 need `knowledge/schema.yaml`; without it, `check` says so and skips them. + +## Repository structure when a knowledge/ folder is present + +Templates for `AGENTS.md` and `CLAUDE.md` are in `references/templates/`. Use `README.md` for a general presentation, quickstart, workflows, non-goals, mostly user-facing. Use `AGENTS.md` for how to work in this repo, skill used, deltas from the method, update at every session if needed. Link to knowledge atoms, never rewrite. Add `CLAUDE.md` and redirect it to `AGENTS.md`. Keep `AGENTS.md` short: it should state things that aren't already elsewhere or link them. It should have the current state of the project. + +You can have other .md files when necessary: a global `PLAN.md`, a global `TODO.md`, or a `CURRENT.md` to keep current state during compacts. When needed, use nested markdown files, for instance, you can have a global `AGENTS.md` and one `AGENTS.md` per sub folder. + +No `SPECS.md`, no `STRUCTURE.md` diff --git a/skills/knowledge-and-memory/pyroli/pyproject.toml b/skills/knowledge-and-memory/pyroli/pyproject.toml new file mode 100644 index 0000000..b778449 --- /dev/null +++ b/skills/knowledge-and-memory/pyroli/pyproject.toml @@ -0,0 +1,56 @@ +[project] +name = "pyroli" +version = "0.8.0" +description = "Serve a knowledge/ folder as a linked, math-rendering (LaTeX or Typst) wiki with a margin, and check its links." +requires-python = ">=3.12" +authors = [{ name = "Sckathach", email = "thomas.winninger@telecom-sudparis.eu" }] +dependencies = [ + "markdown-it-py>=4.2.0", + "mdit-py-plugins>=0.6.1", + "pyyaml>=6.0", + "rich>=13.0", + "starlette>=1.3.1", + "typer>=0.12", + "uvicorn>=0.52.0", +] + +[project.scripts] +pyroli = "pyroli.cli:main" + +[build-system] +requires = ["uv_build>=0.12.0,<0.13.0"] +build-backend = "uv_build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + "ruff>=0.6", + "pyrefly>=0.16", + "httpx2>=2.13.1", +] + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "COM812", # conflicts with the formatter + "D203", # incompatible with D211 + "D213", # incompatible with D212 + "CPY001", # no copyright headers + # Malformed *input* raises ValueError here, including bad shapes: one + # exception type keeps `except ValueError` sufficient at the CLI. + "TRY004", +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["D", "S101", "PLR2004", "ANN", "INP001", "DTZ001"] +# typer's API is boolean options with positional defaults. +"src/pyroli/cli.py" = ["FBT001", "FBT002"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/skills/knowledge-and-memory/pyroli/src/pyroli/__init__.py b/skills/knowledge-and-memory/pyroli/src/pyroli/__init__.py new file mode 100644 index 0000000..eee6bea --- /dev/null +++ b/skills/knowledge-and-memory/pyroli/src/pyroli/__init__.py @@ -0,0 +1,27 @@ +"""pyroli — serve a `knowledge/` folder as a wiki, and check its links. + +Two commands: `serve` (the wiki, KaTeX, the annotation margin) and `check` +(links, types, statuses, edges, evidence). The `knowledge-and-memory` skill in +this repo, under `skills/`, is the format they read. + +Importable for one thing: rendering `[[slug]]` citations as atom titles, which +reports would otherwise each reimplement. + + from pyroli import link_titles, load_atoms + + atoms = load_atoms(REPO / "knowledge") + def md(text: str): + return mo.md(link_titles(text, atoms)) +""" + +from importlib.metadata import version + +from pyroli.atoms import Atom, link_titles +from pyroli.atoms import load as load_atoms + +# Read from the installed distribution, never hardcoded: a string in two places +# is a string that disagrees with itself, and the number's only job is to tell +# you which build you are looking at. +__version__ = version("pyroli") + +__all__ = ["Atom", "__version__", "link_titles", "load_atoms"] diff --git a/skills/knowledge-and-memory/pyroli/src/pyroli/annotations.py b/skills/knowledge-and-memory/pyroli/src/pyroli/annotations.py new file mode 100644 index 0000000..8dee211 --- /dev/null +++ b/skills/knowledge-and-memory/pyroli/src/pyroli/annotations.py @@ -0,0 +1,238 @@ +"""Margin notes on atoms: an append-only sidecar keyed by quoted text. + +Pure: JSONL text in, records out. Reading and appending the file is the +caller's job — `server.py` for the browser, `cli.py` for the terminal. + +An annotation never touches the atom it is about. That is the whole design: +the atom stays authored, in one editor, by one person, and the margin +accretes reactions that are refactored back into prose later — C2's +ThreadMode and DocumentMode, kept in separate files instead of separate +sections. Which means the wiki can be writable without `pyroli serve` +becoming an editor. + +Anchoring is by quoted text, not by offset, because an offset into a file +under active edit is wrong within the hour. When the quote stops matching, +the annotation is an **orphan** — and the orphan list is not an error state, +it is the revision backlog: every entry is a passage you had something to say +about and have since rewritten. + +Invariants: + - Append-only, one JSON object per line. Nothing rewrites a line. A note + that no longer applies is orphaned; a note the reader removes gets a + second line, `{"id": ..., "deleted":