Integration of Patchstorage for sharing pedalboards.

GUIDE.md

@@ -58,6 +58,10 @@ python3 server.py
 
 On the device these values are injected by the `mod-ui.service` in `pi-gen-pistomp` (see `debpkgs/mod-ui/debian/mod-ui.mod-ui.service`).
 
+## Testing a branch on a pi-Stomp
+
+`./deploy.sh` copies this working tree over the packaged mod-ui on the device and restarts it; `./deploy.sh restore` puts the packaged version back. See [`docs/deploying-to-pistomp.md`](docs/deploying-to-pistomp.md) for the recipe and its limits.
+
 ## Repo layout
 
 | Path | Purpose |

deploy.sh

@@ -1,42 +1,156 @@
 #!/usr/bin/env bash
-# Deploy mod-ui source to the running device via rsync and restart the service.
+# Deploy this mod-ui working tree onto a pi-Stomp over the packaged install, and
+# put the packaged version back when you are done. See docs/deploying-to-pistomp.md.
 # Override PISTOMP_HOST / PISTOMP_USER if your device differs from the default.
 #
 # Usage:
-#   ./deploy.sh                 # sync Python + HTML, restart service
+#   ./deploy.sh                 # sync Python + HTML, restart service, wait until it serves
 #   ./deploy.sh --build-utils   # also rsync utils/ source, build on device, install .so
+#   ./deploy.sh status          # packaged version, what is deployed over it, service state
+#   ./deploy.sh logs [args]     # follow the mod-ui journal (args go to journalctl)
+#   ./deploy.sh snapshot        # save the packaged install (done automatically on first deploy)
+#   ./deploy.sh restore         # put the snapshot back and restart
 set -euo pipefail
 
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
 HOST="${PISTOMP_HOST:-pistomp.local}"
 USER="${PISTOMP_USER:-pistomp}"
 TARGET="${USER}@${HOST}"
 
 VENV="/opt/pistomp/venvs/mod-ui"
 HTML_DIR="/opt/pistomp/mod-ui/html"
+MARKER="/opt/pistomp/mod-ui/DEV-DEPLOY"
+SNAPSHOT_DIR="/home/${USER}/mod-ui-snapshots"
+
+# One SSH connection for the whole run: one password prompt at most, and faster.
+SSH_OPTS=(-o ControlMaster=auto -o "ControlPath=/tmp/mod-ui-deploy-%r@%h:%p" -o ControlPersist=60 -o ConnectTimeout=10)
+export RSYNC_RSH="ssh ${SSH_OPTS[*]}"
+
+remote() { ssh "${SSH_OPTS[@]}" "${TARGET}" "$@"; }
+
+connect() {
+    if ! remote true; then
+        echo "error: cannot reach ${TARGET}. Is the pi-Stomp on the network? Set PISTOMP_HOST to its IP if mDNS fails." >&2
+        exit 1
+    fi
+    PKG_VERSION="$(remote "dpkg-query -W -f='\${Version}' mod-ui")"
+    PYTHON_VERSION="$(remote "ls ${VENV}/lib/" | grep -oE 'python[0-9]+\.[0-9]+' | head -1 | grep -oE '[0-9]+\.[0-9]+')"
+    SITE_PACKAGES="${VENV}/lib/python${PYTHON_VERSION}/site-packages"
+    SNAPSHOT="${SNAPSHOT_DIR}/mod-ui-${PKG_VERSION}.tar.gz"
+}
+
+# Must only ever capture the packaged files: refuses while a dev deploy is in place.
+snapshot() {
+    if remote "test -f ${SNAPSHOT}"; then
+        echo "==> Snapshot of packaged mod-ui ${PKG_VERSION} already at ${SNAPSHOT}"
+        return
+    fi
+    # A marker from an older package version means an upgrade already replaced the dev files.
+    if remote "grep -qx 'package=${PKG_VERSION}' ${MARKER} 2>/dev/null"; then
+        echo "error: a dev build is deployed and there is no snapshot of mod-ui ${PKG_VERSION}." >&2
+        echo "       Reinstall the package first (sudo apt install --reinstall mod-ui), then retry." >&2
+        exit 1
+    fi
+    echo "==> Saving packaged mod-ui ${PKG_VERSION} to ${SNAPSHOT}"
+    remote "mkdir -p ${SNAPSHOT_DIR} && tar -C / -czf ${SNAPSHOT}.tmp \
+        ${SITE_PACKAGES#/}/mod ${SITE_PACKAGES#/}/modtools ${HTML_DIR#/} && mv ${SNAPSHOT}.tmp ${SNAPSHOT}"
+}
+
+wait_until_serving() {
+    echo "==> Waiting for mod-ui to answer on port 80"
+    for _ in $(seq 1 45); do
+        if remote "${VENV}/bin/python -c \"import urllib.request; urllib.request.urlopen('http://localhost/', timeout=3)\"" 2>/dev/null; then
+            echo "==> mod-ui is up: http://${HOST}/"
+            return
+        fi
+        sleep 2
+    done
+    echo "error: mod-ui did not come up within 90s. Last journal lines:" >&2
+    remote "sudo journalctl -u mod-ui -n 40 --no-pager" >&2 || true
+    echo "Put the packaged version back with: ./deploy.sh restore" >&2
+    exit 1
+}
+
+deploy() {
+    local build_utils="${1:-}"
+
+    echo "==> Checking Python syntax locally"
+    python3 -W ignore -m py_compile mod/*.py modtools/*.py
+
+    connect
+    remote "command -v rsync >/dev/null" || { echo "error: rsync is not installed on the device (sudo apt install rsync)" >&2; exit 1; }
+    snapshot
+
+    local sha branch dirty=""
+    sha="$(git rev-parse --short HEAD)"
+    branch="$(git rev-parse --abbrev-ref HEAD)"
+    [[ -n "$(git status --porcelain)" ]] && dirty=" + uncommitted changes"
+    echo "==> Deploying ${branch} @ ${sha}${dirty} to ${TARGET} (Python ${PYTHON_VERSION})"
+
+    rsync -az --delete --exclude='__pycache__' --exclude='*.pyc' \
+        mod/ "${TARGET}:${SITE_PACKAGES}/mod/"
 
-PYTHON_VERSION="$(ssh "${TARGET}" "ls ${VENV}/lib/" | grep -oE 'python[0-9]+\.[0-9]+' | head -1 | grep -oE '[0-9]+\.[0-9]+')"
-SITE_PACKAGES="${VENV}/lib/python${PYTHON_VERSION}/site-packages"
+    # Exclude libmod_utils.so: it is ARM64 and must be built on the device (see --build-utils).
+    rsync -az --delete --exclude='__pycache__' --exclude='*.pyc' --exclude='libmod_utils.so' \
+        modtools/ "${TARGET}:${SITE_PACKAGES}/modtools/"
 
-echo "==> Deploying to ${TARGET} (Python ${PYTHON_VERSION})"
+    rsync -az --delete --exclude='node_modules' --exclude='.DS_Store' \
+        html/ "${TARGET}:${HTML_DIR}/"
 
-rsync -az --delete --exclude='__pycache__' --exclude='*.pyc' \
-    mod/ "${TARGET}:${SITE_PACKAGES}/mod/"
+    if [[ "${build_utils}" == "--build-utils" ]]; then
+        echo "==> Building utils on device"
+        rsync -az --delete --exclude='*.o' --exclude='*.so' \
+            utils/ "${TARGET}:/tmp/mod-ui-utils/"
+        remote "make -C /tmp/mod-ui-utils && cp /tmp/mod-ui-utils/libmod_utils.so ${SITE_PACKAGES}/modtools/"
+    fi
 
-# Exclude libmod_utils.so: it is ARM64 and must be built on the device (see --build-utils).
-rsync -az --delete --exclude='__pycache__' --exclude='*.pyc' --exclude='libmod_utils.so' \
-    modtools/ "${TARGET}:${SITE_PACKAGES}/modtools/"
+    remote "printf '%s\n' 'branch=${branch}' 'commit=${sha}${dirty}' 'package=${PKG_VERSION}' \
+        'deployed=$(date '+%Y-%m-%d %H:%M:%S %z')' 'from=$(whoami)@$(hostname -s)' > ${MARKER}"
 
-rsync -az --delete \
-    html/ "${TARGET}:${HTML_DIR}/"
+    echo "==> Restarting mod-ui"
+    remote "sudo systemctl restart mod-ui"
+    wait_until_serving
+}
 
-if [[ "${1:-}" == "--build-utils" ]]; then
-    echo "==> Building utils on device"
-    rsync -az --delete --exclude='*.o' --exclude='*.so' \
-        utils/ "${TARGET}:/tmp/mod-ui-utils/"
-    ssh "${TARGET}" "make -C /tmp/mod-ui-utils && cp /tmp/mod-ui-utils/libmod_utils.so ${SITE_PACKAGES}/modtools/"
-fi
+restore() {
+    connect
+    if ! remote "test -f ${SNAPSHOT}"; then
+        echo "error: no snapshot of mod-ui ${PKG_VERSION} at ${SNAPSHOT}." >&2
+        echo "       Fall back to: ssh ${TARGET} sudo apt install --reinstall mod-ui" >&2
+        exit 1
+    fi
+    echo "==> Restoring packaged mod-ui ${PKG_VERSION} from ${SNAPSHOT}"
+    remote "rm -rf ${SITE_PACKAGES}/mod ${SITE_PACKAGES}/modtools ${HTML_DIR} \
+        && tar -C / -xzf ${SNAPSHOT} && rm -f ${MARKER}"
+    echo "==> Restarting mod-ui"
+    remote "sudo systemctl restart mod-ui"
+    wait_until_serving
+}
 
-echo "==> Restarting mod-ui"
-ssh "${TARGET}" "sudo systemctl restart mod-ui"
+status() {
+    connect
+    echo "Device:          ${TARGET}"
+    echo "Package:         mod-ui ${PKG_VERSION}"
+    if remote "test -f ${MARKER}"; then
+        echo "Dev deploy:      yes"
+        remote "sed 's/^/                 /' ${MARKER}"
+        if ! remote "grep -qx 'package=${PKG_VERSION}' ${MARKER}"; then
+            echo "                 (the package was upgraded after this deploy, so its files have replaced the dev build)"
+        fi
+    else
+        echo "Dev deploy:      no (packaged files)"
+    fi
+    echo "Snapshots:       $(remote "ls ${SNAPSHOT_DIR} 2>/dev/null | tr '\n' ' '")"
+    echo "Service:         $(remote "systemctl is-active mod-ui" || true)"
+}
 
-echo "==> Done"
+case "${1:-}" in
+    ""|--build-utils) deploy "${1:-}" ;;
+    status)           status ;;
+    logs)             shift; ssh "${SSH_OPTS[@]}" -t "${TARGET}" "sudo journalctl -u mod-ui ${*:--f}" ;;
+    snapshot)         connect; snapshot ;;
+    restore)          restore ;;
+    -h|--help)        sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ;;
+    *)                echo "unknown command: $1 (try --help)" >&2; exit 1 ;;
+esac

docs/deploying-to-pistomp.md

@@ -0,0 +1,76 @@
+# Testing a mod-ui branch on a pi-Stomp
+
+`deploy.sh` copies this working tree over the mod-ui that the `mod-ui` Debian
+package installed on the device, restarts the service, and can put the packaged
+version back afterwards. Nothing is rebuilt or repackaged, so a round trip takes
+seconds.
+
+## What gets replaced
+
+The package (built by `pi-gen-pistomp/debpkgs/mod-ui`) installs mod-ui in three
+places. `deploy.sh` overwrites the first two and leaves the third alone:
+
+| On the device | From this repo | Deployed |
+|---------------|----------------|----------|
+| `/opt/pistomp/venvs/mod-ui/lib/python3.11/site-packages/{mod,modtools}` | `mod/`, `modtools/` | always |
+| `/opt/pistomp/mod-ui/html` (`MOD_HTML_DIR`) | `html/` | always |
+| `.../site-packages/modtools/libmod_utils.so` | `utils/` (C++) | only with `--build-utils` |
+
+The service unit, its environment and the Python dependencies in the venv stay
+as packaged.
+
+## The recipe
+
+```bash
+git switch my-branch
+./deploy.sh            # snapshot the packaged files (first time only), copy, restart, wait for port 80
+# test at http://pistomp.local/ — a normal reload picks up new JS/CSS (the ?v= key changes on each deploy)
+./deploy.sh logs       # follow the mod-ui journal while you test
+# edit, then ./deploy.sh again; repeat
+./deploy.sh status     # what is on the device right now
+./deploy.sh restore    # back to the packaged mod-ui when you are done
+```
+
+Uncommitted changes are deployed too; `status` says so. The device is
+`pistomp@pistomp.local` unless you set `PISTOMP_HOST` / `PISTOMP_USER`
+(use the IP address if mDNS does not resolve).
+
+## Snapshots and markers
+
+- **Snapshot.** Before the first deploy, the packaged `mod`, `modtools` and `html`
+  are saved to `~pistomp/mod-ui-snapshots/mod-ui-<package version>.tar.gz`.
+  Later deploys reuse it. `restore` puts it back. It is keyed by package version,
+  so after an `apt upgrade` of mod-ui the next deploy takes a fresh one.
+- **Marker.** Each deploy writes `/opt/pistomp/mod-ui/DEV-DEPLOY` (branch, commit,
+  time, package version). `restore` removes it. `deploy.sh` will not snapshot
+  while a dev build sits on top of the installed package version, so a dev build
+  is never saved as "packaged".
+  If you hit that error, run `sudo apt install --reinstall mod-ui` on the device
+  and deploy again.
+
+## Things to know
+
+- **A package upgrade overwrites a dev deploy.** `status` notices and says so.
+- **Crash loops start recovery.** The unit has `OnFailure=pistomp-recovery.service`
+  and `StartLimitBurst=3` in 60s. If the deployed code dies on startup three
+  times, the device drops into the recovery UI. `deploy.sh` waits 90s for port
+  80, prints the journal if it never answers, and you run `./deploy.sh restore`.
+- **Edited `.less`? Rebuild `main.css` before deploying.** The device serves
+  the compiled `html/css/main.css` and never compiles `.less` itself. This
+  reproduces the committed file exactly:
+
+  ```bash
+  cd html/css
+  npm install --no-save bootstrap@3.4.1
+  npx -y -p less@2 lessc --compress less/main.less main.css
+  ```
+
+- **C++ changes (`utils/`) need `--build-utils`.** The `.so` must be built for
+  ARM64 on the device, which needs `make`, `g++`, `liblilv-dev` and
+  `libjack-jackd2-dev` there. Without the flag the packaged `.so` is kept, which
+  is right as long as your branch does not touch `utils/` beyond what the
+  installed package was built from.
+- **Dependency changes are not deployed.** If a branch needs a new Python
+  package or service environment variable, test it by building the package
+  instead (set `MOD_UI_BRANCH` in `pi-gen-pistomp/config.sh` and run
+  `debpkgs/mod-ui/build.sh`).

html/allguis.html

@@ -25,7 +25,7 @@
     <script type="text/javascript">
       var desktop    = null
       var SAMPLERATE = 48000
-      var VERSION    = '{{version}}'
+      var VERSION    = '{{image_version}}'
       var options    = {
           bypassed: 0,
       }

html/css/less/blokas.less

@@ -4,6 +4,8 @@
     #bank-library header,
     #cloud-plugins-library header,
     #patchstorage-library header,
+    #patchstorage-pedalboards-library header,
+    #patchstorage-hub header,
     #pedalboards-library header,
     #file-manager-library header,
     #sfzbuilder-library header {
@@ -23,6 +25,7 @@
     #bank-library #bank-edit,
     #file-manager-library,
     #patchstorage-library .patchstorage-plugins,
+    #patchstorage-pedalboards-library .patchstorage-plugins,
     #cloud-plugins-library .cloud-plugins {
         background-image: none !important;
     }

html/css/less/main.less

@@ -570,7 +570,7 @@ hr.dotted {
   float: right;
   font-size: 21px;
 }
-#cloud_update_all, #patchstorage_update_all {
+#cloud_update_all, #patchstorage_update_all, #patchstorage_pedalboards_update_all {
   background-image: url(../img/icons/36/update.png);
   float: right;
   font-size: 21px;
@@ -582,6 +582,8 @@ header#pedalboard-actions,
 #bank-library header,
 #cloud-plugins-library header,
 #patchstorage-library header,
+#patchstorage-pedalboards-library header,
+#patchstorage-hub header,
 #pedalboards-library header,
 #file-manager-library header,
 #sfzbuilder-library header {

html/css/less/patchstorage.less

@@ -1,4 +1,6 @@
-#patchstorage-library {
+#patchstorage-library,
+#patchstorage-pedalboards-library,
+#patchstorage-hub {
   background: #2c2c2c;
   bottom: 45px;
   left: 0;