From 37bd3bc70f598e9731694830628fe4420926d2c8 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:50:51 -0500 Subject: [PATCH 01/56] data: rewrite legacy converter with packet matching + add combine_csvs The legacy capture .txt converter previously emitted one row per packet with (device_id, ticks, data) columns -- not directly trainable. Rewrite it to deque-match the 3 SensorBand banks against LASK5 label packets by recency, producing the standard 12-sensor + 4-label paired-CSV shape plus per-stream timestamp columns. Dedup back-to-back identical records. Also lift combine_csvs() into this module so cli `train` and the web `/api/train` route can share one row-wise concatenation helper. dataset.py: detect_columns() now excludes any column with "Timestamp" in its name so the new Sensor_Timestamp / Label_Timestamp columns don't get fed to the model as features. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/data/converter.py | 102 ++++++++++++++++++++++++---- pc/src/openmuscle/data/dataset.py | 8 ++- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/pc/src/openmuscle/data/converter.py b/pc/src/openmuscle/data/converter.py index f159d2b..c31c39c 100644 --- a/pc/src/openmuscle/data/converter.py +++ b/pc/src/openmuscle/data/converter.py @@ -1,16 +1,28 @@ -"""Convert legacy capture formats to standard CSV.""" +"""Convert legacy capture formats to standard CSV with packet matching.""" import ast import csv import os +from collections import deque from pathlib import Path +# Legacy device IDs for SensorBand (3 banks × 4 sensors = 12 total) +SENSOR_IDS = ['OM-SB-V1-C.0', 'OM-SB-V1-C.1', 'OM-SB-V1-C.2'] +LABEL_ID = 'OM-LASK5' + + def convert_legacy_capture(input_path: str, output_path: str) -> int: - """Convert a legacy capture_*.txt file to standard CSV format. + """Convert a legacy capture_*.txt file to matched CSV format. + + Matches sensor packets from 3 SensorBand banks with LASK5 label packets + by temporal proximity, producing rows with 12 sensor values + 4 labels. Legacy format: one Python dict repr per line, e.g.: - {'id': 'OM-LASK5', 'ticks': 164587, 'time': (2000, 1, 1, ...), 'data': [-30, -35, -30, -37]} + {'id': 'OM-SB-V1-C.0', 'ticks': 164587, 'data': [2686.1, 2926.1, 2519.7, 2653.1], ...} + + Output CSV columns: + Sensor_0..Sensor_11, Sensor_Timestamp, Label_0..Label_3, Label_Timestamp Args: input_path: path to legacy .txt capture file @@ -20,11 +32,19 @@ def convert_legacy_capture(input_path: str, output_path: str) -> int: Number of rows written """ Path(output_path).parent.mkdir(parents=True, exist_ok=True) + + buffers = {sid: deque(maxlen=5) for sid in SENSOR_IDS} + buffers[LABEL_ID] = deque(maxlen=5) + + header = ([f"Sensor_{i}" for i in range(12)] + ['Sensor_Timestamp'] + + [f"Label_{i}" for i in range(4)] + ['Label_Timestamp']) + rows_written = 0 + last_record = None with open(input_path, "r") as f_in, open(output_path, "w", newline="") as f_out: writer = csv.writer(f_out) - header_written = False + writer.writerow(header) for line in f_in: line = line.strip() @@ -38,17 +58,71 @@ def convert_legacy_capture(input_path: str, output_path: str) -> int: if not isinstance(pkt, dict) or "data" not in pkt: continue - device_id = pkt.get("id", "unknown") - ticks = pkt.get("ticks", 0) - data = pkt["data"] + pkt_id = pkt.get("id") + if pkt_id not in buffers: + continue + + buffers[pkt_id].append(pkt) + + # Try to match: need at least one packet from each sensor bank + labels + if not all(buffers[sid] for sid in SENSOR_IDS): + continue + if not buffers[LABEL_ID]: + continue + + # Combine latest from each sensor bank + sensor_values = [] + sensor_times = [] + for sid in SENSOR_IDS: + latest = buffers[sid][-1] + sensor_values.extend(latest.get('data', [])) + sensor_times.append(latest.get('rec_time', 0)) + + label_pkt = buffers[LABEL_ID][-1] + labels = label_pkt.get('data', []) + label_time = label_pkt.get('rec_time', 0) + sensor_time = min(sensor_times) - if not header_written: - n = len(data) - header = ["device_id", "ticks"] + [f"value_{i}" for i in range(n)] - writer.writerow(header) - header_written = True + record = sensor_values + [sensor_time] + labels + [label_time] - writer.writerow([device_id, ticks] + data) - rows_written += 1 + # Skip duplicate records (same data as last write) + if record == last_record: + continue + + if len(record) == 18: + writer.writerow(record) + rows_written += 1 + last_record = record return rows_written + + +def combine_csvs(csv_paths: list[str], output_path: str) -> int: + """Concatenate multiple CSVs (same schema) into one file. + + Args: + csv_paths: list of CSV file paths to combine + output_path: path for the combined output CSV + + Returns: + Total number of data rows written + """ + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + total = 0 + header_written = False + + with open(output_path, "w", newline="") as f_out: + writer = csv.writer(f_out) + + for csv_path in csv_paths: + with open(csv_path, "r") as f_in: + reader = csv.reader(f_in) + header = next(reader) + if not header_written: + writer.writerow(header) + header_written = True + for row in reader: + writer.writerow(row) + total += 1 + + return total diff --git a/pc/src/openmuscle/data/dataset.py b/pc/src/openmuscle/data/dataset.py index 2b48fe9..b23d951 100644 --- a/pc/src/openmuscle/data/dataset.py +++ b/pc/src/openmuscle/data/dataset.py @@ -19,8 +19,12 @@ def detect_columns(df: pd.DataFrame) -> tuple[list[str], list[str]]: Returns: (sensor_columns, label_columns) """ - sensor_cols = [c for c in df.columns if c.startswith("R") or c.startswith("Sensor_")] - label_cols = [c for c in df.columns if c.startswith("label_") or c.startswith("Label_")] + sensor_cols = [c for c in df.columns + if (c.startswith("R") or c.startswith("Sensor_")) + and "Timestamp" not in c] + label_cols = [c for c in df.columns + if (c.startswith("label_") or c.startswith("Label_")) + and "Timestamp" not in c] if not sensor_cols: raise ValueError("No sensor columns found (expected R*C* or Sensor_* prefix)") From 49ec62321293e2e9f5f38d7e3283c5a72ebf2f3b Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:00 -0500 Subject: [PATCH 02/56] web: hand forwarder -- thumb on ch1, piston-reverse for LASK5 parity Rewrite _forward_to_hand() to match the OpenHand firmware's anatomical channel layout (FINGER_CHANNELS = [1, 3, 5, 7, 9] -> thumb, index, middle, ring, pinky). The previous implementation appended joystick X as the 5th angle, which lined up with channel 9 (pinky) instead of channel 1 (thumb) -- the hand was getting "thumb=predicted_pinky" and "pinky=joystick_x", silently inverted. Also reverse the piston order (P4..P1) on the way out so the PC path matches the LASK5 ESP-NOW path, whose default 'L5' device config has reverse=True. Without this reversal the same prediction array drives different fingers depending on whether the packet came via the model or via direct ESP-NOW from the LASK5. Add rate-limited logs (first hit + every 500th) so the operator can verify forwarding is actually happening without strace. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/state.py | 84 +++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/pc/src/openmuscle/web/state.py b/pc/src/openmuscle/web/state.py index 4dde98a..83e7d10 100644 --- a/pc/src/openmuscle/web/state.py +++ b/pc/src/openmuscle/web/state.py @@ -400,24 +400,22 @@ def _write_jsonl(stream: Optional[IO], pkt: OpenMusclePacket): def _forward_to_hand(self, pred: list): """Send the prediction to the robot hand as a `PC,...` UDP datagram. - Builds 5 servo angles in 0..179 from the 4 piston predictions (assumed - normalized 0..1; clamped) plus the most recent LASK5 joystick X as the - 5th. The hand's `'PC'` device config uses linear 0..179 -> 0..179 - mapping, so values land directly on servo angles. + Builds 5 servo angles in 0..179. Channel order on the hand + (FINGER_CHANNELS = [1, 3, 5, 7, 9]) is anatomically: + channel 1 -> thumb + channel 3 -> index + channel 5 -> middle + channel 7 -> ring + channel 9 -> pinky + + The LASK5 has 4 pistons (the 4 closing fingers) and a joystick. + We map joystick X -> thumb, pistons 0..3 -> index..pinky. + The hand's 'PC' device config uses linear 0..179 -> 0..179, so values + land directly on servo angles. """ - # Pistons -> 0..179, assuming model output is normalized 0..1. - # Anything else gets clamped, which is the right failure mode -- - # bracelet finger goes to extreme rather than 4000-degree angle. - angles = [] - for v in pred[:4]: - try: - v = max(0.0, min(1.0, float(v))) - except Exception: - v = 0.0 - angles.append(int(v * 179)) - - # 5th finger = joystick X from the most recent LASK5 packet. Range - # 0..4095 -> 0..179. Default to center (90) if no LASK5 has been seen. + # Thumb (channel 1) = joystick X from the most recent LASK5 packet. + # Range 0..4095 -> 0..179. Default to center (90) if no LASK5 has been + # seen yet (so the thumb sits in a neutral pose instead of slamming open). joy_x = None for d in self.devices.values(): if d.device_type == "lask5" and d.last_joystick: @@ -425,23 +423,57 @@ def _forward_to_hand(self, pred: list): if isinstance(jx, (int, float)): joy_x = jx break - if joy_x is None: - angles.append(90) - else: - angles.append(max(0, min(179, int((joy_x / 4095.0) * 179)))) + thumb_angle = 90 if joy_x is None else max(0, min(179, int((joy_x / 4095.0) * 179))) + + # Index..pinky (channels 3, 5, 7, 9) from pistons 0..3. + # Model output is assumed normalized 0..1; anything else gets clamped, + # which is the right failure mode -- finger goes to extreme rather + # than commanding a 4000-degree servo angle. + finger_angles = [] + for v in pred[:4]: + try: + v = max(0.0, min(1.0, float(v))) + except Exception: + v = 0.0 + finger_angles.append(int(v * 179)) + + # The hand's 'PC' device config has reverse=False, but the LASK5's + # native ESPNow path uses the 'default' / 'L5' config (reverse=True) + # which flips the piston order before mapping to FINGER_CHANNELS. + # To match that mapping from our PC path, we reverse the pistons + # ourselves: P1 -> index, P2 -> middle, P3 -> ring, P4 -> pinky. + # (Documented in DEVICES of the hand firmware, archived 2026-05-14.) + angles = [thumb_angle] + finger_angles[::-1] # [thumb, P4, P3, P2, P1] # Build the CSV the hand expects: 'PC,a1,a2,a3,a4,a5' payload = ("PC," + ",".join(str(a) for a in angles)).encode("utf-8") + # Rate-limited log so we can SEE whether forwarding is working. + # First time + every 500th packet: log success/failure to the buffer + # so the operator can debug without strace. + self._hand_forward_count = getattr(self, "_hand_forward_count", 0) + 1 + log_now = (self._hand_forward_count == 1 + or self._hand_forward_count % 500 == 0) try: if self._hand_sock is None: self._hand_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._hand_sock.setblocking(False) - self._hand_sock.sendto(payload, self.hand_target) - except Exception: - # Non-fatal: hand might be offline / on a different subnet. - # We don't spam logs since this fires per FlexGrid packet. - pass + n = self._hand_sock.sendto(payload, self.hand_target) + if log_now: + self.log_buffer.info("inference", + "hand forward #{}: sent {} bytes to {}:{} -> {!r}".format( + self._hand_forward_count, n, + self.hand_target[0], self.hand_target[1], + payload.decode("utf-8", errors="replace"))) + except Exception as e: + # Always log the first failure so the operator sees it; rate-limit + # subsequent ones (every 500) so we don't spam. + if log_now or not getattr(self, "_hand_forward_error_logged", False): + self.log_buffer.warn("inference", + "hand forward #{} FAILED: {} ({!r}) -> target={}".format( + self._hand_forward_count, type(e).__name__, str(e), + self.hand_target)) + self._hand_forward_error_logged = True async def _broadcast_latest_frames(self): """Push the latest frame for each device to all WS clients.""" From cd2ec75f23e2088fc752d3584b5c321389318c99 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:09 -0500 Subject: [PATCH 03/56] web: file-manager reveal + retroactive session<->capture linking Two additive endpoints for the Captures and Sessions panels: POST /api/reveal {name?} Opens captures_dir in the OS file manager. With a capture name, highlights that specific .csv inside its folder (Explorer /select on Windows, open -R on macOS, xdg-open on Linux). Whitelist-guarded via state.capture_path(). POST/DELETE /api/sessions/{id}/captures{/name} The "I forgot to start a session before recording" recovery path. Bulk-add or remove existing captures from a session after the fact. Updates both the session JSON's `captures` list AND the capture's .meta.json (tag `session:` + auto.session_id) so the captures filter and session expansion stay consistent. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/app.py | 146 +++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/pc/src/openmuscle/web/app.py b/pc/src/openmuscle/web/app.py index 68f1c7f..ba22a3d 100644 --- a/pc/src/openmuscle/web/app.py +++ b/pc/src/openmuscle/web/app.py @@ -9,6 +9,9 @@ # string and falls back to treating the param as a query string field). import asyncio +import shutil +import subprocess +import sys from contextlib import asynccontextmanager from pathlib import Path from typing import Optional @@ -24,6 +27,48 @@ STATIC_DIR = Path(__file__).parent / "static" +def _reveal_path_in_file_manager(path: Path, select_file: bool) -> None: + """Open `path` in the OS file manager. If `select_file=True` and the + platform supports it, highlight the file inside its parent folder + rather than just opening the folder. Raises RuntimeError on failure. + + Whitelist-guarded by the caller: this function does NOT verify that + `path` is inside captures_dir. That check happens in the route. + """ + if not path.exists(): + raise RuntimeError(f"Path does not exist: {path}") + + # Explorer / Finder / xdg-open all need *absolute* paths -- they don't + # inherit our CWD predictably and a relative path like + # "data/raw/merged/foo.csv" silently fails with "location not found". + path = path.resolve() + + try: + if sys.platform.startswith("win"): + if select_file and path.is_file(): + # explorer /select,"C:\full\path\file.csv" -- highlights file + subprocess.Popen(["explorer", f"/select,{path}"]) + else: + # Open the folder itself + folder = path if path.is_dir() else path.parent + subprocess.Popen(["explorer", str(folder)]) + elif sys.platform == "darwin": + if select_file and path.is_file(): + subprocess.Popen(["open", "-R", str(path)]) + else: + folder = path if path.is_dir() else path.parent + subprocess.Popen(["open", str(folder)]) + else: + # Linux / other -- xdg-open only opens directories cleanly + opener = shutil.which("xdg-open") or shutil.which("gio") + if opener is None: + raise RuntimeError("No file-manager opener found (xdg-open / gio)") + folder = path if path.is_dir() else path.parent + subprocess.Popen([opener, str(folder)]) + except Exception as e: + raise RuntimeError(f"Failed to open file manager: {e}") + + def create_app(udp_port: int = 3141, captures_dir: Optional[str] = None, model_path: Optional[str] = None, hand_target: Optional[tuple] = None) -> FastAPI: @@ -168,6 +213,31 @@ async def download_capture(name: str): raise HTTPException(status_code=404, detail="Capture not found") return FileResponse(p, filename=p.name, media_type="text/csv") + class RevealBody(BaseModel): + # If empty/None -> just open captures_dir. Otherwise must be a + # capture name whitelisted by state.capture_path(). + name: Optional[str] = None + + @app.post("/api/reveal") + async def reveal_in_folder(body: RevealBody): + """Open the captures folder (and optionally highlight a specific + capture) in the OS file manager. Local-only convenience; the server + is intended for localhost use.""" + if body.name: + p = state.capture_path(body.name) + if p is None: + raise HTTPException(status_code=404, detail="Capture not found") + target = p + select = True + else: + target = state.captures_dir + select = False + try: + _reveal_path_in_file_manager(target, select_file=select) + except RuntimeError as e: + raise HTTPException(status_code=500, detail=str(e)) + return {"opened": str(target), "selected": select} + @app.delete("/api/captures/{name}") async def delete_capture(name: str): ok = state.delete_capture(name) @@ -358,6 +428,82 @@ async def delete_session_endpoint(session_id: str, unlink_captures: bool = True) raise HTTPException(status_code=404, detail="Session not found") return {"deleted": session_id} + # ----- REST: retroactive session<->capture linking ----- + # + # A capture made *outside* an active session can be added to one + # afterwards, and vice versa removed. This is the "I forgot to start a + # session before recording" recovery path. We update both: + # 1. the session JSON's `captures` list (authoritative) + # 2. the capture's meta sidecar (tag `session:` + auto.session_id) + # so the captures-panel filter, the past-sessions expansion, and any + # future export all agree on which session a capture belongs to. + + class LinkCapturesBody(BaseModel): + capture_names: list[str] # bulk add + + @app.post("/api/sessions/{session_id}/captures") + async def add_captures_to_session(session_id: str, body: LinkCapturesBody): + s = state.get_session(session_id) + if s is None: + raise HTTPException(status_code=404, detail="Session not found") + + tag = "session:" + session_id + added, skipped = [], [] + for name in body.capture_names: + if state.capture_path(name) is None: + skipped.append({"name": name, "reason": "capture not found"}) + continue + if name in s.get("captures", []): + skipped.append({"name": name, "reason": "already in session"}) + continue + try: + state.link_capture_to_session(session_id, name) + # Update the capture's meta so the tag-based filter + any + # future export sees this capture as part of the session. + meta = state.read_capture_meta(name) or {} + tags = list(meta.get("tags") or []) + if tag not in tags: + tags.append(tag) + state.write_capture_meta(name, { + "tags": tags, + "auto": {"session_id": session_id}, + }) + added.append(name) + except Exception as e: + skipped.append({"name": name, "reason": str(e)}) + + return { + "added": added, + "skipped": skipped, + "session": state.get_session(session_id), + } + + @app.delete("/api/sessions/{session_id}/captures/{capture_name}") + async def remove_capture_from_session(session_id: str, capture_name: str): + s = state.get_session(session_id) + if s is None: + raise HTTPException(status_code=404, detail="Session not found") + if capture_name not in s.get("captures", []): + raise HTTPException(status_code=404, detail="Capture not in session") + try: + state.unlink_capture_from_session(session_id, capture_name) + # Strip the session tag + clear auto.session_id, but ONLY for this + # session (leave any other 'session:xxx' tags alone -- though by + # the data model a capture should only ever belong to one session). + tag = "session:" + session_id + meta = state.read_capture_meta(capture_name) or {} + new_tags = [t for t in (meta.get("tags") or []) if t != tag] + state.write_capture_meta(capture_name, { + "tags": new_tags, + "auto": {"session_id": None}, + }) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + return { + "removed": capture_name, + "session": state.get_session(session_id), + } + return app From ab8be83e55075f3cc02861b6c05cb6e6198d21b2 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:19 -0500 Subject: [PATCH 04/56] web: Studio UI redesign with pipeline strip + stage-based layout Rebrand the web UI from "OpenMuscle Live" to "OpenMuscle Studio" and restructure the page around the data-pipeline narrative: SENSOR -> LABEL -> CAPTURE -> MODEL -> HAND A topbar pipeline-strip shows live status for each stage with click- to-scroll anchors. Body splits into numbered stages: (1) Live -- heatmap + GT-vs-Predicted comparator hero, (2) Capture, (3) Models, (4) Output. The Devices list collapses into a thin left rail. styles.css gets the matching grid-template + new pipe-pill / stage-* classes. app.js adds renderPipelinePills() (called every WS tick) and moves the per-panel renderers into the new stage containers. No behavior changes -- same WS contract, same REST surface. Pure restructure + visual rework. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/static/app.js | 387 +++++++++++++++++++- pc/src/openmuscle/web/static/index.html | 420 +++++++++++++-------- pc/src/openmuscle/web/static/styles.css | 463 ++++++++++++++++++++++-- 3 files changed, 1091 insertions(+), 179 deletions(-) diff --git a/pc/src/openmuscle/web/static/app.js b/pc/src/openmuscle/web/static/app.js index 474637c..7e86cc9 100644 --- a/pc/src/openmuscle/web/static/app.js +++ b/pc/src/openmuscle/web/static/app.js @@ -18,10 +18,31 @@ const selStatus = document.getElementById('captures-sel-status'); const checkAll = document.getElementById('captures-check-all'); const modelsBody = document.getElementById('models-body'); const modelsCount = document.getElementById('models-count'); +const openFolderBtn = document.getElementById('captures-open-folder'); + +// Ask the server to open the captures folder in the OS file manager. +// If `name` is given, highlight that capture file inside the folder. +async function revealCaptureFolder(name) { + try { + const r = await fetch('/api/reveal', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({name: name || null}), + }); + if (!r.ok) throw new Error(await readError(r)); + } catch (e) { + alert('Could not open folder: ' + e.message); + } +} + +if (openFolderBtn) { + openFolderBtn.onclick = () => revealCaptureFolder(null); +} // Per-user pick preferences that survive a refresh const STORE_SENSOR = 'om.sensor_device_id'; const STORE_LABEL = 'om.label_device_id'; +const STORE_HAND = 'om.hand_target'; // last successfully-applied "host:port" — auto-restored on next launch // Set of capture filenames currently checked in the table const selectedCaptures = new Set(); @@ -41,6 +62,11 @@ function connectWS() { ws.onopen = () => { wsStatus.textContent = 'connected'; wsStatus.className = 'badge online'; + // Re-arm the hand-target auto-restore: every fresh WS connect (which + // includes server restarts) gets a chance to re-apply the saved hand + // target. Otherwise the operator has to remember to click Apply + // after every `openmuscle web` restart. + handTargetRestoreAttempted = false; }; ws.onclose = () => { wsStatus.textContent = 'disconnected'; @@ -84,6 +110,11 @@ function handleTick(msg) { const lask = lastDevices.find(d => d.device_type === 'lask5'); renderLask(lask); renderInference(msg.inference); + // Comparator + top-bar pipeline strip are Studio-shell additions. + // They derive everything from the per-tick snapshot, so they update + // in lockstep with the underlying bars and the WS message. + renderResiduals(lask, msg.inference); + renderPipelinePills(msg, lask); } // ---------- device list ---------- @@ -370,6 +401,7 @@ function renderActiveSession() { ${armBit}${subj} · ${s.capture_count || 0} captures · ${formatUptime(dur)}${gestures}
+
@@ -377,6 +409,8 @@ function renderActiveSession() { ${s.notes ? `
${escapeHtml(s.notes)}
` : ''} `; document.getElementById('session-end-btn').onclick = endSession; + const addBtn = document.getElementById('active-session-add-btn'); + if (addBtn) addBtn.onclick = () => openLinkModal(activeSession); sessionStartBtn.disabled = true; sessionStartBtn.title = 'End the current session before starting a new one'; capturesFilterLabel.textContent = `· filtered to ${s.name || s.id}`; @@ -400,6 +434,137 @@ async function refreshPastSessions() { } catch (e) { /* best-effort */ } } +// Sessions whose capture list is currently expanded in the UI. Persisted +// across re-renders (refreshPastSessions can fire on its own) so a poll +// doesn't collapse what the user just opened. +const expandedSessions = new Set(); + +// ---------- Add-captures-to-session picker modal ---------- +// +// Lets the operator retroactively assign past recordings (made without an +// active session) to a session. The picker shows every capture NOT +// currently linked to the target session, with checkboxes for bulk add. +// +// Wires up: +// - "+ Add captures" button in each past-session card +// - "×" remove button on each capture in the expanded view + +const linkModal = document.getElementById('link-modal'); +const linkSessionName = document.getElementById('link-session-name'); +const linkCaptureList = document.getElementById('link-capture-list'); +const linkAddBtn = document.getElementById('link-add-btn'); +let linkSessionId = null; // current session being edited +const linkSelected = new Set(); // capture names currently checked + +function openLinkModal(session) { + linkSessionId = session.id; + linkSelected.clear(); + linkSessionName.textContent = session.name || session.id; + linkAddBtn.disabled = true; + linkAddBtn.textContent = 'Add 0 captures'; + linkCaptureList.innerHTML = '
Loading captures…
'; + linkModal.classList.add('open'); + linkModal.setAttribute('aria-hidden', 'false'); + + // Fetch the full capture list, filter out ones already in this session. + fetch('/api/captures') + .then(r => r.ok ? r.json() : Promise.reject('fetch failed')) + .then(list => { + const alreadyLinked = new Set(session.captures || []); + const candidates = list.filter(c => !alreadyLinked.has(c.name)); + if (!candidates.length) { + linkCaptureList.innerHTML = '
All captures are already in this session.
'; + return; + } + // Render rows with checkbox + name + meta summary + (if linked + // to a different session) an annotation so the operator doesn't + // accidentally yank a capture out of another session. + linkCaptureList.innerHTML = candidates.map(c => { + const meta = c.meta || {}; + const otherSession = (meta.tags || []).find(t => t.startsWith('session:')); + const otherNote = otherSession + ? `⚠ ${escapeHtml(otherSession)}` + : ''; + const kb = (c.size_bytes / 1024).toFixed(1); + return ``; + }).join(''); + linkCaptureList.querySelectorAll('input[type=checkbox]').forEach(cb => { + cb.onchange = () => { + if (cb.checked) linkSelected.add(cb.dataset.name); + else linkSelected.delete(cb.dataset.name); + const n = linkSelected.size; + linkAddBtn.disabled = (n === 0); + linkAddBtn.textContent = `Add ${n} capture${n === 1 ? '' : 's'}`; + }; + }); + }) + .catch(err => { + linkCaptureList.innerHTML = '
Could not load captures.
'; + console.warn('link picker fetch:', err); + }); +} + +function closeLinkModal() { + linkModal.classList.remove('open'); + linkModal.setAttribute('aria-hidden', 'true'); + linkSessionId = null; + linkSelected.clear(); +} + +linkModal.querySelectorAll('[data-close]').forEach(el => { + el.addEventListener('click', closeLinkModal); +}); +document.addEventListener('keydown', e => { + if (e.key === 'Escape' && linkModal.classList.contains('open')) closeLinkModal(); +}); + +linkAddBtn.onclick = async () => { + if (!linkSessionId || linkSelected.size === 0) return; + linkAddBtn.disabled = true; + linkAddBtn.textContent = 'Adding…'; + try { + const r = await fetch(`/api/sessions/${encodeURIComponent(linkSessionId)}/captures`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({capture_names: [...linkSelected]}), + }); + if (!r.ok) throw new Error(await readError(r)); + const result = await r.json(); + if ((result.skipped || []).length) { + // Surface skips inline -- e.g. "already in another session" + console.warn('some captures skipped:', result.skipped); + } + closeLinkModal(); + await refreshPastSessions(); + await refreshCaptures(); + } catch (e) { + alert('Add failed: ' + (e.message || e)); + linkAddBtn.disabled = false; + const n = linkSelected.size; + linkAddBtn.textContent = `Add ${n} capture${n === 1 ? '' : 's'}`; + } +}; + +async function removeCaptureFromSession(sessionId, captureName) { + if (!confirm(`Remove ${captureName} from this session?\n(The capture file itself stays — just the link is cleared.)`)) return; + try { + const r = await fetch( + `/api/sessions/${encodeURIComponent(sessionId)}/captures/${encodeURIComponent(captureName)}`, + {method: 'DELETE'} + ); + if (!r.ok) throw new Error(await readError(r)); + await refreshPastSessions(); + await refreshCaptures(); + } catch (e) { + alert('Remove failed: ' + (e.message || e)); + } +} + function renderPastSessions() { if (!pastSessions.length) { pastSessionsList.innerHTML = '
No past sessions yet.
'; @@ -408,21 +573,59 @@ function renderPastSessions() { pastSessionsList.innerHTML = pastSessions.map(s => { const dur = (s.ended_at && s.started_at) ? Math.floor(s.ended_at - s.started_at) : null; const armBit = s.arm ? escapeHtml(s.arm) + ' arm' : '—'; - const captures = s.capture_count || (s.captures || []).length; - return `
-
+ const captureList = Array.isArray(s.captures) ? s.captures : []; + const captureCount = s.capture_count != null ? s.capture_count : captureList.length; + const isOpen = expandedSessions.has(s.id); + const caret = captureList.length ? (isOpen ? '▾' : '▸') : '·'; + // The captures sub-list is a sibling div, toggled by .hidden. We + // render it eagerly (with .hidden if closed) so the open/close + // animation isn't required and so screen readers can find it. + const capturesInner = captureList.length + ? captureList.map(name => ` +
  • + ${escapeHtml(name)} + + + + download + + +
    • `).join('') + : '
  • No captures linked to this session.
    • '; + + return `
    +
    + ${caret} ${escapeHtml(s.name || s.id)} - ${armBit} · ${escapeHtml(s.subject || '—')} · ${captures} captures${dur != null ? ' · ' + formatUptime(dur) : ''} + ${armBit} · ${escapeHtml(s.subject || '—')} · ${captureCount} captures${dur != null ? ' · ' + formatUptime(dur) : ''}
    +
    ${s.notes ? `
    ${escapeHtml(s.notes)}
    ` : ''} +
      ${capturesInner}
    `; }).join(''); + + // Stop session-action buttons from triggering the row-toggle handler + pastSessionsList.querySelectorAll('.session-actions button').forEach(btn => { + btn.addEventListener('click', e => e.stopPropagation()); + }); + + // Toggle expand/collapse when the session header row is clicked + pastSessionsList.querySelectorAll('[data-toggle-session]').forEach(head => { + head.onclick = () => { + const sid = head.dataset.toggleSession; + if (expandedSessions.has(sid)) expandedSessions.delete(sid); + else expandedSessions.add(sid); + renderPastSessions(); + }; + }); + pastSessionsList.querySelectorAll('button[data-delete-session]').forEach(btn => { btn.onclick = async () => { const sid = btn.dataset.deleteSession; @@ -435,6 +638,34 @@ function renderPastSessions() { } catch (e) { alert('Delete failed: ' + e.message); } }; }); + + // Per-capture actions inside the expanded list + pastSessionsList.querySelectorAll('button[data-reveal-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + revealCaptureFolder(btn.dataset.revealCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-edit-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + openMetaModal(btn.dataset.editCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-unlink-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + removeCaptureFromSession(btn.dataset.fromSession, btn.dataset.unlinkCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-add-to-session]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + const sid = btn.dataset.addToSession; + const session = pastSessions.find(s => s.id === sid); + if (session) openLinkModal(session); + }; + }); } pastSessionsToggle.onclick = () => { @@ -643,6 +874,7 @@ function renderCaptures(list) { ${escapeHtml(date)} + download @@ -669,6 +901,9 @@ function renderCaptures(list) { capturesBody.querySelectorAll('button[data-edit]').forEach(btn => { btn.onclick = () => openMetaModal(btn.dataset.edit); }); + capturesBody.querySelectorAll('button[data-reveal]').forEach(btn => { + btn.onclick = () => revealCaptureFolder(btn.dataset.reveal); + }); updateSelectionStatus(); } @@ -1035,7 +1270,43 @@ function renderInference(inf) { }); } +// One-shot: if the server has no hand_target on first snapshot but we have +// one saved in localStorage, auto-apply it so launching `openmuscle web` +// doesn't lose the address every time. UDP-only (the only protocol we +// support); port defaults to 3145. +let handTargetRestoreAttempted = false; +function maybeRestoreHandTarget(inf) { + if (handTargetRestoreAttempted) return; + if (!inf) return; // wait for first inference snapshot + handTargetRestoreAttempted = true; // one-shot regardless of outcome + if (inf.hand_target) return; // server already has one (e.g. --hand on CLI) + const saved = localStorage.getItem(STORE_HAND); + if (!saved) return; + autoApplyHandTarget(saved); +} + +async function autoApplyHandTarget(raw) { + let host = raw, port = 3145; + if (raw.includes(':')) { + const idx = raw.lastIndexOf(':'); + host = raw.slice(0, idx); + const portN = parseInt(raw.slice(idx + 1), 10); + if (Number.isFinite(portN) && portN > 0 && portN < 65536) port = portN; + } + try { + await fetch('/api/inference/hand', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ host, port }), + }); + } catch (e) { + console.warn('hand target auto-restore failed', e); + } +} + function renderInferenceControls(inf) { + maybeRestoreHandTarget(inf); + const hasModel = !!(inf && inf.model); const enabled = !!(inf && inf.enabled); @@ -1111,6 +1382,10 @@ async function applyHandTarget() { body: JSON.stringify({ host, port }), }); if (!r.ok) throw new Error(await readError(r)); + // Persist so next launch auto-restores. Clear on explicit empty + // so the operator can "forget" the target deliberately. + if (host) localStorage.setItem(STORE_HAND, raw); + else localStorage.removeItem(STORE_HAND); // Force the snapshot side to refresh by clearing the cache so the // next tick syncs the (possibly normalized) value back into the input. lastSnapshotHand = undefined; @@ -1124,6 +1399,110 @@ inferHandInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') applyHandTarget(); }); +// ---------- Studio shell: comparator residuals (Δ) ---------- + +// Compute per-piston residual (predicted - ground_truth) and write it into +// the .delta-row elements in the comparator. Color-codes by direction so +// the operator can see at a glance whether the model is over- or under- +// shooting each finger. +// +// CLOSE_THRESHOLD picked at 0.05 (5% of the 0..1 scale) — below that, the +// difference is below the noise floor of the LASK5 measurement itself. +const RESIDUAL_CLOSE_THRESHOLD = 0.05; + +function renderResiduals(laskDev, inf) { + const deltaRows = document.querySelectorAll('#comparator-deltas .delta-row'); + if (!deltaRows.length) return; + const gt = laskDev && Array.isArray(laskDev.values) ? laskDev.values : null; + const pred = inf && Array.isArray(inf.piston_values) ? inf.piston_values : null; + + deltaRows.forEach((row, i) => { + const valEl = row.querySelector('.delta-val'); + row.classList.remove('over', 'under', 'close'); + if (!gt || !pred || i >= gt.length || i >= pred.length) { + if (valEl) valEl.textContent = '--'; + return; + } + const g = pistonFraction(gt[i]); + const p = pistonFraction(pred[i]); + const d = p - g; + valEl.textContent = (d >= 0 ? '+' : '') + d.toFixed(2); + if (Math.abs(d) < RESIDUAL_CLOSE_THRESHOLD) row.classList.add('close'); + else if (d > 0) row.classList.add('over'); + else row.classList.add('under'); + }); +} + +// ---------- Studio shell: top-bar pipeline status strip ---------- + +// Set a pipe-pill's status + value text. State controls colour: +// 'live' -- blue accent (data flowing) +// 'ok' -- green (idle but healthy) +// 'warn' -- orange +// 'bad' -- red +// '' -- neutral grey +function setPipePill(id, state, valText) { + const el = document.getElementById(id); + if (!el) return; + el.classList.remove('ok', 'warn', 'bad', 'live'); + if (state) el.classList.add(state); + const valEl = el.querySelector('.pipe-val'); + if (valEl) valEl.textContent = valText; +} + +function renderPipelinePills(msg, laskDev) { + // SENSOR pill = the active flexgrid (the one driving the heatmap) + const dev = selectedDevice(); + if (dev && dev.device_type === 'flexgrid') { + const stale = dev.last_seen_age > 2.0; + setPipePill('pipe-sensor', stale ? 'warn' : 'live', `${dev.hz.toFixed(0)}Hz`); + } else { + setPipePill('pipe-sensor', '', '--'); + } + + // LABEL pill = LASK5 stream + if (laskDev) { + const stale = laskDev.last_seen_age > 2.0; + setPipePill('pipe-label', stale ? 'warn' : 'live', `${laskDev.hz.toFixed(0)}Hz`); + } else { + setPipePill('pipe-label', '', '--'); + } + + // CAPTURE pill + if (recordingState) { + const matchRate = recordingState.match_rate ?? 0; + const cls = matchRate < 0.5 ? 'bad' : (matchRate < 0.9 ? 'warn' : 'live'); + setPipePill('pipe-capture', cls, `REC ${recordingState.rows ?? 0}r`); + } else if (activeSession) { + setPipePill('pipe-capture', 'ok', `session: ${activeSession.name || activeSession.id}`); + } else { + setPipePill('pipe-capture', '', 'idle'); + } + + // MODEL pill + const inf = msg.inference; + if (inf && inf.model && inf.enabled) setPipePill('pipe-model', 'live', inf.model); + else if (inf && inf.model && !inf.enabled) setPipePill('pipe-model', 'ok', inf.model + ' (paused)'); + else setPipePill('pipe-model', '', 'none'); + + // HAND pill = UDP forwarding target + if (inf && inf.hand_target) setPipePill('pipe-hand', 'live', inf.hand_target); + else setPipePill('pipe-hand', '', 'off'); +} + +// ---------- Studio shell: diagnostics drawer ---------- + +const diagToggle = document.getElementById('diag-toggle'); +const diagBody = document.getElementById('diag-body'); +if (diagToggle && diagBody) { + diagToggle.onclick = () => { + const isHidden = diagBody.classList.toggle('hidden'); + diagToggle.setAttribute('aria-expanded', isHidden ? 'false' : 'true'); + diagToggle.textContent = (isHidden ? '▸' : '▾') + ' Diagnostics & logs'; + // Logs poll runs unconditionally; we just hide the DOM. Cheap. + }; +} + // ---------- utils ---------- function escapeHtml(s) { diff --git a/pc/src/openmuscle/web/static/index.html b/pc/src/openmuscle/web/static/index.html index b795c34..dc44e29 100644 --- a/pc/src/openmuscle/web/static/index.html +++ b/pc/src/openmuscle/web/static/index.html @@ -3,179 +3,292 @@ - OpenMuscle Live + OpenMuscle Studio -
    -

    OpenMuscle Live

    -
    +
    +
    +

    OpenMuscle Studio

    +
    + + + + +
    disconnected
    -
    -
    -

    Devices

    -
      -
    • Waiting for a device to send a packet…
      • -
    -
    +
    -
    -
    -

    Heatmap

    - -
    - -
    + +
    +
    +

    1 Live

    + what the band feels · what the model thinks +
    -
    -
    -

    LASK5 — Ground Truth

    - no device -
    -
    -
    P1
    --
    -
    P2
    --
    -
    P3
    --
    -
    P4
    --
    -
    -
    - joystick - - --, -- -
    -
    +
    + + -
    -
    -

    LASK Inference — Predicted

    - no model loaded -
    -
    -
    P1̂
    --
    -
    P2̂
    --
    -
    P3̂
    --
    -
    P4̂
    --
    -
    -
    - - - - -
    -
    + +
    +
    + +
    + +
    -
    -

    Record

    -
    - - - - + +
    +
    +

    Ground truth vs Predicted

    +
    + GT: no device + MODEL: no model loaded +
    +
    + +
    + +
    +
    P1
    --
    +
    P2
    --
    +
    P3
    --
    +
    P4
    --
    +
    + + +
    +
    Δ--
    +
    Δ--
    +
    Δ--
    +
    Δ--
    +
    + + +
    +
    P1̂
    --
    +
    P2̂
    --
    +
    P3̂
    --
    +
    P4̂
    --
    +
    +
    + + +
    + + --, -- +
    + + +
    + + predictions to a robot hand: see Stage 4 +
    +
    -
    -
    -
    -

    Sessions

    -
    - + +
    +
    +

    2 Capture

    + session + record · paired sensor/label rows go to disk +
    + +
    + +
    +
    + Session + +
    +
    +
    No active session — recordings won't be grouped. Click "New session" to start one.
    +
    +
    + + +
    +
    + Record +
    +
    + + + + +
    +
    -
    -
    No active session — recordings won't be grouped. Click "New session" to start one.
    -
    + +
    -
    -
    -

    Captures

    -
    - 0 selected - + +
    +
    +

    3 Data & Models

    + captures become models · audit honestly · iterate +
    + +
    + +
    +
    +

    Captures

    +
    + 0 selected + +
    +
    + + + + + + + + + + + + + + +
    NameMetaSizeModified
    No captures saved yet.
    +
    + + +
    + +
    +
    + + +
    +
    +

    Models

    + +
    + + + + + + + + + + + + + + +
    NameCreatedR²MSEFeatures × Labels
    No models trained yet.
    - - - - - - - - - - - - - - -
    NameMetaSizeModified
    No captures saved yet.
    -
    -
    -
    -

    Models

    - + +
    +
    +

    4 Output

    + where the predictions are sent +
    +
    + + + no hand target
    - - - - - - - - - - - - - - -
    NameCreatedR²MSEFeatures × Labels
    No models trained yet.
    -
    -
    -

    Logs

    -
    - - - -
    + +
    + +
    - Open-Muscle · FlexGrid · github + Open-Muscle · FlexGrid Studio · github
    - + - + + + +
    + +
    +
    +
    + ● real + ● predicted + drag to rotate +
    +
    + diff --git a/pc/src/openmuscle/web/static/styles.css b/pc/src/openmuscle/web/static/styles.css index 10d938e..d833aeb 100644 --- a/pc/src/openmuscle/web/static/styles.css +++ b/pc/src/openmuscle/web/static/styles.css @@ -1269,3 +1269,37 @@ footer { font-size: 12px; text-align: center; } + +/* ---- quest_hand 3D viewer (Studio Live stage) ---- + Shown in place of the LASK5 piston comparator when a quest_hand label + source is streaming. Toggled by app.js adding .hand-mode to .comparator; + the viewer's own visibility is also set imperatively by OMHandViewer + (inline display), which defers to these rules when shown. */ +.hand-viewer { + display: none; + flex-direction: column; + gap: 6px; +} +.comparator.hand-mode .comparator-row { display: none; } +.comparator.hand-mode .hand-viewer { display: flex; } + +.hand-viewer-canvas { + width: 100%; + height: 240px; + border-radius: 8px; + background: radial-gradient(ellipse at center, #11161f 0%, #0b0e14 100%); + border: 1px solid var(--border, #23262d); + touch-action: none; /* let pointer-drag rotate instead of scroll */ +} +.hand-viewer-canvas canvas { display: block; border-radius: 8px; } + +.hand-viewer-legend { + display: flex; + gap: 16px; + font-size: 12px; + color: var(--fg-faint, #8b96a8); + align-items: center; +} +.hand-viewer-legend .hv-real { color: #34d399; } +.hand-viewer-legend .hv-pred { color: #fbbf24; } +.hand-viewer-legend .hv-hint { margin-left: auto; opacity: 0.7; font-style: italic; } From 5c3a4959e2f12658bbccca6083103e9176590aea Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 07:18:36 -0500 Subject: [PATCH 47/56] tests: lock the snapshot contract the desktop hand viewer depends on The desktop Studio 3D hand viewer reads each device's flat `values` from the /ws/live snapshot. Add a test asserting a quest_hand device surfaces in _snapshot() as device_type 'quest_hand' with its full 25*7 flat joint vector, so a future snapshot refactor can't silently break the viewer's only data source. Suite now 32 passing. Co-Authored-By: turfptax-claude O4.8 --- pc/tests/test_quest_ingest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pc/tests/test_quest_ingest.py b/pc/tests/test_quest_ingest.py index 059aa9e..86dcaab 100644 --- a/pc/tests/test_quest_ingest.py +++ b/pc/tests/test_quest_ingest.py @@ -100,3 +100,29 @@ def test_repeated_packets_increment_packet_count(self): s.ingest_quest_packet({"joints": [_full_joint("wrist", offset=i * 0.01)]}) d = next(iter(s.devices.values())) assert d.packets_total == 5 + + +class TestSnapshotExposesQuestHand: + """The desktop Studio 3D hand viewer reads each device's flat `values` + from the /ws/live snapshot. Lock that contract: a quest_hand device must + surface in _snapshot() as device_type 'quest_hand' with its full flat + joint vector, so a future snapshot refactor can't silently break the + viewer's only data source. + """ + + def test_snapshot_has_quest_device_with_flat_values(self): + # _snapshot() touches the full inference machinery, so use a real + # AppState (its __init__ sets engine_status etc.) rather than the bare + # __new__ helper. The UDP listener is never started, so no socket binds. + import tempfile + with tempfile.TemporaryDirectory() as d: + s = AppState(udp_port=53997, captures_dir=d) + joints = [_full_joint(f"j{i}", i * 0.01) for i in range(25)] + s.ingest_quest_packet({"device_id": "quest-right", "handedness": "right", + "joints": joints}) + snap = s._snapshot() + quest = [dev for dev in snap["devices"] if dev["device_type"] == "quest_hand"] + assert len(quest) == 1 + # 25 joints * 7 floats -> the viewer slices [i*7 .. i*7+6] per joint. + assert len(quest[0]["values"]) == 25 * 7 + assert quest[0]["device_id"] == "quest-right" From ba618e3aed470bc317cd2542fcbe1aad28c8574e Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 19:21:48 -0500 Subject: [PATCH 48/56] web: hand viewer auto-framing + snapshot(), render visually verified Two upgrades to the desktop Studio 3D hand viewer: * Auto-framing: each update recomputes the real hand's centroid and fit radius, and the orbit camera frames that sphere (FOV-fit with margin) instead of a fixed 0.34m orbit around the wrist. Any hand size/pose now fills the viewport instead of wasting the lower half. * OMHandViewer.snapshot(): synchronous render + toDataURL PNG capture (works without preserveDrawingBuffer), for frame thumbnails and headless verification. Camera positioning is shared with the animate loop so a snapshot never renders from an unpositioned camera. Bump hand-viewer.js?v=4. Closes the "actual 3D RENDER is unverified" caveat from 242750e: drove the viewer with a synthetic 25-joint hand (real + perturbed predicted) in a live browser session; snapshot() returned a PNG showing both skeletons (green real, amber predicted) correctly framed. Co-Authored-By: turfptax-claude O4.8 --- pc/src/openmuscle/web/static/hand-viewer.js | 56 +++++++++++++++++---- pc/src/openmuscle/web/static/index.html | 2 +- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/pc/src/openmuscle/web/static/hand-viewer.js b/pc/src/openmuscle/web/static/hand-viewer.js index 1c8bb74..f548359 100644 --- a/pc/src/openmuscle/web/static/hand-viewer.js +++ b/pc/src/openmuscle/web/static/hand-viewer.js @@ -129,6 +129,24 @@ function layoutHand(flat, rig, outPositions) { const _realOut = Array.from({ length: N_JOINTS }, () => new THREE.Vector3()); const _predOut = Array.from({ length: N_JOINTS }, () => new THREE.Vector3()); +// Auto-framing target + fit radius, recomputed from the real hand each update +// so any hand size/pose fills the viewport (the hand extends up from the +// wrist, so looking at the wrist alone wastes the lower half of the view). +const _focus = new THREE.Vector3(); +let _fitRadius = 0.1; + +function computeFraming(pts) { + _focus.set(0, 0, 0); + for (let i = 0; i < N_JOINTS; i++) _focus.add(pts[i]); + _focus.multiplyScalar(1 / N_JOINTS); + let maxR = 0; + for (let i = 0; i < N_JOINTS; i++) { + const d = pts[i].distanceTo(_focus); + if (d > maxR) maxR = d; + } + _fitRadius = maxR || 0.1; +} + function resize() { if (!container || !renderer) return; const w = container.clientWidth || 320; @@ -138,18 +156,27 @@ function resize() { camera.updateProjectionMatrix(); } +// Place the orbit camera from the current yaw/pitch. Hands are wrist-centered +// at the origin, so the camera always looks at (0,0,0). Shared by the animate +// loop and snapshot() so a snapshot never renders from an unpositioned camera. +function positionCamera() { + // Distance to fit a sphere of _fitRadius given the camera's vertical FOV, + // with margin. Orbit around the hand centroid (_focus), not the wrist. + const half = THREE.MathUtils.degToRad(camera.fov) / 2; + const r = Math.max(0.12, (_fitRadius / Math.sin(half)) * 1.4); + camera.position.set( + _focus.x + r * Math.cos(pitch) * Math.sin(yaw), + _focus.y + r * Math.sin(pitch), + _focus.z + r * Math.cos(pitch) * Math.cos(yaw), + ); + camera.lookAt(_focus); +} + function animate() { raf = requestAnimationFrame(animate); if (!visible) return; if (autoRotate && !dragging) yaw += 0.005; - // Orbit camera around the origin (hands are wrist-centered at origin). - const r = 0.34; - camera.position.set( - r * Math.cos(pitch) * Math.sin(yaw), - r * Math.sin(pitch), - r * Math.cos(pitch) * Math.cos(yaw), - ); - camera.lookAt(0, 0, 0); + positionCamera(); renderer.render(scene, camera); } @@ -197,7 +224,7 @@ const OMHandViewer = { // for a non-quest model -> predicted hand hidden). update(realFlat, predFlat) { if (!renderer) return; - layoutHand(realFlat, realRig, _realOut); + if (layoutHand(realFlat, realRig, _realOut)) computeFraming(_realOut); if (predFlat && predFlat.length >= N_JOINTS * FLOATS_PER_JOINT) { layoutHand(predFlat, predRig, _predOut); } else { @@ -211,6 +238,17 @@ const OMHandViewer = { if (v) resize(); }, + // Render one frame synchronously and return it as a PNG data URL. The + // synchronous render + toDataURL captures the drawing buffer before the + // compositor clears it, so it works without preserveDrawingBuffer. Handy + // for frame thumbnails and for headless verification of the render. + snapshot() { + if (!renderer) return null; + positionCamera(); + renderer.render(scene, camera); + return renderer.domElement.toDataURL('image/png'); + }, + isReady() { return !!renderer; }, }; diff --git a/pc/src/openmuscle/web/static/index.html b/pc/src/openmuscle/web/static/index.html index 6e1de5c..191fad6 100644 --- a/pc/src/openmuscle/web/static/index.html +++ b/pc/src/openmuscle/web/static/index.html @@ -11,7 +11,7 @@ - +
    From 0648d58b6af71e68578a65918b0f8bd4c75d9d82 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 19:23:31 -0500 Subject: [PATCH 49/56] ml: commit the missing export module behind 'openmuscle export' cli.py has shipped an 'openmuscle export' command since the original web UI commit (d631951), but the module it imports (openmuscle/ml/export.py) was never committed, so the command raised ImportError on any fresh clone. The file existed only in the local working tree. export_predictions() runs a trained model over a capture CSV and writes a JSON bundle (sensor channels, ground-truth labels, predictions, overall + per-label MSE/R^2) for web visualization. Two small touch-ups while committing: ASCII R^2 in the console summary (the unicode superscript garbles on Windows cp1252 consoles) and no em dash in the docstring. Smoke-tested: exported 3242 samples from capture_72.csv with a local random forest model; JSON and metrics came out correct. --- pc/src/openmuscle/ml/export.py | 107 +++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 pc/src/openmuscle/ml/export.py diff --git a/pc/src/openmuscle/ml/export.py b/pc/src/openmuscle/ml/export.py new file mode 100644 index 0000000..b05657f --- /dev/null +++ b/pc/src/openmuscle/ml/export.py @@ -0,0 +1,107 @@ +"""Export predictions and ground truth to JSON for web visualization.""" + +import json +import numpy as np +import pandas as pd +from pathlib import Path + +from openmuscle.data.dataset import load_training_data +from openmuscle.ml.registry import ModelRegistry + + +def export_predictions(model_path: str, data_path: str, output_path: str) -> dict: + """Run a trained model on a capture CSV and export results as JSON. + + The JSON output contains sensor data, actual labels (ground truth), + and model predictions, suitable for web visualization. + + Args: + model_path: path to trained model .pkl file + data_path: path to CSV with sensor + label columns + output_path: path for output .json file + + Returns: + dict with summary info (n_samples, metrics, etc.) + """ + from sklearn.metrics import mean_squared_error, r2_score + + registry = ModelRegistry() + model = registry.load(model_path) + + X, y = load_training_data(data_path) + sensor_cols = list(X.columns) + label_cols = list(y.columns) + + # Load full dataframe for timestamps + df = pd.read_csv(data_path) + timestamps = None + for ts_col in ['Sensor_Timestamp', 'timestamp', 'time']: + if ts_col in df.columns: + timestamps = df[ts_col].tolist() + break + + # Run predictions + predictions = model.predict(X) + if predictions.ndim == 1: + predictions = predictions.reshape(-1, 1) + + actuals = y.values + + # Compute metrics + mse = float(mean_squared_error(actuals, predictions)) + r2 = float(r2_score(actuals, predictions)) + + # Per-label metrics + per_label = {} + for i, col in enumerate(label_cols): + per_label[col] = { + "mse": float(mean_squared_error(actuals[:, i], predictions[:, i])), + "r2": float(r2_score(actuals[:, i], predictions[:, i])), + } + + # Build output structure + n_samples = len(X) + output = { + "metadata": { + "model_path": model_path, + "data_path": data_path, + "n_samples": n_samples, + "sensor_columns": sensor_cols, + "label_columns": label_cols, + "metrics": { + "overall_mse": mse, + "overall_r2": r2, + "per_label": per_label, + }, + }, + "sensor_data": {}, + "labels": {}, + "predictions": {}, + } + + if timestamps: + output["timestamps"] = timestamps + + # Sensor values per channel + for col in sensor_cols: + output["sensor_data"][col] = X[col].tolist() + + # Actual labels per channel + for i, col in enumerate(label_cols): + output["labels"][col] = actuals[:, i].tolist() + + # Predicted labels per channel + for i, col in enumerate(label_cols): + output["predictions"][col] = predictions[:, i].tolist() + + # Write JSON + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + + print(f"Exported {n_samples} samples to {output_path}") + print(f" Overall MSE: {mse:.4f} R^2: {r2:.4f}") + for col, m in per_label.items(): + print(f" {col}: MSE={m['mse']:.4f} R^2={m['r2']:.4f}") + + return output["metadata"] From 9ba93b31be78de31c29b584c43336f6e62e10783 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 20:06:02 -0500 Subject: [PATCH 50/56] web: fix hand-viewer canvas growth feedback loop Live verification with a streaming quest_hand device caught the canvas growing every frame until it hit the GPU's 16384px max texture size. Cause: renderer.setSize(w, h, false) sets the canvas width/height ATTRIBUTES only, and with no CSS size on the canvas element those become its layout size. The flex container then grows to fit, the per-tick setVisible(true) -> resize() reads the bigger clientWidth, and the loop runs away (any devicePixelRatio > 1 compounds it). Fixes: * CSS-pin the canvas to its container (style 100%/100%) at init so the drawing-buffer size can never drive layout. * Skip no-op resizes; setSize clears the drawing buffer even when the size is unchanged, and resize() runs every tick via setVisible(true). * Clamp pixelRatio to 2. Bump hand-viewer.js?v=5. Verified live: canvas holds 436x238 (CSS px == buffer px) with a quest_hand device streaming at 25 Hz, and the snapshot() output shows the hand correctly framed. Co-Authored-By: turfptax-claude F5 --- pc/src/openmuscle/web/static/hand-viewer.js | 16 +++++++++++++++- pc/src/openmuscle/web/static/index.html | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pc/src/openmuscle/web/static/hand-viewer.js b/pc/src/openmuscle/web/static/hand-viewer.js index f548359..ea5cbb7 100644 --- a/pc/src/openmuscle/web/static/hand-viewer.js +++ b/pc/src/openmuscle/web/static/hand-viewer.js @@ -147,10 +147,16 @@ function computeFraming(pts) { _fitRadius = maxR || 0.1; } +let _lastW = 0, _lastH = 0; + function resize() { if (!container || !renderer) return; const w = container.clientWidth || 320; const h = container.clientHeight || 240; + // Skip no-op resizes: setVisible(true) fires every tick, and setSize + // clears the drawing buffer even when the size hasn't changed. + if (w === _lastW && h === _lastH) return; + _lastW = w; _lastH = h; renderer.setSize(w, h, false); camera.aspect = w / h; camera.updateProjectionMatrix(); @@ -187,7 +193,15 @@ const OMHandViewer = { scene = new THREE.Scene(); camera = new THREE.PerspectiveCamera(45, 1, 0.01, 10); renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }); - renderer.setPixelRatio(window.devicePixelRatio || 1); + renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); + // The canvas must never drive layout: setSize(..., false) sets the + // canvas width/height ATTRIBUTES (drawing buffer), and without CSS + // sizing those become the layout size. The container then grows to + // fit, resize() reads the bigger clientWidth, and the loop runs the + // canvas up to the GPU's max texture size. CSS-pin it to the + // container instead. + renderer.domElement.style.width = '100%'; + renderer.domElement.style.height = '100%'; container.appendChild(renderer.domElement); realRig = makeHandRig(COLOR_REAL, 1.0); diff --git a/pc/src/openmuscle/web/static/index.html b/pc/src/openmuscle/web/static/index.html index 191fad6..f0df00a 100644 --- a/pc/src/openmuscle/web/static/index.html +++ b/pc/src/openmuscle/web/static/index.html @@ -11,7 +11,7 @@ - +
    From bd5209311065e680c59f815483afa9776e6337f4 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 20:06:28 -0500 Subject: [PATCH 51/56] simulate: synthetic quest_hand + combo modes for zero-hardware pipeline The quest pipeline previously needed a physical Quest 3 to test anything past unit level. New simulator modes close that gap: * --device-type quest_hand: streams a synthetic articulated 25-joint WebXR hand (canonical joint order, per-finger sinusoidal curls with distinct frequencies, wandering wrist, slow global yaw so the inverse-wrist-rotation path gets non-identity input) to the running web server's /ws/quest WebSocket at 25 Hz. Reuses the exact ingest path the headset uses, so the recorder, matcher, snapshot, and both hand viewers see it as a real device. Reconnects if the server restarts. No new dependency: the websockets package already ships with uvicorn[standard]. * --device-type combo: the same latent finger-curl signal drives BOTH the quest hand and a flexgrid UDP device (gaussian per-finger spatial weighting across the 16 columns + noise), so a recorded capture has a learnable sensor->hand mapping. simulate/quest_hand.py keeps the generators pure (t in, data out) for testability; tests/test_quest_simulator.py pins the joint layout, the articulation actually moving fingertips, the flexgrid correlation and per-finger locality, and a real ingest round-trip (175 finite floats). Verified end to end against a live `openmuscle web` server, all through the public surfaces (web API + CLI), zero hardware: * both devices register (quest-sim ~25 Hz / 175 values, flexgrid-sim ~12 Hz heatmap) * desktop 3D hand viewer renders the live articulated hand * 90 s recording: 1105 paired rows, 100% match rate, labels-schema sidecar written, zero label-width mismatches * training on that capture: held-out R^2 = 0.73 across all 175 joint channels * loading the model into live inference: predicted hand tracks the real hand at ~2.5 mm mean per-joint error in wrist-local space Suite: 42 passing. Co-Authored-By: turfptax-claude F5 --- pc/src/openmuscle/cli.py | 14 +- pc/src/openmuscle/simulate/quest_hand.py | 202 ++++++++++++++++++++++ pc/src/openmuscle/simulate/transmitter.py | 76 +++++++- pc/tests/test_quest_simulator.py | 120 +++++++++++++ 4 files changed, 405 insertions(+), 7 deletions(-) create mode 100644 pc/src/openmuscle/simulate/quest_hand.py create mode 100644 pc/tests/test_quest_simulator.py diff --git a/pc/src/openmuscle/cli.py b/pc/src/openmuscle/cli.py index 597c62d..1eaf4ba 100644 --- a/pc/src/openmuscle/cli.py +++ b/pc/src/openmuscle/cli.py @@ -190,15 +190,21 @@ def predict(model, port): @main.command() @click.option("--port", default=3141, help="UDP port to send to") @click.option("--device-type", default="flexgrid", - help="Device type to simulate (flexgrid, lask5)") + type=click.Choice(["flexgrid", "lask5", "quest_hand", "combo"]), + help="Device type to simulate. quest_hand streams synthetic " + "WebXR hand frames to the web server's /ws/quest; combo " + "adds a correlated flexgrid UDP device so record/train/" + "predict works end to end without hardware.") @click.option("--replay", type=click.Path(exists=True), help="Replay a capture file instead of generating synthetic data") @click.option("--target-ip", default="127.0.0.1", help="Target IP address") -def simulate(port, device_type, replay, target_ip): - """Send synthetic or replayed sensor data over UDP.""" +@click.option("--web-port", default=8000, + help="openmuscle web HTTP port (quest_hand/combo only)") +def simulate(port, device_type, replay, target_ip, web_port): + """Send synthetic or replayed sensor data.""" from openmuscle.simulate.transmitter import run_simulator run_simulator(port=port, device_type=device_type, - replay_file=replay, target_ip=target_ip) + replay_file=replay, target_ip=target_ip, web_port=web_port) @main.command() diff --git a/pc/src/openmuscle/simulate/quest_hand.py b/pc/src/openmuscle/simulate/quest_hand.py new file mode 100644 index 0000000..409504d --- /dev/null +++ b/pc/src/openmuscle/simulate/quest_hand.py @@ -0,0 +1,202 @@ +"""Synthetic WebXR hand frames for hardware-free pipeline testing. + +Generates the same JSON payload shape the Quest WebXR client pushes over +/ws/quest (see AppState.ingest_quest_packet), driven by a smooth latent +finger-curl signal. The companion flexgrid_matrix() derives correlated +pressure-sensor values from the SAME curls, so a capture recorded from +the simulator pair is actually learnable: train on it and the model's +predicted hand visibly tracks the real one in both viewers. + +Everything here is pure (t in, data out) so tests can call it directly +without sockets or timing. +""" + +import math + +# Canonical WebXR hand joint order, 25 joints. Must match what the VR +# client sends and what the desktop hand viewer assumes (wrist, 4 thumb +# joints, then 5 per finger). +JOINT_NAMES = ( + "wrist", + "thumb-metacarpal", "thumb-phalanx-proximal", "thumb-phalanx-distal", + "thumb-tip", + "index-finger-metacarpal", "index-finger-phalanx-proximal", + "index-finger-phalanx-intermediate", "index-finger-phalanx-distal", + "index-finger-tip", + "middle-finger-metacarpal", "middle-finger-phalanx-proximal", + "middle-finger-phalanx-intermediate", "middle-finger-phalanx-distal", + "middle-finger-tip", + "ring-finger-metacarpal", "ring-finger-phalanx-proximal", + "ring-finger-phalanx-intermediate", "ring-finger-phalanx-distal", + "ring-finger-tip", + "pinky-finger-metacarpal", "pinky-finger-phalanx-proximal", + "pinky-finger-phalanx-intermediate", "pinky-finger-phalanx-distal", + "pinky-finger-tip", +) + +N_JOINTS = len(JOINT_NAMES) + +# Per-finger latent curl oscillators (Hz, radians). Distinct frequencies +# so no two fingers move in lockstep and a model can't cheat by learning +# one shared phase. +_CURL_FREQ = (0.13, 0.21, 0.17, 0.26, 0.31) # thumb..pinky +_CURL_PHASE = (0.0, 1.3, 2.6, 3.9, 5.2) + +# Finger geometry for a plausible right hand, meters. The hand extends +# +y from the wrist with the palm facing -z, so curling rotates segment +# directions from +y toward -z. +# (lateral x offset of metacarpal head, metacarpal length, +# segment lengths..., max curl angle per segment in radians) +_FINGERS = { + "index": (0.030, 0.085, (0.042, 0.025, 0.020), 1.35), + "middle": (0.010, 0.090, (0.046, 0.028, 0.021), 1.40), + "ring": (-0.012, 0.085, (0.042, 0.026, 0.020), 1.40), + "pinky": (-0.032, 0.075, (0.033, 0.020, 0.017), 1.45), +} + + +def finger_curls(t: float) -> tuple: + """Latent curl signal per finger at time t, each in [0, 1]. + + Order: thumb, index, middle, ring, pinky. + """ + return tuple( + 0.5 + 0.5 * math.sin(2 * math.pi * f * t + p) + for f, p in zip(_CURL_FREQ, _CURL_PHASE) + ) + + +def _chain(start, direction_y, direction_z, lengths, curl, max_angle): + """Walk a finger's phalanx chain in the y/-z plane. + + Starts at `start` heading (0, direction_y, direction_z) (unit), bends + by curl*max_angle at each joint. Returns the list of joint positions + after each segment. + """ + out = [] + px, py, pz = start + angle = math.atan2(-direction_z, direction_y) + step = curl * max_angle / len(lengths) + for seg in lengths: + angle += step + py += seg * math.cos(angle) + pz -= seg * math.sin(angle) + out.append((px, py, pz)) + return out + + +def _yaw_quat(a: float) -> list: + """Quaternion [x,y,z,w] for a rotation of `a` radians about +y.""" + return [0.0, math.sin(a / 2), 0.0, math.cos(a / 2)] + + +def _yaw_rotate(p, origin, a): + """Rotate point p about a vertical axis through `origin` by angle a.""" + x, y, z = p[0] - origin[0], p[1], p[2] - origin[2] + ca, sa = math.cos(a), math.sin(a) + return (origin[0] + x * ca + z * sa, y, origin[2] - x * sa + z * ca) + + +def hand_pose(t: float): + """All 25 joint positions (world space, meters) plus the shared + orientation quaternion at time t. + + Returns (positions, quat) where positions is a list of 25 (x, y, z) + tuples in JOINT_NAMES order. + """ + curls = finger_curls(t) + + # Wrist wanders slowly so the wrist-local canonicalization in both + # viewers is exercised, not just fed a static origin. + wrist = ( + 0.10 * math.sin(2 * math.pi * 0.05 * t), + 1.00 + 0.05 * math.sin(2 * math.pi * 0.07 * t), + -0.30 + 0.08 * math.cos(2 * math.pi * 0.06 * t), + ) + + positions = [wrist] + + # Thumb: 4 joints, angled out +x, curling toward the palm. + tc = curls[0] + base = (wrist[0] + 0.025, wrist[1] + 0.020, wrist[2]) + positions.append(base) + px, py, pz = base + angle = 0.5 + tc * 0.9 # radians of accumulated flexion + for seg in (0.045, 0.032, 0.028): + px += seg * math.sin(0.6) * (1 - tc * 0.5) # keeps thumb fanned +x + py += seg * math.cos(angle) * 0.8 + pz -= seg * math.sin(angle) * 0.6 + angle += tc * 0.5 + positions.append((px, py, pz)) + + # Four fingers: metacarpal head, then a curling 3-segment chain plus tip. + for i, name in enumerate(("index", "middle", "ring", "pinky")): + dx, meta_len, segs, max_angle = _FINGERS[name] + curl = curls[i + 1] + meta = (wrist[0] + dx, wrist[1] + meta_len, wrist[2]) + positions.append(meta) + chain = _chain(meta, 1.0, 0.0, segs, curl, max_angle) + positions.extend(chain) + # Tip: short continuation of the last segment direction. + lx, ly, lz = chain[-1] + sx, sy, sz = chain[-2] if len(chain) > 1 else meta + norm = math.dist((lx, ly, lz), (sx, sy, sz)) or 1.0 + positions.append(( + lx + (lx - sx) / norm * 0.012, + ly + (ly - sy) / norm * 0.012, + lz + (lz - sz) / norm * 0.012, + )) + + # Slow global yaw about the wrist; every joint reports the same + # orientation so the viewers' inverse-wrist-rotation path gets real + # (non-identity) input. + yaw = 0.6 * math.sin(2 * math.pi * 0.04 * t) + positions = [wrist] + [_yaw_rotate(p, wrist, yaw) for p in positions[1:]] + return positions, _yaw_quat(yaw) + + +def hand_frame(t: float, device_id: str = "quest-sim", + handedness: str = "right") -> dict: + """One /ws/quest JSON payload for time t.""" + positions, quat = hand_pose(t) + return { + "device_id": device_id, + "ts": int(t * 1000) % 2**31, + "handedness": handedness, + "joints": [ + {"name": name, "pos": [round(c, 5) for c in pos], "rot": quat} + for name, pos in zip(JOINT_NAMES, positions) + ], + "meta": {"sim": True}, + } + + +# Flexgrid layout used by the existing simulator: 16 column-groups of 4. +_FG_COLS = 16 +_FG_ROWS = 4 + + +def flexgrid_matrix(curls, rng=None) -> list: + """A 16x4 flexgrid matrix correlated with the given finger curls. + + Each sensor responds to a weighted blend of nearby fingers (gaussian + falloff across the 16 columns, finger centers spread over them), plus + optional noise from `rng` (anything with .gauss). Values are ints in + [0, 4095] like real FlexGrid ADC counts. + """ + centers = (1.5, 5.0, 8.0, 11.0, 14.0) # thumb..pinky across 16 columns + matrix = [] + for col in range(_FG_COLS): + group = [] + for row in range(_FG_ROWS): + signal = 0.0 + for f, center in enumerate(centers): + w = math.exp(-((col - center) ** 2) / 6.0) + # Rows sample the muscle at slightly different gains. + signal += w * curls[f] * (0.7 + 0.1 * row) + v = 400 + 3000 * signal + if rng is not None: + v += rng.gauss(0, 60) + group.append(max(0, min(4095, int(v)))) + matrix.append(group) + return matrix diff --git a/pc/src/openmuscle/simulate/transmitter.py b/pc/src/openmuscle/simulate/transmitter.py index 3040fda..ad730a0 100644 --- a/pc/src/openmuscle/simulate/transmitter.py +++ b/pc/src/openmuscle/simulate/transmitter.py @@ -10,19 +10,29 @@ def run_simulator(port: int = 3141, device_type: str = "flexgrid", - replay_file: str = None, target_ip: str = "127.0.0.1"): - """Send synthetic or replayed sensor data over UDP. + replay_file: str = None, target_ip: str = "127.0.0.1", + web_port: int = 8000): + """Send synthetic or replayed sensor data. Args: port: UDP port to send to - device_type: device type to simulate ("flexgrid" or "lask5") + device_type: device type to simulate ("flexgrid", "lask5", + "quest_hand", or "combo"). quest_hand streams synthetic WebXR + hand frames to the web server's /ws/quest WebSocket (no UDP). + combo streams a flexgrid UDP device AND a quest hand driven by + the same latent finger curls, so a recorded capture is + actually learnable end to end without hardware. replay_file: path to a capture .txt file to replay (optional) target_ip: IP address to send to + web_port: HTTP port of the running `openmuscle web` server + (quest_hand/combo only) """ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) if replay_file: _replay(sock, replay_file, target_ip, port) + elif device_type in ("quest_hand", "combo"): + _generate_quest(sock, device_type, target_ip, port, web_port) else: _generate(sock, device_type, target_ip, port) @@ -71,6 +81,66 @@ def _generate(sock, device_type: str, ip: str, port: int): print(f"\nStopped after {pkt_num} packets") +def _generate_quest(sock, device_type: str, ip: str, udp_port: int, + web_port: int): + """Stream synthetic WebXR hand frames to /ws/quest, and for combo mode + a correlated flexgrid device over UDP alongside. + + The hand goes over WebSocket because that's the only transport the + real Quest client has; reusing the same ingest path means everything + downstream (recorder, matcher, snapshot, both hand viewers) sees the + simulator exactly as it would see a headset. + """ + import random + from websockets.sync.client import connect + from openmuscle.simulate.quest_hand import finger_curls, hand_frame, \ + flexgrid_matrix + + url = f"ws://{ip}:{web_port}/ws/quest" + print(f"Streaming synthetic quest_hand frames -> {url}") + if device_type == "combo": + print(f" + correlated flexgrid UDP -> {ip}:{udp_port}") + print("Press Ctrl+C to stop (reconnects if the server restarts)") + + rng = random.Random(1138) + frame_num = 0 + t0 = time.time() + try: + while True: + try: + with connect(url) as ws: + while True: + t = time.time() - t0 + ws.send(json.dumps(hand_frame(t))) + frame_num += 1 + # Flexgrid at half the hand rate (~12.5 Hz vs 25 Hz), + # like the real rig where sensor and label rates differ. + if device_type == "combo" and frame_num % 2 == 0: + pkt = { + "v": "1.0", + "type": "flexgrid", + "id": "flexgrid-sim", + "ts": int(time.time() * 1000) % 2**31, + "data": { + "matrix": flexgrid_matrix(finger_curls(t), rng), + "rows": 4, "cols": 16, + }, + } + sock.sendto(json.dumps(pkt).encode("utf-8"), + (ip, udp_port)) + if frame_num % 250 == 0: + print(f"Sent {frame_num} hand frames") + time.sleep(0.04) # 25 Hz + except Exception as e: + # KeyboardInterrupt is a BaseException, so Ctrl+C still + # reaches the outer handler. + print(f"WebSocket dropped ({type(e).__name__}: {e}); " + f"retrying in 2s") + time.sleep(2) + except KeyboardInterrupt: + print(f"\nStopped after {frame_num} hand frames") + + def _replay(sock, filepath: str, ip: str, port: int): """Replay a capture file by re-sending each line as a UDP packet.""" import ast diff --git a/pc/tests/test_quest_simulator.py b/pc/tests/test_quest_simulator.py new file mode 100644 index 0000000..ae73df0 --- /dev/null +++ b/pc/tests/test_quest_simulator.py @@ -0,0 +1,120 @@ +"""Tests for the synthetic quest_hand simulator (simulate/quest_hand.py). + +The simulator exists so the whole quest pipeline (ingest -> record -> +train -> predict -> both hand viewers) can run with zero hardware. These +tests pin the contracts that pipeline depends on: + - frame shape matches what ingest_quest_packet expects (25 joints, + canonical WebXR names, pos + rot per joint) + - the latent curls actually articulate the hand (fingertips move) + - the flexgrid matrix is correlated with the curls (learnable signal) + - a generated frame survives real ingestion as a 175-float device +""" + +import math + +from openmuscle.simulate.quest_hand import ( + JOINT_NAMES, N_JOINTS, finger_curls, hand_frame, hand_pose, + flexgrid_matrix, +) +from openmuscle.web.state import AppState + + +class TestJointLayout: + def test_25_joints_canonical_order(self): + assert N_JOINTS == 25 + assert JOINT_NAMES[0] == "wrist" + assert JOINT_NAMES[1] == "thumb-metacarpal" + assert JOINT_NAMES[4] == "thumb-tip" + assert JOINT_NAMES[5] == "index-finger-metacarpal" + assert JOINT_NAMES[9] == "index-finger-tip" + assert JOINT_NAMES[24] == "pinky-finger-tip" + + def test_hand_pose_returns_25_positions(self): + positions, quat = hand_pose(0.0) + assert len(positions) == 25 + assert len(quat) == 4 + # Unit quaternion + assert math.isclose(sum(c * c for c in quat), 1.0, rel_tol=1e-6) + + def test_frame_payload_shape(self): + frame = hand_frame(1.5) + assert frame["handedness"] == "right" + assert frame["device_id"] == "quest-sim" + assert len(frame["joints"]) == 25 + for j in frame["joints"]: + assert len(j["pos"]) == 3 + assert len(j["rot"]) == 4 + assert [j["name"] for j in frame["joints"]] == list(JOINT_NAMES) + + +class TestArticulation: + def test_curls_in_unit_range(self): + for t in (0.0, 0.7, 3.3, 12.9): + for c in finger_curls(t): + assert 0.0 <= c <= 1.0 + + def test_curls_move_fingertips(self): + """Pick two times where the index curl differs a lot; its tip must + be in a meaningfully different place relative to the wrist.""" + t_a, t_b = 0.0, 2.0 + # Make sure the latent signal actually differs at these times. + assert abs(finger_curls(t_a)[1] - finger_curls(t_b)[1]) > 0.2 + pa, _ = hand_pose(t_a) + pb, _ = hand_pose(t_b) + tip = JOINT_NAMES.index("index-finger-tip") + rel_a = [pa[tip][i] - pa[0][i] for i in range(3)] + rel_b = [pb[tip][i] - pb[0][i] for i in range(3)] + assert math.dist(rel_a, rel_b) > 0.01 + + def test_hand_is_hand_sized(self): + """Every joint within 25 cm of the wrist, fingers above it.""" + positions, _ = hand_pose(4.2) + wrist = positions[0] + for p in positions[1:]: + assert math.dist(p, wrist) < 0.25 + + +class TestFlexgridCorrelation: + def test_matrix_shape_and_range(self): + m = flexgrid_matrix(finger_curls(1.0)) + assert len(m) == 16 + assert all(len(col) == 4 for col in m) + for col in m: + for v in col: + assert isinstance(v, int) + assert 0 <= v <= 4095 + + def test_matrix_tracks_curls(self): + """Open hand vs closed fist must produce clearly different totals.""" + open_hand = flexgrid_matrix((0.0,) * 5) + fist = flexgrid_matrix((1.0,) * 5) + total_open = sum(v for col in open_hand for v in col) + total_fist = sum(v for col in fist for v in col) + assert total_fist > total_open + 10000 + + def test_per_finger_locality(self): + """Curling only the pinky should move the right edge of the array + more than the left edge (sensors are spatially weighted).""" + base = flexgrid_matrix((0.0,) * 5) + pinky = flexgrid_matrix((0.0, 0.0, 0.0, 0.0, 1.0)) + left_delta = sum(pinky[c][r] - base[c][r] for c in range(4) for r in range(4)) + right_delta = sum(pinky[c][r] - base[c][r] for c in range(12, 16) for r in range(4)) + assert right_delta > left_delta + + +class TestIngestRoundTrip: + def test_generated_frame_ingests_as_quest_device(self): + s = AppState.__new__(AppState) + s.devices = {} + s.recording = None + s.engine = None + s.inference_enabled = False + s.hand_target = None + + s.ingest_quest_packet(hand_frame(0.5)) + assert "quest-sim" in s.devices + d = s.devices["quest-sim"] + assert d.device_type == "quest_hand" + assert len(d.last_values) == 25 * 7 + # No NaN/inf anywhere in the flattened vector + assert all(math.isfinite(v) for v in d.last_values) From 39b6897d442dae62d38a521c214bc913075a7e51 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 20:06:45 -0500 Subject: [PATCH 52/56] docs: zero-hardware smoke test for the quest pipeline Document the new simulator modes where people will look for them: * pc-cli.md: quest_hand and combo examples under `openmuscle simulate` * vr-testing-scenarios.md: a "Zero-hardware smoke test" section ahead of the headset runbook, with the expected results at each stage (devices, hand viewer, recording, training R^2, predicted hand), so a real-headset failure can be bisected to the headset/network layer vs the PC pipeline * README quick start: one-liner pointing at combo mode Co-Authored-By: turfptax-claude F5 --- README.md | 4 ++++ docs/pc-cli.md | 10 ++++++++++ docs/vr-testing-scenarios.md | 29 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/README.md b/README.md index f2f1c77..4efd197 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,10 @@ openmuscle predict -m data/models/random_forest_*/model.pkl # Test without hardware openmuscle simulate --device-type flexgrid + +# Test the whole VR pipeline without hardware (pair with `openmuscle web`): +# synthetic flexgrid + synthetic Quest hand driven by the same finger curls +openmuscle simulate --device-type combo ``` ### VR companion (Meta Quest 3) diff --git a/docs/pc-cli.md b/docs/pc-cli.md index 558557e..dd3cfab 100644 --- a/docs/pc-cli.md +++ b/docs/pc-cli.md @@ -57,6 +57,16 @@ Send synthetic data for testing without hardware. openmuscle simulate --device-type flexgrid openmuscle simulate --device-type lask5 --target-ip 192.168.1.100 openmuscle simulate --replay data/raw/legacy/capture_45.txt + +# Synthetic WebXR hand: streams 25-joint frames to the running +# `openmuscle web` server's /ws/quest WebSocket (no headset needed). +openmuscle simulate --device-type quest_hand + +# Combo: the same latent finger-curl signal drives BOTH a flexgrid UDP +# device and the quest hand, so the capture is learnable and the whole +# record -> train -> predict pipeline can be exercised end to end with +# zero hardware. Use --web-port if the web server is not on 8000. +openmuscle simulate --device-type combo ``` ### `openmuscle models` diff --git a/docs/vr-testing-scenarios.md b/docs/vr-testing-scenarios.md index 81d579c..3e0c0ae 100644 --- a/docs/vr-testing-scenarios.md +++ b/docs/vr-testing-scenarios.md @@ -25,6 +25,35 @@ If any step fails, the [troubleshooting wiki page](https://github.com/Open-Muscl --- +## Zero-hardware smoke test (no headset, no FlexGrid) + +Before any headset session, you can exercise the entire PC-side quest +pipeline with the simulator. Two terminals: + +```bash +cd pc +openmuscle web # terminal 1 +openmuscle simulate --device-type combo # terminal 2 +``` + +Then in the desktop UI (`http://localhost:8000/`): + +1. Devices panel lists `flexgrid-sim` (~12 Hz, live heatmap) and + `quest-sim` (~25 Hz, `quest_hand`). +2. The Live stage swaps the piston comparator for the 3D hand viewer; a + green articulated hand auto-rotates and its fingers visibly curl. +3. REC pairs frames (match rate goes green), STOP writes a CSV with 64 + sensor columns and 175 label columns plus the labels-schema sidecar. +4. Train on that capture; the combo simulator's sensor values are + derived from the same latent finger curls as the hand, so the model + should reach a clearly positive held-out R^2. Load the model and the + amber predicted hand tracks the green real hand in the viewer. + +If all of that works, a real-headset failure is in the headset/network +layer, not the PC pipeline. + +--- + ## 2-minute smoke test (no FlexGrid required) The fastest "is anything obviously broken" check. Only needs the headset + PC + server. Do this first whenever you pick the project back up after code changes. From 655f185d80418ce8e77f2b44132507a210b81889 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 23 Jun 2026 22:03:14 -0500 Subject: [PATCH 53/56] LASK5 + bridge_subscriber: V4 discovery + subscribe protocol Adds the V4 device-discovery and subscribe protocol to the LASK5 firmware living under embedded/devices/lask5_v2/, plus a small PC-side helper that lets the existing openmuscle web server receive V4-protocol sensor and label frames without waiting on the full PC-app discovery rewrite. The same code lives in dedicated firmware repos as the authoritative source going forward: - https://github.com/Open-Muscle/LASK5-Firmware - https://github.com/Open-Muscle/FlexGridV4-Firmware The copies committed here keep the in-tree embedded/ subtree working as a reference for anyone pulling from this repo, but the firmware repos are the primary source of truth. New shared modules under embedded/lib/ (vendored to LASK5-Firmware and FlexGridV4-Firmware): - om_subscribers.py: per-device subscriber list with 4-slot cap and 5 s heartbeat timeout per spec section 5.1 - om_discovery.py: mDNS hostname + UDP broadcast announce beacon - om_commands.py: TCP newline-delimited JSON command server with the standard handler verbs (subscribe, unsubscribe, heartbeat, get_info, start_stream, stop_stream, reboot) Modified shared lib: - om_network.py: added send_udp_to_subscribers fan-out for the new protocol; existing send_udp untouched so FlexGrid V1 and OpenHand keep working as is Device firmware: - lask5_v2/labeler.py: wires discovery + cmd server + subscribe into the LASK5 lifecycle. _send_loop now fans out to subscribers instead of a hardcoded target IP. ESP-NOW path to OpenHand is byte-identical. Tooling: - embedded/_upload_lask5.py: pyserial-based raw-REPL uploader for boards where mpremote cannot enter raw REPL because the device logger floods the UART. Used during the LASK5 bring-up. - pc/bridge_subscriber.py: standalone helper that subscribes a list of V4-protocol device IPs to the PC's UDP 3141 listener and keeps them alive with 1 Hz heartbeats. Workaround until the PC app's native discovery support lands (see OpenMuscle-Lab-Discovery workspace, Phase 1 + 2). Verified end-to-end: - LASK5 (lask5-01) announces + accepts subscribe + streams labels - FlexGrid V4 (flexgrid-d7af0b, flexgrid-1524c3) likewise - openmuscle web receives 18 Hz sensor frames via bridge_subscriber Co-Authored-By: turfptax-claude O4.7 --- embedded/_upload_lask5.py | 168 ++++++++++++++++++++++ embedded/devices/lask5_v2/labeler.py | 132 +++++++++++++++-- embedded/lib/om_commands.py | 167 +++++++++++++++++++++ embedded/lib/om_discovery.py | 127 ++++++++++++++++ embedded/lib/om_network.py | 26 ++++ embedded/lib/om_subscribers.py | 113 +++++++++++++++ pc/bridge_subscriber.py | 207 +++++++++++++++++++++++++++ 7 files changed, 932 insertions(+), 8 deletions(-) create mode 100644 embedded/_upload_lask5.py create mode 100644 embedded/lib/om_commands.py create mode 100644 embedded/lib/om_discovery.py create mode 100644 embedded/lib/om_subscribers.py create mode 100644 pc/bridge_subscriber.py diff --git a/embedded/_upload_lask5.py b/embedded/_upload_lask5.py new file mode 100644 index 0000000..ef50e35 --- /dev/null +++ b/embedded/_upload_lask5.py @@ -0,0 +1,168 @@ +"""Direct pyserial uploader for the LASK5 device. + +mpremote keeps hitting "could not enter raw REPL" because the running labeler +floods the UART. This script speaks the MicroPython raw-REPL protocol over +COM24 directly: Ctrl-C to interrupt, Ctrl-A to enter raw REPL, write each +file by sending Python code that creates it, Ctrl-B to exit, then reset. + +Run from the embedded/ folder so the relative paths to lib/ and devices/ +resolve correctly. +""" +import os +import sys +import time +import serial + +PORT = "COM24" +BAUD = 115200 + +# (local_path, remote_path) tuples to push +PUSH_LIST = [ + ("lib/om_network.py", "/lib/om_network.py"), + ("lib/om_subscribers.py", "/lib/om_subscribers.py"), + ("lib/om_discovery.py", "/lib/om_discovery.py"), + ("lib/om_commands.py", "/lib/om_commands.py"), + ("devices/lask5_v2/labeler.py", "/labeler.py"), +] + + +def drain(s, ms=200): + """Read everything currently buffered on the serial port.""" + end = time.time() + ms / 1000.0 + buf = bytearray() + while time.time() < end: + b = s.read(s.in_waiting or 1) + if b: + buf.extend(b) + else: + time.sleep(0.01) + return bytes(buf) + + +def enter_raw_repl(s): + """Halt any running script and enter raw REPL mode.""" + # Two Ctrl-C: halt any running script + s.write(b"\r\x03\x03") + time.sleep(0.3) + drain(s, 300) + # Ctrl-A: enter raw REPL + s.write(b"\r\x01") + time.sleep(0.5) + resp = drain(s, 500) + if b"raw REPL" not in resp: + # Try one more time after a fresh interrupt + s.write(b"\x03\x03\r\x01") + time.sleep(0.7) + resp = drain(s, 700) + if b"raw REPL" not in resp: + raise RuntimeError("Could not enter raw REPL. Got: {!r}".format(resp[:200])) + print(" raw REPL active") + + +def exit_raw_repl(s): + s.write(b"\r\x02") + time.sleep(0.2) + drain(s, 200) + + +def exec_raw(s, code): + """Execute Python code in raw REPL; return (stdout, stderr) bytes.""" + s.write(code.encode("utf-8")) + s.write(b"\x04") # EOT: run + # Expect b"OK" + output + 0x04 + error + 0x04> + buf = bytearray() + end = time.time() + 8.0 + while time.time() < end: + b = s.read(s.in_waiting or 1) + if b: + buf.extend(b) + if buf.count(b"\x04") >= 2 and buf.endswith(b"\x04>"): + break + else: + time.sleep(0.01) + raw = bytes(buf) + if raw[:2] != b"OK": + raise RuntimeError("raw REPL did not OK: {!r}".format(raw[:80])) + # Strip leading OK, trailing > + body = raw[2:] + if body.endswith(b"\x04>"): + body = body[:-2] + parts = body.split(b"\x04") + out = parts[0] if len(parts) >= 1 else b"" + err = parts[1] if len(parts) >= 2 else b"" + return out, err + + +def ensure_dir(s, remote_path): + """If remote_path is in a subdirectory, mkdir the parent.""" + parts = remote_path.lstrip("/").split("/") + if len(parts) <= 1: + return + d = "/" + "/".join(parts[:-1]) + code = ( + "import uos\n" + "try:\n" + " uos.stat('{0}')\n" + "except OSError:\n" + " uos.mkdir('{0}')\n" + "print('dir ok')\n" + ).format(d) + out, err = exec_raw(s, code) + if err.strip(): + raise RuntimeError("mkdir failed for {}: {!r}".format(d, err[:120])) + + +def push_file(s, local_path, remote_path): + """Upload local file to remote path via raw REPL.""" + with open(local_path, "rb") as f: + data = f.read() + print(" pushing {} -> {} ({} bytes)".format(local_path, remote_path, len(data))) + ensure_dir(s, remote_path) + # Open file on device for writing + open_code = "f=open('{}','wb')\n".format(remote_path) + out, err = exec_raw(s, open_code) + if err.strip(): + raise RuntimeError("open() failed: {!r}".format(err[:120])) + # Write in chunks. Each chunk goes through raw REPL exec; small enough + # to fit comfortably under the stdin buffer (typ. ~1KB safe). + chunk = 256 + for i in range(0, len(data), chunk): + piece = data[i:i+chunk] + # Encode as repr() of bytes so embedded backslashes / newlines work + code = "f.write({!r})\n".format(piece) + out, err = exec_raw(s, code) + if err.strip(): + raise RuntimeError("write() failed at offset {}: {!r}".format(i, err[:120])) + out, err = exec_raw(s, "f.close()\nimport uos\nprint(uos.stat('{}')[6])\n".format(remote_path)) + if err.strip(): + raise RuntimeError("close()/stat() failed: {!r}".format(err[:120])) + written = int(out.strip()) + if written != len(data): + raise RuntimeError("size mismatch: wrote {} but stat says {}".format(len(data), written)) + print(" confirmed {} bytes on device".format(written)) + + +def hard_reset(s): + print(" triggering machine.reset()...") + s.write(b"\r\x01") # re-enter raw REPL just in case + time.sleep(0.3) + drain(s, 200) + s.write(b"import machine\nmachine.reset()\n\x04") + time.sleep(0.5) + + +def main(): + s = serial.Serial(PORT, BAUD, timeout=2) + print("Opened {}".format(PORT)) + try: + enter_raw_repl(s) + for local, remote in PUSH_LIST: + push_file(s, local, remote) + hard_reset(s) + print("Done; device is rebooting") + finally: + s.close() + + +if __name__ == "__main__": + main() diff --git a/embedded/devices/lask5_v2/labeler.py b/embedded/devices/lask5_v2/labeler.py index a5d0f60..9524218 100644 --- a/embedded/devices/lask5_v2/labeler.py +++ b/embedded/devices/lask5_v2/labeler.py @@ -14,10 +14,14 @@ # ADCs are used for the on-OLED bar chart and the Calibrate flow. import time +import machine from machine import Pin, ADC import uasyncio as asyncio import om_logger as log from om_device import BaseDevice +from om_subscribers import Subscribers +from om_discovery import Discovery +from om_commands import CommandServer, build_standard_handlers from sensor_pistons import SensorPistons from display_lask5 import draw_taskbar, play_splash @@ -50,6 +54,15 @@ class LASK5(BaseDevice): # "espnow" broadcasts directly to a paired bracelet/robot hand. # Persisted in settings.json -- can be toggled live from the menu. "stream_mode": "udp", + # Discovery + subscribe protocol additions (see spec section 4-6). + # No udp_target_ip is used for sensor frames in this mode; sources + # unicast to subscribed hubs only. ESP-NOW path is unchanged. + "fw_version": "v3.0.0", + "cmd_port": 8002, + "udp_sensor_port": 3141, # broadcast beacon port; hubs also pick this for default + "announce_interval_s": 1, + "max_subscribers": 4, + "heartbeat_timeout_s": 5, } # Menu UX: a single "Start" button both OPENS the menu (from live) and @@ -101,6 +114,51 @@ def __init__(self): # active, _display_loop yields and lets them write the OLED directly. self._display_owner = "live" # "live" | "menu" | "modal" + # Discovery + subscribe state (V3 protocol): + # subscribers - hubs currently receiving label frames + # _streaming - false halts UDP send_loop without removing subscribers + # _reboot_requested - flag polled by reboot_watcher + # Discovery + CommandServer instances are built in run() after the + # network is up and the Wi-Fi STA has an IP. + self.subscribers = Subscribers( + max_subscribers=self.settings.get("max_subscribers", 4), + heartbeat_timeout_s=self.settings.get("heartbeat_timeout_s", 5), + ) + self._streaming = True + self._reboot_requested = False + + # ----- DeviceState interface for om_commands.build_standard_handlers ----- + # The standard handler factory needs: subscribers, device_id, device_type, + # fw_version, caps, extra_info, start_stream(), stop_stream(), request_reboot() + + @property + def fw_version(self): + return self.settings.get("fw_version", "v3.0.0") + + @property + def device_type(self): + return self.DEVICE_TYPE + + @property + def caps(self): + return ["label", "status", "cmd"] + + @property + def extra_info(self): + return {"pistons": 4, "joystick": True, "espnow_paired": True} + + def start_stream(self): + self._streaming = True + log.info("Streaming started") + + def stop_stream(self): + self._streaming = False + log.info("Streaming stopped") + + def request_reboot(self): + self._reboot_requested = True + log.info("Reboot requested") + # ----- helpers ----- def blink(self, count=2, on_ms=300, off_ms=200): @@ -126,36 +184,94 @@ async def run(self): self.blink(2) self.network.init_espnow() - # Streaming + display + menu all run concurrently. UDP and ESPNow - # broadcast in parallel so both the web UI and any peer bracelet - # receive ground-truth labels with no menu gating. + # New in V3 firmware: discovery + subscribe protocol on Wi-Fi. + # ESP-NOW path to OpenHand is untouched. The cmd server binds before + # we spawn any other task so hubs hitting the device immediately + # after boot can connect. + self._discovery = Discovery( + self.settings, self.subscribers, self.network.sta, + device_type=self.DEVICE_TYPE, + services={ + "label": self.settings.get("udp_sensor_port", 3141), + "cmd": self.settings.get("cmd_port", 8002), + }, + caps=self.caps, + extra_fields={"pistons": 4, "joystick": True}, + beacon_port=self.settings.get("udp_sensor_port", 3141), + announce_interval_s=self.settings.get("announce_interval_s", 1), + fw_version=self.fw_version, + device_id=self.device_id, + ) + self._cmd_server = CommandServer( + port=self.settings.get("cmd_port", 8002), + handlers=build_standard_handlers(self), + ) + await self._cmd_server.start() + + # Streaming + display + menu + discovery + subscriber pruning + reboot + # watcher all run concurrently. UDP fans out to subscribers and ESPNow + # broadcasts to the paired bracelet in parallel; both kept independent + # so a Wi-Fi flap does not disturb the ESP-NOW link. asyncio.create_task(self._send_loop()) asyncio.create_task(self._espnow_loop()) asyncio.create_task(self._display_loop()) asyncio.create_task(self._menu_loop()) + asyncio.create_task(self._discovery.announce_loop()) + asyncio.create_task(self._subscriber_prune_loop()) + asyncio.create_task(self._reboot_watcher()) while True: await asyncio.sleep(1) + async def _subscriber_prune_loop(self): + """Drop subscribers whose last heartbeat aged past the configured + timeout. Self-cleans the list when a hub disappears.""" + while True: + try: + dropped = self.subscribers.prune_stale() + if dropped: + log.info("Pruned {} stale subscriber(s); remaining={}".format( + dropped, self.subscribers.count())) + except Exception as e: + log.warn("subscriber_prune_loop failed: {}".format(e)) + await asyncio.sleep(1) + + async def _reboot_watcher(self): + """Polls the reboot flag set by a `reboot` command verb.""" + while True: + if self._reboot_requested: + log.info("Soft-resetting in 500 ms...") + await asyncio.sleep_ms(500) + machine.reset() + await asyncio.sleep_ms(200) + # ----- streaming ----- def _stream_mode(self): return self.settings.get("stream_mode", "udp") async def _send_loop(self): - """UDP send loop. Only transmits when stream_mode == 'udp'. Keeps - running (and reading sensors) even when idle so the on-OLED taskbar - stays responsive immediately after a mode toggle.""" + """UDP send loop. Only transmits when stream_mode == 'udp' AND at + least one hub is subscribed. Keeps reading sensors regardless so + the on-OLED taskbar stays responsive immediately after a mode toggle. + + V3 protocol change: replaces send_udp() to a hardcoded udp_target_ip + with send_udp_to_subscribers() that fans out to every entry in the + live subscriber list. The list is populated by hubs talking to the + cmd server (see om_commands).""" interval = 1.0 / self.settings.get("sample_rate_hz", 25) while True: - if self._stream_mode() == "udp": + if self._stream_mode() == "udp" and self._streaming: data = self.sensor.read() # {"values": [0..1, ...]} data["joystick"] = { "x": self.joystick_x.read(), "y": self.joystick_y.read(), } packet = self.make_packet(data) - await self.network.send_udp(packet) + # send_udp_to_subscribers no-ops when the list is empty, so + # the work above is wasted when idle; we still do it so the + # very first frame after a subscribe is ready to go. + await self.network.send_udp_to_subscribers(packet, self.subscribers) await asyncio.sleep(interval) async def _espnow_loop(self): diff --git a/embedded/lib/om_commands.py b/embedded/lib/om_commands.py new file mode 100644 index 0000000..a9e0cbb --- /dev/null +++ b/embedded/lib/om_commands.py @@ -0,0 +1,167 @@ +# om_commands.py - TCP command server for source-role devices. +# +# Hubs connect to this device's cmd_port and send JSON command messages, +# one per line. The server replies with a JSON ack per line. See spec +# section 6.1 for the standard verb taxonomy. +# +# This module provides: +# - CommandServer: the asyncio TCP server + dispatch loop +# - build_standard_handlers(state): factory for the verbs that every +# source device needs (subscribe / unsubscribe / heartbeat / get_info / +# start_stream / stop_stream / reboot). Device-specific verbs can be +# added on top by the caller. +# +# `state` is an object the device provides. Required attributes: +# subscribers om_subscribers.Subscribers +# device_id str +# device_type str ("flexgrid"|"lask5"|...) +# fw_version str +# caps list[str] +# extra_info dict (merged into get_info responses) +# start_stream() method +# stop_stream() method +# request_reboot() method + +import uasyncio as asyncio +import ujson +import om_logger as log + + +_OK = "ok" +_ERR = "error" + + +class CommandServer: + def __init__(self, port, handlers): + """ + port: TCP port to bind + handlers: dict mapping verb -> async fn(data, peer) -> dict + Each handler returns the ack's `data` field. + Raising any exception turns the ack into an error. + """ + self.port = port + self.handlers = handlers + self._server = None + + async def start(self): + """Bind and start accepting connections in the background.""" + self._server = await asyncio.start_server(self._handle_client, "0.0.0.0", self.port) + log.info("Command server listening on TCP {}".format(self.port)) + + async def _handle_client(self, reader, writer): + peer = writer.get_extra_info("peername") or ("?", 0) + log.info("Command client connected: {}".format(peer)) + try: + while True: + line = await reader.readline() + if not line: + break + line = line.strip() + if not line: + continue + ack = await self._handle_one(line, peer) + writer.write(ujson.dumps(ack).encode("utf-8") + b"\n") + await writer.drain() + except Exception as e: + log.warn("Command client {} errored: {}".format(peer, e)) + finally: + try: + writer.close() + await writer.wait_closed() + except Exception: + pass + log.info("Command client disconnected: {}".format(peer)) + + async def _handle_one(self, raw_line, peer): + try: + pkt = ujson.loads(raw_line.decode("utf-8")) + except Exception as e: + return {"v": "1.0", "type": "ack", "status": _ERR, + "msg_id": None, "data": {"message": "invalid_json: {}".format(e)}} + + msg_id = pkt.get("msg_id") + data = pkt.get("data") or {} + verb = data.get("verb") + if not verb: + return {"v": "1.0", "type": "ack", "status": _ERR, + "msg_id": msg_id, "data": {"message": "missing verb in data"}} + handler = self.handlers.get(verb) + if handler is None: + return {"v": "1.0", "type": "ack", "status": _ERR, + "msg_id": msg_id, "data": {"message": "unknown_verb: " + str(verb)}} + + try: + ack_data = await handler(data, peer) + except Exception as e: + log.warn("Command verb={} from {} raised: {}".format(verb, peer, e)) + return {"v": "1.0", "type": "ack", "status": _ERR, + "msg_id": msg_id, "data": {"verb": verb, "message": str(e)}} + return {"v": "1.0", "type": "ack", "status": _OK, + "msg_id": msg_id, "data": dict({"verb": verb}, **(ack_data or {}))} + + +def build_standard_handlers(state): + """Return a dict of handlers for the verbs every source device needs. + Devices can extend this dict with their own verbs before passing it + to CommandServer.""" + + async def subscribe(data, peer): + host = data.get("host") or peer[0] + port = int(data["port"]) + transport = data.get("transport", "wifi") + hub_id = data.get("hub_id") + ok = state.subscribers.add(host, port, transport, hub_id) + return { + "accepted": ok, + "subscriber_count": state.subscribers.count(), + "max_subscribers": state.subscribers.max_subscribers, + } + + async def unsubscribe(data, peer): + host = data.get("host") or peer[0] + port = int(data["port"]) + transport = data.get("transport", "wifi") + removed = state.subscribers.remove(host, port, transport) + return {"removed": removed, "subscriber_count": state.subscribers.count()} + + async def heartbeat(data, peer): + host = data.get("host") or peer[0] + port = int(data["port"]) + transport = data.get("transport", "wifi") + refreshed = state.subscribers.heartbeat(host, port, transport) + return {"refreshed": refreshed} + + async def get_info(data, peer): + info = { + "id": state.device_id, + "dev": state.device_type, + "fw": state.fw_version, + "caps": list(state.caps), + "subscribers": state.subscribers.snapshot(), + } + # Device-specific extras (matrix dims for FlexGrid, piston count for LASK5, etc.) + for k, v in getattr(state, "extra_info", {}).items(): + info[k] = v + return info + + async def start_stream(data, peer): + state.start_stream() + return {"streaming": True} + + async def stop_stream(data, peer): + state.stop_stream() + return {"streaming": False} + + async def reboot(data, peer): + state.request_reboot() + return {"rebooting": True} + + return { + "subscribe": subscribe, + "unsubscribe": unsubscribe, + "heartbeat": heartbeat, + "get_info": get_info, + "start_stream": start_stream, + "stop_stream": stop_stream, + "reboot": reboot, + } diff --git a/embedded/lib/om_discovery.py b/embedded/lib/om_discovery.py new file mode 100644 index 0000000..bef4e7d --- /dev/null +++ b/embedded/lib/om_discovery.py @@ -0,0 +1,127 @@ +# om_discovery.py - Wi-Fi discovery for source-role devices. +# +# Two parallel mechanisms per spec section 4: +# 1. mDNS hostname registration (best-effort; uses sta.config(hostname=...) +# which some MicroPython builds publish as an mDNS A record). The +# `_openmuscle._udp` service type with TXT records is not registered +# via this path; the UDP broadcast beacon below covers the gap. +# 2. UDP broadcast beacon to 255.255.255.255:beacon_port (default 3141) +# carrying the announce JSON. Reliable; works across consumer routers +# that disable mDNS / multicast. +# +# Cadence: ~1 Hz while no hub is subscribed. Once subscribers.has_any() +# returns True the beacon stops to keep the channel quiet, and resumes +# the moment the subscriber list empties. + +import uasyncio as asyncio +import socket +import ujson +import time +import om_logger as log + + +class Discovery: + def __init__(self, settings, subscribers, sta, + device_type, services, caps, + extra_fields=None, + beacon_port=3141, announce_interval_s=1, + fw_version=None, + device_id=None, + mdns_service="_openmuscle._udp"): + """ + settings: om_settings.Settings instance + subscribers: om_subscribers.Subscribers instance + sta: network.WLAN(STA_IF), used for hostname + IP lookup + device_type: "flexgrid" | "lask5" | "openhand" | ... + services: dict mapping capability name -> port + e.g. {"label": 3141, "cmd": 8002} + caps: list of capability strings, e.g. ["label","status","cmd"] + extra_fields: optional dict merged into the announce payload + (e.g. {"matrix": [15,4]} for FlexGrid) + beacon_port: UDP port to broadcast announces to (default 3141) + device_id: optional override; defaults to settings["device_id"] + fw_version: optional override; defaults to settings.get("fw_version","unknown") + mdns_service: mDNS service type string for documentation + """ + self.settings = settings + self.subscribers = subscribers + self.sta = sta + self.device_type = device_type + self.services = services + self.caps = caps + self.extra_fields = extra_fields or {} + self.beacon_port = beacon_port + self.announce_interval_s = announce_interval_s + self.device_id = device_id or settings.get("device_id") + self.fw_version = fw_version or settings.get("fw_version", "unknown") + self.mdns_service = mdns_service + + self._beacon_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self._beacon_sock.setblocking(False) + try: + self._beacon_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + except Exception as e: + log.warn("SO_BROADCAST setsockopt failed: {}".format(e)) + + self._mdns_registered = False + + def _announce_payload(self): + """Build the announce JSON. The IP is NOT in the payload; consumers + take it from the broadcast packet source or the mDNS A record.""" + pkt = { + "v": "1.0", + "type": "announce", + "id": self.device_id, + "role": "source", + "dev": self.device_type, + "fw": self.fw_version, + "transports": ["wifi"], # phase 4 appends "ble" + "caps": list(self.caps), + "services": dict(self.services), + "ts": time.ticks_ms(), + } + for k, v in self.extra_fields.items(): + pkt[k] = v + return pkt + + def register_mdns(self): + """Best-effort mDNS hostname registration. Some MicroPython builds + publish the STA hostname as an mDNS A record (so `.local` + resolves); others do not. Either way the broadcast beacon is the + reliable discovery path.""" + if self._mdns_registered: + return + try: + self.sta.config(hostname=self.device_id) + except Exception as e: + log.warn("mDNS hostname set failed (beacon will cover it): {}".format(e)) + self._mdns_registered = True + log.info("mDNS hostname best-effort registered as {}".format(self.device_id)) + + async def announce_loop(self): + """Periodic broadcast beacon. Goes quiet once at least one hub is + subscribed; resumes when the subscriber list empties.""" + try: + self.register_mdns() + except Exception: + pass + + while True: + try: + if not self.subscribers.has_any(): + self._send_beacon() + except Exception as e: + log.warn("announce_loop iter failed: {}".format(e)) + await asyncio.sleep(self.announce_interval_s) + + def _send_beacon(self): + try: + payload = ujson.dumps(self._announce_payload()).encode("utf-8") + self._beacon_sock.sendto(payload, ("255.255.255.255", self.beacon_port)) + except OSError as e: + errno = getattr(e, "errno", None) or (e.args[0] if e.args else None) + # ENOMEM (12) and EAGAIN (11) are non-fatal; next tick retries. + if errno not in (11, 12): + log.warn("Beacon send failed: errno={} {}".format(errno, e)) + except Exception as e: + log.warn("Beacon send unexpected error: {}".format(e)) diff --git a/embedded/lib/om_network.py b/embedded/lib/om_network.py index 2a4e747..175b5c6 100644 --- a/embedded/lib/om_network.py +++ b/embedded/lib/om_network.py @@ -72,6 +72,32 @@ def send_udp_sync(self, payload_bytes): except Exception as e: log.error("UDP send error: " + str(e)) + async def send_udp_to_subscribers(self, payload_bytes, subscribers): + """Fan out one UDP payload to every Wi-Fi subscriber in the list. + + New in the discovery + subscribe protocol (spec section 5.1). The + existing send_udp() to a hardcoded target is kept for backward + compat with FlexGrid V1 and any device that has not yet been + migrated to discovery. + + subscribers: om_subscribers.Subscribers instance. + """ + if not self._sock: + return + targets = subscribers.wifi_targets() + if not targets: + return + for host, port in targets: + try: + self._sock.sendto(payload_bytes, (host, port)) + except OSError as e: + errno = getattr(e, "errno", None) or (e.args[0] if e.args else None) + # ENOMEM (12) and EAGAIN (11) are non-fatal; next iter retries. + if errno not in (11, 12): + log.warn("UDP send to {}:{} failed errno={} {}".format(host, port, errno, e)) + except Exception as e: + log.warn("UDP send to {}:{} unexpected: {}".format(host, port, e)) + # --- ESPNOW (optional, for devices that need P2P communication) --- def init_espnow(self): diff --git a/embedded/lib/om_subscribers.py b/embedded/lib/om_subscribers.py new file mode 100644 index 0000000..1deef2c --- /dev/null +++ b/embedded/lib/om_subscribers.py @@ -0,0 +1,113 @@ +# om_subscribers.py - Subscriber list for source-role devices. +# +# A hub (phone, PC) discovers this device via om_discovery, opens the command +# channel exposed by om_commands, and sends a `subscribe` message with its +# (host, port, transport). This module stores the resulting subscriber list, +# fields heartbeats, and prunes entries whose last heartbeat aged past the +# configured timeout (default 5 s per spec section 5.1). +# +# Devices fan out frames to wifi_targets() each scan. + +import time +import om_logger as log + + +class Subscribers: + def __init__(self, max_subscribers=4, heartbeat_timeout_s=5): + self.max_subscribers = max_subscribers + self.heartbeat_timeout_ms = int(heartbeat_timeout_s * 1000) + # Each entry: dict {host, port, transport, hub_id, last_heartbeat_ms} + self._entries = [] + + def add(self, host, port, transport="wifi", hub_id=None): + """Add or refresh a subscriber. Returns True if accepted, False if the + list is full. Refreshing an existing (host, port, transport) tuple + does not consume a new slot.""" + now = time.ticks_ms() + for e in self._entries: + if e["host"] == host and e["port"] == port and e["transport"] == transport: + e["last_heartbeat_ms"] = now + if hub_id: + e["hub_id"] = hub_id + log.info("Subscriber refreshed: {} {}:{} (hub_id={})".format( + transport, host, port, hub_id)) + return True + if len(self._entries) >= self.max_subscribers: + log.warn("Subscriber list full ({}/{}); rejecting {}:{}".format( + len(self._entries), self.max_subscribers, host, port)) + return False + self._entries.append({ + "host": host, + "port": port, + "transport": transport, + "hub_id": hub_id, + "last_heartbeat_ms": now, + }) + log.info("Subscriber added: {} {}:{} (hub_id={}, {}/{})".format( + transport, host, port, hub_id, len(self._entries), self.max_subscribers)) + return True + + def remove(self, host, port, transport="wifi"): + """Explicit unsubscribe. Returns True if found+removed, False if not.""" + for i, e in enumerate(self._entries): + if e["host"] == host and e["port"] == port and e["transport"] == transport: + self._entries.pop(i) + log.info("Subscriber removed: {} {}:{}".format(transport, host, port)) + return True + return False + + def heartbeat(self, host, port, transport="wifi"): + """Refresh a subscriber's heartbeat. If the (host,port,transport) is + not currently subscribed this is a no-op + warning.""" + now = time.ticks_ms() + for e in self._entries: + if e["host"] == host and e["port"] == port and e["transport"] == transport: + e["last_heartbeat_ms"] = now + return True + log.warn("Heartbeat from unknown subscriber {}:{} ({})".format(host, port, transport)) + return False + + def prune_stale(self): + """Drop subscribers whose last heartbeat is older than the timeout. + Returns the number of entries dropped.""" + now = time.ticks_ms() + kept = [] + dropped = 0 + for e in self._entries: + age_ms = time.ticks_diff(now, e["last_heartbeat_ms"]) + if age_ms > self.heartbeat_timeout_ms: + log.info("Subscriber stale, dropping: {}:{} age={}ms".format( + e["host"], e["port"], age_ms)) + dropped += 1 + else: + kept.append(e) + self._entries = kept + return dropped + + def wifi_targets(self): + """List of (host, port) for every active Wi-Fi subscriber.""" + return [(e["host"], e["port"]) for e in self._entries if e["transport"] == "wifi"] + + def ble_targets(self): + """List of hub identifiers for every active BLE subscriber (phase 4).""" + return [e.get("hub_id") for e in self._entries if e["transport"] == "ble"] + + def count(self): + return len(self._entries) + + def has_any(self): + return len(self._entries) > 0 + + def snapshot(self): + """List of subscriber summaries for diagnostics / UI / get_info.""" + now = time.ticks_ms() + return [ + { + "host": e["host"], + "port": e["port"], + "transport": e["transport"], + "hub_id": e.get("hub_id"), + "age_ms": time.ticks_diff(now, e["last_heartbeat_ms"]), + } + for e in self._entries + ] diff --git a/pc/bridge_subscriber.py b/pc/bridge_subscriber.py new file mode 100644 index 0000000..a3ac750 --- /dev/null +++ b/pc/bridge_subscriber.py @@ -0,0 +1,207 @@ +"""Bridge subscriber: keeps the PC subscribed to V4-protocol sources. + +The PC app (`openmuscle web`) does not yet implement the V4 discovery + +subscribe protocol natively (that work lives in the OpenMuscle-Lab-Discovery +workspace). This helper bridges the gap: for each device IP you pass on the +command line, it opens a TCP command channel, sends `subscribe` with the PC's +own address, and sends a `heartbeat` once per second. Devices then unicast +sensor / label frames to the PC's UDP 3141 listener where openmuscle web +picks them up. + +This script does NOT listen on UDP 3141 itself, so it never conflicts with +openmuscle web for the port. Both can run side by side. + +Usage: + python bridge_subscriber.py 10.0.0.23 + python bridge_subscriber.py 10.0.0.23 10.0.0.232 10.0.0.192:8002 + +Default cmd ports: 8001 for FlexGrid, 8002 for LASK5. Append `:` to a +target to override. Auto-detect of device type happens by trying 8001 first +then 8002. + +Press Ctrl-C to unsubscribe cleanly and exit. +""" + +import argparse +import json +import socket +import sys +import threading +import time + + +PROTO_PORT = 3141 # UDP port openmuscle web listens on for sensor frames +HEARTBEAT_S = 1.0 +HUB_ID = "bridge-subscriber" + + +def get_default_pc_ip(test_target=("10.0.0.1", 80)): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(test_target) + return s.getsockname()[0] + except Exception: + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "127.0.0.1" + finally: + s.close() + + +def try_subscribe(ip, port, pc_ip): + """Open TCP, send subscribe, return (tcp_socket, ack_dict) on success or + (None, None) on failure.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(3.0) + s.connect((ip, port)) + msg = json.dumps({ + "v": "1.0", "type": "cmd", "msg_id": 1, + "data": { + "verb": "subscribe", + "host": pc_ip, "port": PROTO_PORT, + "transport": "wifi", "hub_id": HUB_ID, + }, + }) + "\n" + s.send(msg.encode()) + ack = s.recv(2048).decode().strip() + return s, json.loads(ack) + except Exception: + try: + s.close() + except Exception: + pass + return None, None + + +class Keeper: + """One subscription + heartbeat keeper running on its own daemon thread.""" + + def __init__(self, ip, port, pc_ip): + self.ip = ip + self.port = port + self.pc_ip = pc_ip + self.tcp = None + self.alive = True + self.thread = None + + def start(self): + self.tcp, ack = try_subscribe(self.ip, self.port, self.pc_ip) + if not self.tcp: + return False + verb_data = (ack or {}).get("data") or {} + accepted = verb_data.get("accepted") + print(" [{}:{}] subscribe ack: accepted={}, count={}/{}".format( + self.ip, self.port, accepted, + verb_data.get("subscriber_count"), + verb_data.get("max_subscribers"), + )) + if not accepted: + print(" (device rejected subscribe; subscriber list may be full)") + self.tcp.close() + return False + self.thread = threading.Thread(target=self._loop, daemon=True) + self.thread.start() + return True + + def _loop(self): + msg_id = 100 + while self.alive: + time.sleep(HEARTBEAT_S) + if not self.alive: + break + msg = json.dumps({ + "v": "1.0", "type": "cmd", "msg_id": msg_id, + "data": { + "verb": "heartbeat", + "host": self.pc_ip, "port": PROTO_PORT, + "transport": "wifi", + }, + }) + "\n" + try: + self.tcp.send(msg.encode()) + self.tcp.recv(2048) + msg_id += 1 + except Exception as e: + print(" [{}:{}] heartbeat failed: {}; restarting subscribe".format( + self.ip, self.port, e)) + try: + self.tcp.close() + except Exception: + pass + # Try to re-subscribe + self.tcp, _ack = try_subscribe(self.ip, self.port, self.pc_ip) + if not self.tcp: + print(" [{}:{}] re-subscribe failed; giving up".format(self.ip, self.port)) + self.alive = False + break + msg_id = 100 + + def stop(self): + self.alive = False + if not self.tcp: + return + try: + self.tcp.send((json.dumps({ + "v": "1.0", "type": "cmd", "msg_id": 999, + "data": {"verb": "unsubscribe", "host": self.pc_ip, "port": PROTO_PORT}, + }) + "\n").encode()) + self.tcp.recv(2048) + except Exception: + pass + try: + self.tcp.close() + except Exception: + pass + + +def parse_target(target_str): + """`'1.2.3.4'` -> [(1.2.3.4, 8001), (1.2.3.4, 8002)] for autodetect. + `'1.2.3.4:8002'` -> [(1.2.3.4, 8002)] exact.""" + if ":" in target_str: + ip, port = target_str.split(":", 1) + return [(ip, int(port))] + return [(target_str, 8001), (target_str, 8002)] + + +def main(): + p = argparse.ArgumentParser(description="Keep PC subscribed to V4 sources") + p.add_argument("targets", nargs="+", help="Device IPs (default ports 8001/8002 tried)") + p.add_argument("--pc-ip", default=None, help="Override PC's own LAN IP") + args = p.parse_args() + + pc_ip = args.pc_ip or get_default_pc_ip() + print("PC IP: {} (sensor/label frames will arrive at this address, UDP {})".format( + pc_ip, PROTO_PORT)) + + keepers = [] + for t in args.targets: + for ip, port in parse_target(t): + print("trying {}:{}".format(ip, port)) + k = Keeper(ip, port, pc_ip) + if k.start(): + keepers.append(k) + break + else: + print(" no responsive cmd server at {}".format(t)) + + if not keepers: + print("\nNo subscriptions established. Check device IPs + cmd ports.") + sys.exit(1) + + print("\n{} subscription(s) active. Heartbeating every {}s.".format(len(keepers), HEARTBEAT_S)) + print("Open http://localhost:8000 (openmuscle web) to see the live data.") + print("Press Ctrl-C to unsubscribe + exit.") + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + print("\nshutting down. unsubscribing...") + for k in keepers: + k.stop() + print("done.") + + +if __name__ == "__main__": + main() From 62a255400b7a7b57a1ff621aebd9da2b9fc5994f Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 23 Jun 2026 22:34:59 -0500 Subject: [PATCH 54/56] discovery: native V4 discovery + subscribe (retires bridge_subscriber) Adds openmuscle.discovery.DiscoveryManager: discovers V4 sources via passive announce beacons, a persisted device cache (re-probed with get_info on startup), and manual probe, then keeps a TCP subscription with a 1 Hz heartbeat to each so the existing UDP listener receives their sensor frames unchanged. Beacons go silent once any hub subscribes (per the phone team), so the cache + get_info recovery path is required to find devices another hub already holds; passive listening alone is not enough. Plumbing: parser routes type=="announce" through .data (full announce preserved) instead of creating a phantom device; udp_listener gains an optional announce_handler that diverts beacons to discovery and never enqueues them as sensor frames. Non-web consumers (inference/heatmap CLI) are unchanged. Verified: 13 new tests (announce parse, cache round-trip, subscribe/heartbeat/ unsubscribe/get_info against a fake V4 command server, auto-subscribe); full suite 55 passing. End-to-end against the live V4 board flexgrid-d7af0b: probe + subscribe + 46 real 15x4 frames in 3s + clean unsubscribe. Retires pc/bridge_subscriber.py once wired into openmuscle web (next step). Co-Authored-By: turfptax-claude O4.8 --- pc/src/openmuscle/discovery/__init__.py | 10 + pc/src/openmuscle/discovery/manager.py | 433 +++++++++++++++++++++ pc/src/openmuscle/protocol/parser.py | 14 + pc/src/openmuscle/receiver/udp_listener.py | 21 +- pc/tests/test_discovery.py | 307 +++++++++++++++ 5 files changed, 782 insertions(+), 3 deletions(-) create mode 100644 pc/src/openmuscle/discovery/__init__.py create mode 100644 pc/src/openmuscle/discovery/manager.py create mode 100644 pc/tests/test_discovery.py diff --git a/pc/src/openmuscle/discovery/__init__.py b/pc/src/openmuscle/discovery/__init__.py new file mode 100644 index 0000000..41466d1 --- /dev/null +++ b/pc/src/openmuscle/discovery/__init__.py @@ -0,0 +1,10 @@ +"""Native OpenMuscle V4 discovery + subscribe for the PC hub. + +Replaces the interim pc/bridge_subscriber.py: discovers V4 sources and keeps a +TCP subscription (1 Hz heartbeat) to each, so the existing UDP listener receives +their sensor frames with no other change. +""" + +from openmuscle.discovery.manager import DiscoveryManager, DiscoveredDevice + +__all__ = ["DiscoveryManager", "DiscoveredDevice"] diff --git a/pc/src/openmuscle/discovery/manager.py b/pc/src/openmuscle/discovery/manager.py new file mode 100644 index 0000000..b044602 --- /dev/null +++ b/pc/src/openmuscle/discovery/manager.py @@ -0,0 +1,433 @@ +"""Native V4 discovery + subscribe manager for the PC hub. + +Replaces the interim pc/bridge_subscriber.py. Discovers OpenMuscle V4 sources +and keeps a TCP subscription (with a 1 Hz heartbeat) to each, so the existing +UDP 3141 listener receives their sensor frames with no change to the parse or +state pipeline. + +Three discovery paths, because no single one is sufficient: + + 1. Passive beacons. V4 devices broadcast an `announce` to 255.255.255.255:3141 + at ~1 Hz WHILE THEY HAVE NO SUBSCRIBER. The moment any hub subscribes, the + beacon goes silent and resumes only when the subscriber list empties + (confirmed by the phone team). So passive listening alone misses any device + another hub (or our own prior run) already holds. + 2. Cache. Every device we have ever seen is persisted (id -> ip, cmd_port, + device_type). On startup we actively re-probe cached devices with the + `get_info` command, recovering them even when their beacon is silent. + 3. Manual probe. Probe an explicit ip:cmd_port (lab setup / UI "add device"). + +Discovered + reachable devices are auto-subscribed when auto_subscribe is set. +The contract this implements is documented in FlexGridV4-Firmware/lib/ +{discovery,commands,subscribers,network_manager}.py and confirmed on the +coordination board (vrpc #0008). +""" + +import json +import os +import socket +import threading +import time + +DEFAULT_UDP_PORT = 3141 # where we ask sources to unicast sensor frames +DEFAULT_CMD_PORTS = (8001, 8002) # flexgrid cmd, lask5 cmd (probe order) +HEARTBEAT_S = 1.0 # firmware drops a subscriber after ~5 s silence +TCP_TIMEOUT_S = 3.0 +RESUBSCRIBE_BACKOFF_S = 2.0 # wait between failed re-subscribe attempts +HUB_ID = "pc-native-discovery" + + +def default_lan_ip(test_target=("10.0.0.1", 80)): + """Best-effort local LAN IP (the address sources should unicast to). + + Opens a throwaway UDP socket toward the gateway and reads the chosen + source address; no packet is actually sent. Falls back to hostname + resolution, then loopback. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(test_target) + return s.getsockname()[0] + except Exception: + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "127.0.0.1" + finally: + s.close() + + +def default_cache_path(): + """~/.openmuscle/discovered_devices.json (override via DiscoveryManager).""" + return os.path.join(os.path.expanduser("~"), ".openmuscle", + "discovered_devices.json") + + +class DiscoveredDevice: + """A source we have discovered and may be subscribed to. + + Plain class (not a dataclass) so it stays import-light and easy to mutate + under the manager lock. + """ + + __slots__ = ("device_id", "device_type", "ip", "cmd_port", "sensor_port", + "fw", "caps", "matrix", "source", "last_seen", + "subscribed", "sub_error") + + def __init__(self, device_id, device_type, ip, cmd_port, + sensor_port=DEFAULT_UDP_PORT, fw="", caps=None, matrix=None, + source="beacon", last_seen=0.0): + self.device_id = device_id + self.device_type = device_type + self.ip = ip + self.cmd_port = int(cmd_port) + self.sensor_port = int(sensor_port) + self.fw = fw + self.caps = list(caps or []) + self.matrix = list(matrix or []) + self.source = source + self.last_seen = last_seen + self.subscribed = False + self.sub_error = "" + + def to_cache(self): + """Minimal persisted form (enough to re-probe next run).""" + return { + "device_id": self.device_id, + "device_type": self.device_type, + "ip": self.ip, + "cmd_port": self.cmd_port, + "sensor_port": self.sensor_port, + } + + def to_snapshot(self, now=None): + now = now if now is not None else time.time() + return { + "device_id": self.device_id, + "device_type": self.device_type, + "ip": self.ip, + "cmd_port": self.cmd_port, + "sensor_port": self.sensor_port, + "fw": self.fw, + "caps": list(self.caps), + "matrix": list(self.matrix), + "source": self.source, + "age_s": round(now - self.last_seen, 2) if self.last_seen else None, + "subscribed": self.subscribed, + "sub_error": self.sub_error, + } + + +def _send_cmd(sock, msg_id, verb, **fields): + """Encode + send one newline-delimited command, return the parsed ack dict. + + Raises on socket error or malformed ack so callers can react (re-subscribe). + """ + msg = {"v": "1.0", "type": "cmd", "msg_id": msg_id, + "data": dict({"verb": verb}, **fields)} + sock.sendall((json.dumps(msg) + "\n").encode("utf-8")) + raw = sock.recv(4096) + if not raw: + raise ConnectionError("command channel closed (empty read)") + # Firmware sends one ack JSON object per line; take the first line. + line = raw.split(b"\n", 1)[0].strip() + return json.loads(line.decode("utf-8")) + + +class _SubscriptionKeeper: + """Owns one device's TCP command channel: subscribe, then heartbeat at + 1 Hz, auto-resubscribing on failure. Sensor frames arrive out-of-band on + the manager's UDP port; this object never touches them. + """ + + def __init__(self, device, pc_host, udp_port, on_state): + self.device = device + self.pc_host = pc_host + self.udp_port = udp_port + self.on_state = on_state # callback(device) on any state change + self._stop = threading.Event() + self._tcp = None + self._thread = None + self._msg_id = 1 + + def start(self): + """Subscribe once synchronously; on success spawn the heartbeat thread. + Returns True if the device accepted the subscription.""" + if not self._connect_and_subscribe(): + return False + self._thread = threading.Thread( + target=self._loop, name="om-keeper-" + self.device.device_id, + daemon=True) + self._thread.start() + return True + + def _connect_and_subscribe(self): + try: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.settimeout(TCP_TIMEOUT_S) + tcp.connect((self.device.ip, self.device.cmd_port)) + ack = _send_cmd(tcp, self._next_id(), "subscribe", + host=self.pc_host, port=self.udp_port, + transport="wifi", hub_id=HUB_ID) + data = (ack or {}).get("data") or {} + if not data.get("accepted"): + tcp.close() + self.device.subscribed = False + self.device.sub_error = "rejected (list full?)" + self._emit() + return False + self._tcp = tcp + self.device.subscribed = True + self.device.sub_error = "" + self._emit() + return True + except Exception as e: + self.device.subscribed = False + self.device.sub_error = str(e) + self._emit() + return False + + def _loop(self): + while not self._stop.is_set(): + if self._stop.wait(HEARTBEAT_S): + break + try: + _send_cmd(self._tcp, self._next_id(), "heartbeat", + host=self.pc_host, port=self.udp_port, + transport="wifi") + except Exception as e: + # Connection dropped (device reboot, Wi-Fi blip). Try to + # re-establish; back off so we don't spin on a dead host. + self.device.subscribed = False + self.device.sub_error = "heartbeat failed: {}".format(e) + self._emit() + self._safe_close() + if self._stop.wait(RESUBSCRIBE_BACKOFF_S): + break + self._connect_and_subscribe() + + def stop(self): + self._stop.set() + if self._tcp is not None: + try: + _send_cmd(self._tcp, 999, "unsubscribe", + host=self.pc_host, port=self.udp_port, + transport="wifi") + except Exception: + pass + self._safe_close() + self.device.subscribed = False + self._emit() + + def _safe_close(self): + if self._tcp is not None: + try: + self._tcp.close() + except Exception: + pass + self._tcp = None + + def _next_id(self): + self._msg_id += 1 + return self._msg_id + + def _emit(self): + if self.on_state: + try: + self.on_state(self.device) + except Exception: + pass + + +class DiscoveryManager: + """Tracks discovered V4 sources and keeps subscriptions to them. + + Thread model: on_announce() runs on the UDP listener thread; probe() and + recover_cache() run on their own short-lived threads; snapshot() runs on + the web request thread. A single lock guards the device + keeper maps. + """ + + def __init__(self, pc_host=None, udp_port=DEFAULT_UDP_PORT, + cache_path=None, auto_subscribe=True): + self.pc_host = pc_host or default_lan_ip() + self.udp_port = udp_port + self.cache_path = cache_path if cache_path is not None else default_cache_path() + self.auto_subscribe = auto_subscribe + self._devices = {} # device_id -> DiscoveredDevice + self._keepers = {} # device_id -> _SubscriptionKeeper + self._lock = threading.Lock() + + # ---- discovery inputs ------------------------------------------------- + + def on_announce(self, obj, src_ip): + """Handle one announce beacon. `obj` is the decoded announce dict; + `src_ip` is the datagram source (the announce never carries its own IP). + """ + try: + device_id = obj["id"] + except (KeyError, TypeError): + return None + services = obj.get("services") or {} + cmd_port = int(services.get("cmd", DEFAULT_CMD_PORTS[0])) + sensor_port = int(services.get("sensor", self.udp_port)) + dev = DiscoveredDevice( + device_id=device_id, + device_type=obj.get("dev", "unknown"), + ip=src_ip, + cmd_port=cmd_port, + sensor_port=sensor_port, + fw=obj.get("fw", ""), + caps=obj.get("caps"), + matrix=obj.get("matrix"), + source="beacon", + last_seen=time.time(), + ) + return self._upsert(dev) + + def probe(self, ip, cmd_port=None): + """Actively query ip:cmd_port (or try the default ports) with get_info. + Returns the DiscoveredDevice on success, or None. Used for cache + recovery and manual add.""" + ports = [int(cmd_port)] if cmd_port else list(DEFAULT_CMD_PORTS) + for port in ports: + try: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.settimeout(TCP_TIMEOUT_S) + tcp.connect((ip, port)) + ack = _send_cmd(tcp, 1, "get_info") + tcp.close() + except Exception: + continue + data = (ack or {}).get("data") or {} + if not data.get("id"): + continue + dev = DiscoveredDevice( + device_id=data["id"], + device_type=data.get("dev", "unknown"), + ip=ip, + cmd_port=port, + sensor_port=self.udp_port, + fw=data.get("fw", ""), + caps=data.get("caps"), + matrix=data.get("matrix"), + source="probe", + last_seen=time.time(), + ) + return self._upsert(dev) + return None + + def recover_cache(self, background=True): + """Re-probe every cached device so we recover ones whose beacon is + silent (held by another hub). Non-blocking by default.""" + cached = self._load_cache() + if not cached: + return + + def _work(): + for entry in cached: + ip = entry.get("ip") + port = entry.get("cmd_port") + if ip: + self.probe(ip, port) + + if background: + threading.Thread(target=_work, name="om-cache-recover", + daemon=True).start() + else: + _work() + + # ---- subscription control -------------------------------------------- + + def subscribe(self, device_id): + """Start (or restart) a subscription keeper for a known device.""" + with self._lock: + dev = self._devices.get(device_id) + if dev is None: + return False + existing = self._keepers.get(device_id) + if existing is not None: + return dev.subscribed + keeper = _SubscriptionKeeper(dev, self.pc_host, self.udp_port, + self._on_keeper_state) + self._keepers[device_id] = keeper + ok = keeper.start() + if not ok: + with self._lock: + self._keepers.pop(device_id, None) + return ok + + def unsubscribe(self, device_id): + with self._lock: + keeper = self._keepers.pop(device_id, None) + if keeper is not None: + keeper.stop() + return True + return False + + def snapshot(self): + """List of device dicts for the web UI / API.""" + now = time.time() + with self._lock: + return [d.to_snapshot(now) for d in self._devices.values()] + + def stop(self): + with self._lock: + keepers = list(self._keepers.values()) + self._keepers.clear() + for k in keepers: + k.stop() + + # ---- internals -------------------------------------------------------- + + def _upsert(self, dev): + """Merge a freshly discovered device into the table, persist the cache, + and auto-subscribe if enabled and not already subscribed.""" + with self._lock: + existing = self._devices.get(dev.device_id) + if existing is None: + self._devices[dev.device_id] = dev + target = dev + else: + # Refresh address/metadata + last_seen; keep subscription state. + existing.ip = dev.ip + existing.cmd_port = dev.cmd_port + existing.sensor_port = dev.sensor_port + existing.device_type = dev.device_type + existing.fw = dev.fw or existing.fw + existing.caps = dev.caps or existing.caps + existing.matrix = dev.matrix or existing.matrix + existing.source = dev.source + existing.last_seen = dev.last_seen + target = existing + already_subscribed = target.device_id in self._keepers + self._save_cache() + if self.auto_subscribe and not already_subscribed: + self.subscribe(target.device_id) + return target + + def _on_keeper_state(self, device): + # Keeper mutated device.subscribed/sub_error in place; nothing else to + # do today, but this is where UI push / logging would hook in. + pass + + def _load_cache(self): + try: + with open(self.cache_path, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, list) else [] + except (FileNotFoundError, ValueError, OSError): + return [] + + def _save_cache(self): + if not self.cache_path: + return + with self._lock: + entries = [d.to_cache() for d in self._devices.values()] + try: + os.makedirs(os.path.dirname(self.cache_path), exist_ok=True) + tmp = self.cache_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(entries, f, indent=2) + os.replace(tmp, self.cache_path) + except OSError: + pass diff --git a/pc/src/openmuscle/protocol/parser.py b/pc/src/openmuscle/protocol/parser.py index 91f9200..e79f2a1 100644 --- a/pc/src/openmuscle/protocol/parser.py +++ b/pc/src/openmuscle/protocol/parser.py @@ -34,6 +34,20 @@ def parse_packet(raw_bytes: bytes) -> OpenMusclePacket | None: # New protocol: JSON object with "v" field if isinstance(obj, dict) and "v" in obj: + # V4 discovery announce beacons share UDP 3141 with sensor frames. + # They are NOT sensor data: route the whole announce dict through .data + # so the listener can hand it to the DiscoveryManager and keep it out of + # the sensor pipeline (otherwise it becomes a phantom "announce" device). + if obj.get("type") == "announce": + return OpenMusclePacket( + version=obj["v"], + device_type="announce", + device_id=obj.get("id", ""), + timestamp_ms=obj.get("ts", 0), + data=obj, + metadata={}, + receive_time=recv_ts, + ) return OpenMusclePacket( version=obj["v"], device_type=obj["type"], diff --git a/pc/src/openmuscle/receiver/udp_listener.py b/pc/src/openmuscle/receiver/udp_listener.py index 534c9e1..e359b83 100644 --- a/pc/src/openmuscle/receiver/udp_listener.py +++ b/pc/src/openmuscle/receiver/udp_listener.py @@ -18,10 +18,15 @@ class UDPListener: # process pkt """ - def __init__(self, port: int = 3141, bind_ip: str = "0.0.0.0"): + def __init__(self, port: int = 3141, bind_ip: str = "0.0.0.0", + announce_handler=None): self.port = port self.bind_ip = bind_ip self.packet_queue: Queue = Queue() + # Optional callback(announce_dict, src_ip) for V4 discovery beacons. + # When set, announce packets are routed here and NOT enqueued as sensor + # frames. When None (CLI inference/heatmap tools), behavior is unchanged. + self.announce_handler = announce_handler self._running = False self._thread = None @@ -44,8 +49,18 @@ def _listen(self): try: data, addr = sock.recvfrom(8192) pkt = parse_packet(data) - if pkt: - self.packet_queue.put(pkt) + if pkt is None: + continue + # Divert V4 discovery beacons to the discovery handler; the + # announce carries no IP, so pass the datagram source address. + if pkt.device_type == "announce": + if self.announce_handler is not None: + try: + self.announce_handler(pkt.data, addr[0]) + except Exception as e: + print(f"announce handler error: {e}") + continue + self.packet_queue.put(pkt) except socket.timeout: continue except Exception as e: diff --git a/pc/tests/test_discovery.py b/pc/tests/test_discovery.py new file mode 100644 index 0000000..4504606 --- /dev/null +++ b/pc/tests/test_discovery.py @@ -0,0 +1,307 @@ +"""Tests for native V4 discovery + subscribe (openmuscle.discovery). + +The centerpiece is a fake in-process TCP command server that mimics the +firmware's lib/commands.py dispatch (newline-delimited JSON, {msg_id, data: +{verb,...}} -> ack). It validates the keeper's subscribe/heartbeat/get_info/ +unsubscribe message shapes against a server that parses them the way a real V4 +device does, with no hardware. +""" + +import json +import socket +import threading +import time + +import pytest + +from openmuscle.discovery.manager import ( + DiscoveryManager, DiscoveredDevice, _send_cmd, +) +from openmuscle.protocol.parser import parse_packet + + +class FakeV4CmdServer: + """Minimal stand-in for a V4 device's command channel + subscriber list. + + Speaks the same wire protocol as FlexGridV4-Firmware/lib/commands.py: + one JSON command per line, one JSON ack per line. + """ + + def __init__(self, device_id="flexgrid-test01", device_type="flexgrid", + matrix=(15, 4), max_subscribers=4, accept=True): + self.device_id = device_id + self.device_type = device_type + self.matrix = list(matrix) + self.max_subscribers = max_subscribers + self.accept = accept + self.subscribers = [] # list of (host, port) + self.received = [] # every (verb, data) we got, for asserts + self._srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._srv.bind(("127.0.0.1", 0)) + self._srv.listen(4) + self.port = self._srv.getsockname()[1] + self._running = True + self._thread = threading.Thread(target=self._accept_loop, daemon=True) + self._thread.start() + + def _accept_loop(self): + while self._running: + try: + self._srv.settimeout(0.5) + conn, _ = self._srv.accept() + except (socket.timeout, OSError): + continue + threading.Thread(target=self._client, args=(conn,), + daemon=True).start() + + def _client(self, conn): + buf = b"" + try: + while self._running: + conn.settimeout(0.5) + try: + chunk = conn.recv(4096) + except socket.timeout: + continue + if not chunk: + break + buf += chunk + while b"\n" in buf: + line, buf = buf.split(b"\n", 1) + line = line.strip() + if line: + conn.sendall(self._handle(line) + b"\n") + except OSError: + pass + finally: + conn.close() + + def _handle(self, line): + try: + pkt = json.loads(line.decode("utf-8")) + except Exception as e: + return self._ack(None, "error", {"message": "invalid_json: %s" % e}) + msg_id = pkt.get("msg_id") + data = pkt.get("data") or {} + verb = data.get("verb") + self.received.append((verb, data)) + if verb == "subscribe": + host, port = data.get("host"), int(data["port"]) + accepted = self.accept and len(self.subscribers) < self.max_subscribers + if accepted and (host, port) not in self.subscribers: + self.subscribers.append((host, port)) + return self._ack(msg_id, "ok", { + "verb": verb, "accepted": accepted, + "subscriber_count": len(self.subscribers), + "max_subscribers": self.max_subscribers}) + if verb == "heartbeat": + return self._ack(msg_id, "ok", {"verb": verb, "refreshed": True}) + if verb == "unsubscribe": + host, port = data.get("host"), int(data["port"]) + if (host, port) in self.subscribers: + self.subscribers.remove((host, port)) + return self._ack(msg_id, "ok", {"verb": verb, "removed": True}) + if verb == "get_info": + return self._ack(msg_id, "ok", { + "verb": verb, "id": self.device_id, "dev": self.device_type, + "fw": "v4.0.0", "matrix": self.matrix, "caps": ["sensor"], + "subscribers": []}) + return self._ack(msg_id, "error", {"message": "unknown_verb"}) + + @staticmethod + def _ack(msg_id, status, data): + return (json.dumps({"v": "1.0", "type": "ack", "status": status, + "msg_id": msg_id, "data": data})).encode("utf-8") + + def stop(self): + self._running = False + try: + self._srv.close() + except OSError: + pass + + +@pytest.fixture +def server(): + s = FakeV4CmdServer() + yield s + s.stop() + + +@pytest.fixture +def mgr(tmp_path): + m = DiscoveryManager(pc_host="127.0.0.1", udp_port=3141, + cache_path=str(tmp_path / "cache.json"), + auto_subscribe=False) + yield m + m.stop() + + +# ---- announce parsing ----------------------------------------------------- + +def _announce(device_id="flexgrid-d7af0b", dev="flexgrid", cmd=8001): + return { + "v": "1.0", "type": "announce", "id": device_id, "role": "source", + "dev": dev, "fw": "v4.0.0", "transports": ["wifi"], "caps": ["sensor"], + "matrix": [15, 4], "services": {"sensor": 3141, "cmd": cmd}, + "ts": 12345, + } + + +def test_on_announce_creates_device(mgr): + dev = mgr.on_announce(_announce(), "10.0.0.23") + assert dev.device_id == "flexgrid-d7af0b" + assert dev.device_type == "flexgrid" + assert dev.ip == "10.0.0.23" # taken from packet source, not payload + assert dev.cmd_port == 8001 + assert dev.sensor_port == 3141 + assert dev.matrix == [15, 4] + assert dev.source == "beacon" + assert mgr.snapshot()[0]["device_id"] == "flexgrid-d7af0b" + + +def test_on_announce_refreshes_not_duplicates(mgr): + mgr.on_announce(_announce(), "10.0.0.23") + mgr.on_announce(_announce(), "10.0.0.99") # same id, new IP + snap = mgr.snapshot() + assert len(snap) == 1 + assert snap[0]["ip"] == "10.0.0.99" + + +def test_on_announce_ignores_garbage(mgr): + assert mgr.on_announce({}, "10.0.0.1") is None + assert mgr.on_announce(None, "10.0.0.1") is None + assert mgr.snapshot() == [] + + +# ---- parser + announce routing ------------------------------------------- + +def test_parser_routes_announce_to_data(): + raw = json.dumps(_announce()).encode("utf-8") + pkt = parse_packet(raw) + assert pkt.device_type == "announce" + assert pkt.data["services"]["cmd"] == 8001 # full announce preserved + assert pkt.data["dev"] == "flexgrid" + + +def test_parser_sensor_frame_unaffected(): + frame = {"v": "1.0", "type": "flexgrid", "id": "flexgrid-d7af0b", + "ts": 1, "seq": 5, "data": {"matrix": [[1, 2], [3, 4]]}} + pkt = parse_packet(json.dumps(frame).encode("utf-8")) + assert pkt.device_type == "flexgrid" + assert pkt.data["matrix"] == [[1, 2], [3, 4]] + + +# ---- cache round-trip ----------------------------------------------------- + +def test_cache_persists_and_reloads(tmp_path): + path = str(tmp_path / "c.json") + m1 = DiscoveryManager(pc_host="127.0.0.1", cache_path=path, + auto_subscribe=False) + m1.on_announce(_announce(device_id="flexgrid-aaa"), "10.0.0.5") + m1.stop() + # A fresh manager reads the same cache file. + m2 = DiscoveryManager(pc_host="127.0.0.1", cache_path=path, + auto_subscribe=False) + cached = m2._load_cache() + assert any(e["device_id"] == "flexgrid-aaa" and e["ip"] == "10.0.0.5" + for e in cached) + m2.stop() + + +# ---- subscribe / heartbeat handshake against the fake device -------------- + +def test_subscribe_handshake(server, mgr): + dev = DiscoveredDevice("flexgrid-test01", "flexgrid", "127.0.0.1", + server.port) + mgr._devices[dev.device_id] = dev + assert mgr.subscribe(dev.device_id) is True + assert dev.subscribed is True + # Server recorded our subscribe with the right host/port/transport. + verbs = [v for v, _ in server.received] + assert "subscribe" in verbs + sub = next(d for v, d in server.received if v == "subscribe") + assert sub["host"] == "127.0.0.1" + assert sub["port"] == 3141 + assert sub["transport"] == "wifi" + assert ("127.0.0.1", 3141) in server.subscribers + + +def test_heartbeats_flow(server, mgr): + dev = DiscoveredDevice("flexgrid-test01", "flexgrid", "127.0.0.1", + server.port) + mgr._devices[dev.device_id] = dev + mgr.subscribe(dev.device_id) + # Wait for at least two heartbeats (1 Hz). + deadline = time.time() + 3.0 + while time.time() < deadline: + if sum(1 for v, _ in server.received if v == "heartbeat") >= 2: + break + time.sleep(0.1) + hb = sum(1 for v, _ in server.received if v == "heartbeat") + assert hb >= 2, "expected >=2 heartbeats, got %d" % hb + + +def test_unsubscribe_clean(server, mgr): + dev = DiscoveredDevice("flexgrid-test01", "flexgrid", "127.0.0.1", + server.port) + mgr._devices[dev.device_id] = dev + mgr.subscribe(dev.device_id) + assert ("127.0.0.1", 3141) in server.subscribers + mgr.unsubscribe(dev.device_id) + # Give the unsubscribe a moment to land. + deadline = time.time() + 2.0 + while time.time() < deadline and server.subscribers: + time.sleep(0.05) + assert server.subscribers == [] + assert dev.subscribed is False + + +def test_subscribe_rejected_when_full(mgr): + server = FakeV4CmdServer(accept=False) + try: + dev = DiscoveredDevice("flexgrid-full", "flexgrid", "127.0.0.1", + server.port) + mgr._devices[dev.device_id] = dev + assert mgr.subscribe(dev.device_id) is False + assert dev.subscribed is False + assert "rejected" in dev.sub_error + finally: + server.stop() + + +# ---- probe / get_info ----------------------------------------------------- + +def test_probe_get_info(server, mgr): + dev = mgr.probe("127.0.0.1", server.port) + assert dev is not None + assert dev.device_id == "flexgrid-test01" + assert dev.device_type == "flexgrid" + assert dev.source == "probe" + assert dev.matrix == [15, 4] + + +def test_probe_unreachable_returns_none(mgr): + # Nothing listening on this port. + assert mgr.probe("127.0.0.1", 1) is None + + +# ---- auto-subscribe path -------------------------------------------------- + +def test_auto_subscribe_on_announce(server, tmp_path): + m = DiscoveryManager(pc_host="127.0.0.1", udp_port=3141, + cache_path=str(tmp_path / "c.json"), + auto_subscribe=True) + try: + # Announce points cmd at the fake server's port. + ann = _announce(device_id="flexgrid-test01", cmd=server.port) + m.on_announce(ann, "127.0.0.1") + deadline = time.time() + 3.0 + while time.time() < deadline: + if ("127.0.0.1", 3141) in server.subscribers: + break + time.sleep(0.05) + assert ("127.0.0.1", 3141) in server.subscribers + finally: + m.stop() From 03f0e24c8cd92f12b6da3914a86cdea2cff03b78 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 23 Jun 2026 22:39:41 -0500 Subject: [PATCH 55/56] web: wire native discovery into openmuscle web + REST API AppState now owns a DiscoveryManager (enable_discovery, default on): the UDP listener routes announce beacons to it, cached devices are re-probed on startup, and discovered sources are auto-subscribed. Running `openmuscle web` on the lab LAN now finds and subscribes to V4 sources with no manual bridge_subscriber.py. REST: GET /api/discovery (known sources + sub state), POST /api/discovery/probe (manual add by ip[:cmd_port]), POST /api/discovery/{subscribe,unsubscribe}. The WS snapshot gains a `discovery` array for the UI panel (next). Verified end-to-end through the running web server against the live V4 board flexgrid-d7af0b: auto-discovered via beacon and manual probe both subscribe; /api/devices then streams real 15x4 frames at ~15 Hz with full status telemetry (IMU, vbat, rssi, subscriber count); clean unsubscribe. Full suite 55 passing. Co-Authored-By: turfptax-claude O4.8 --- pc/src/openmuscle/web/app.py | 46 ++++++++++++++++++++++++++++++++++ pc/src/openmuscle/web/state.py | 30 ++++++++++++++++++++-- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/pc/src/openmuscle/web/app.py b/pc/src/openmuscle/web/app.py index dd17b93..b7a6128 100644 --- a/pc/src/openmuscle/web/app.py +++ b/pc/src/openmuscle/web/app.py @@ -186,6 +186,52 @@ async def ws_quest(websocket: WebSocket): async def list_devices(): return JSONResponse(state._snapshot()["devices"]) + # ----- REST: native V4 discovery ----- + + @app.get("/api/discovery") + async def list_discovery(): + """Known V4 sources + subscription state (auto-discover replaces the + manual bridge_subscriber.py).""" + return JSONResponse(state.discovery.snapshot() if state.discovery else []) + + class ProbeBody(BaseModel): + ip: str + cmd_port: Optional[int] = None # try 8001 then 8002 when omitted + + @app.post("/api/discovery/probe") + async def discovery_probe(body: ProbeBody): + """Manually probe an address (lab setup / device not beaconing).""" + if not state.discovery: + raise HTTPException(status_code=409, detail="discovery disabled") + dev = await asyncio.to_thread(state.discovery.probe, body.ip, body.cmd_port) + if dev is None: + raise HTTPException( + status_code=404, + detail=f"no V4 command server responded at {body.ip}") + return dev.to_snapshot() + + class DeviceIdBody(BaseModel): + device_id: str + + @app.post("/api/discovery/subscribe") + async def discovery_subscribe(body: DeviceIdBody): + if not state.discovery: + raise HTTPException(status_code=409, detail="discovery disabled") + ok = await asyncio.to_thread(state.discovery.subscribe, body.device_id) + if not ok: + raise HTTPException( + status_code=400, + detail=f"could not subscribe to {body.device_id} " + f"(unknown, unreachable, or list full)") + return {"device_id": body.device_id, "subscribed": True} + + @app.post("/api/discovery/unsubscribe") + async def discovery_unsubscribe(body: DeviceIdBody): + if not state.discovery: + raise HTTPException(status_code=409, detail="discovery disabled") + removed = state.discovery.unsubscribe(body.device_id) + return {"device_id": body.device_id, "unsubscribed": bool(removed)} + # ----- REST: recording ----- class StartRecordingBody(BaseModel): diff --git a/pc/src/openmuscle/web/state.py b/pc/src/openmuscle/web/state.py index fb251a5..1794ac5 100644 --- a/pc/src/openmuscle/web/state.py +++ b/pc/src/openmuscle/web/state.py @@ -56,6 +56,7 @@ def _quest_label_column(joint_index: int, channel_index: int) -> int: return joint_index * len(QUEST_JOINT_CHANNEL_ORDER) + channel_index from openmuscle.receiver.matcher import TemporalMatcher from openmuscle.receiver.udp_listener import UDPListener +from openmuscle.discovery import DiscoveryManager from openmuscle.web.inference import InferenceEngine from openmuscle.web.log_buffer import LogBuffer, install as install_log_handler @@ -220,11 +221,15 @@ class AppState: def __init__(self, udp_port: int = 3141, captures_dir: Path | None = None, model_path: Optional[str] = None, - hand_target: Optional[tuple] = None): + hand_target: Optional[tuple] = None, + enable_discovery: bool = True): """ Args: udp_port: incoming device telemetry port (FlexGrid + LASK5) captures_dir: where recorded CSVs go + enable_discovery: run native V4 discovery + auto-subscribe (replaces + the interim bridge_subscriber.py). Set False to bind the + UDP port only and rely on an external subscriber/bridge. model_path: optional path to a trained model .pkl. When set, every FlexGrid packet is run through the model and its predictions populate the WS snapshot's `inference` @@ -246,7 +251,17 @@ def __init__(self, udp_port: int = 3141, captures_dir: Path | None = None, self.log_buffer.info("server", "AppState init udp_port={} captures_dir={}".format( udp_port, str(self.captures_dir))) - self.listener = UDPListener(port=udp_port) + # Native V4 discovery + subscribe. Discovers announcing sources, keeps a + # TCP subscription + 1 Hz heartbeat to each, and re-probes cached devices + # on startup (a device's beacon goes silent once any hub subscribes, so + # the cache is how we recover one another hub already holds). Replaces + # pc/bridge_subscriber.py. The listener hands announce beacons straight + # to it; subscribed devices' sensor frames flow through the normal queue. + self.discovery: Optional[DiscoveryManager] = ( + DiscoveryManager(udp_port=udp_port, auto_subscribe=True) + if enable_discovery else None) + announce_handler = self.discovery.on_announce if self.discovery else None + self.listener = UDPListener(port=udp_port, announce_handler=announce_handler) self.devices: dict[str, DeviceInfo] = {} self.recording: Optional[ActiveCapture] = None # At most one active session at a time. Recordings made while a @@ -288,10 +303,16 @@ def start(self): if self._started: return self.listener.start() + if self.discovery: + # Re-probe cached devices in the background so we recover sources + # whose beacon is silent (held by another hub) at startup. + self.discovery.recover_cache() self._started = True def stop(self): self.listener.stop() + if self.discovery: + self.discovery.stop() if self.recording: self.stop_recording() if self._hand_sock is not None: @@ -757,6 +778,11 @@ def _snapshot(self) -> dict: "recording": rec, "inference": self._inference_snapshot(), "active_session": self.active_session, + # Native V4 discovery: known sources + subscription state. Empty + # list when discovery is disabled. Devices that are subscribed and + # streaming also appear in `devices` above (via their frames); this + # adds the ones that are known-but-silent plus per-device sub status. + "discovery": self.discovery.snapshot() if self.discovery else [], } def _inference_snapshot(self) -> dict: From 86b5f13dbffef4a9d3b23f9cc2d7555ea010abab Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 23 Jun 2026 22:46:20 -0500 Subject: [PATCH 56/56] web: Discovered Sources panel in the Studio Live stage Adds a "Sources" panel to the devices rail showing every native-discovery source (announce beacon / cache / manual probe) with its subscribe state, address, and a per-source Subscribe/Unsubscribe toggle, plus an "add by IP" probe form. Driven by renderDiscovery() from the WS snapshot's `discovery` array; the toggle and form call the /api/discovery REST endpoints. Replaces running bridge_subscriber.py in a terminal. Verified in a browser preview against the live V4 board: a cold `openmuscle web` boot recovered flexgrid-d7af0b from the device cache, auto-subscribed, and the panel rendered "1/1 subscribed" with the device streaming at ~15 Hz. No console errors. Co-Authored-By: turfptax-claude O4.8 --- pc/src/openmuscle/web/static/app.js | 117 ++++++++++++++++++++++++ pc/src/openmuscle/web/static/index.html | 15 +++ pc/src/openmuscle/web/static/styles.css | 59 ++++++++++++ 3 files changed, 191 insertions(+) diff --git a/pc/src/openmuscle/web/static/app.js b/pc/src/openmuscle/web/static/app.js index 14062c6..90be312 100644 --- a/pc/src/openmuscle/web/static/app.js +++ b/pc/src/openmuscle/web/static/app.js @@ -98,6 +98,7 @@ function handleTick(msg) { } renderActiveSession(); renderDevices(); + renderDiscovery(msg.discovery || []); renderRecordPickers(); renderRecording(); const dev = selectedDevice(); @@ -165,6 +166,122 @@ function renderHandViewer(questDev, inference) { } } +// ---------- native V4 discovery (Sources rail) ---------- + +let _discoveryProbeWired = false; + +function renderDiscovery(discovery) { + const list = document.getElementById('discovery-list'); + const count = document.getElementById('discovery-count'); + if (!list) return; + if (!_discoveryProbeWired) wireDiscoveryProbe(); + + const subs = discovery.filter(d => d.subscribed).length; + if (count) count.textContent = discovery.length + ? `${subs}/${discovery.length} subscribed` : ''; + + if (!discovery.length) { + list.innerHTML = '
  • No V4 sources discovered yet…
    • '; + return; + } + list.innerHTML = discovery.map(d => { + // State badge: subscribed (green) / error (red) / known (grey). + let stateCls = 'known', stateTxt = 'known'; + if (d.subscribed) { stateCls = 'subscribed'; stateTxt = 'subscribed'; } + else if (d.sub_error) { stateCls = 'err'; stateTxt = 'error'; } + const btnTxt = d.subscribed ? 'Unsubscribe' : 'Subscribe'; + const btnAct = d.subscribed ? 'unsubscribe' : 'subscribe'; + const age = (d.age_s != null) ? `${d.age_s.toFixed(0)}s ago` : ''; + const errLine = d.sub_error + ? `
    ${escapeHtml(d.sub_error)}
    ` + : ''; + return ` +
  • +
    + ${escapeHtml(d.device_id)} + ${stateTxt} +
    +
    + ${escapeHtml(d.device_type)} + ${escapeHtml(d.ip)}:${d.cmd_port} + via ${escapeHtml(d.source)} + ${age} +
    + ${errLine} + +
    • `; + }).join(''); + + list.querySelectorAll('.src-btn').forEach(btn => { + btn.onclick = async (e) => { + e.stopPropagation(); + const id = btn.dataset.id; + const act = btn.dataset.act; + btn.disabled = true; + btn.textContent = act === 'subscribe' ? 'Subscribing…' : 'Unsubscribing…'; + try { + const res = await fetch(`/api/discovery/${act}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ device_id: id }), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + setProbeMsg(err.detail || `${act} failed`, true); + } + } catch (err) { + setProbeMsg(String(err), true); + } + // Next WS tick re-renders the true state; no manual refresh needed. + }; + }); +} + +function wireDiscoveryProbe() { + const form = document.getElementById('discovery-probe-form'); + const input = document.getElementById('discovery-probe-ip'); + if (!form || !input) return; + _discoveryProbeWired = true; + form.onsubmit = async (e) => { + e.preventDefault(); + const raw = input.value.trim(); + if (!raw) return; + // Accept "ip" or "ip:port". + let ip = raw, cmd_port = null; + if (raw.includes(':')) { + const parts = raw.split(':'); + ip = parts[0]; + const p = parseInt(parts[1], 10); + if (!isNaN(p)) cmd_port = p; + } + setProbeMsg(`probing ${ip}…`, false); + try { + const res = await fetch('/api/discovery/probe', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(cmd_port ? { ip, cmd_port } : { ip }), + }); + if (res.ok) { + const d = await res.json(); + setProbeMsg(`found ${d.device_id} (${d.device_type})`, false); + input.value = ''; + } else { + const err = await res.json().catch(() => ({})); + setProbeMsg(err.detail || `no V4 source at ${ip}`, true); + } + } catch (err) { + setProbeMsg(String(err), true); + } + }; +} + +function setProbeMsg(text, isError) { + const el = document.getElementById('discovery-probe-msg'); + if (!el) return; + el.textContent = text; + el.classList.toggle('err', !!isError); +} + // ---------- device list ---------- function selectedDevice() { diff --git a/pc/src/openmuscle/web/static/index.html b/pc/src/openmuscle/web/static/index.html index f0df00a..ed1ac14 100644 --- a/pc/src/openmuscle/web/static/index.html +++ b/pc/src/openmuscle/web/static/index.html @@ -59,6 +59,21 @@

    Devices

    • Waiting for a device to send a packet…
    + + +

    Sources

    +
      +
    • No V4 sources discovered yet…
      • +
    +
    + + +
    +
    diff --git a/pc/src/openmuscle/web/static/styles.css b/pc/src/openmuscle/web/static/styles.css index d833aeb..97eff40 100644 --- a/pc/src/openmuscle/web/static/styles.css +++ b/pc/src/openmuscle/web/static/styles.css @@ -1303,3 +1303,62 @@ footer { .hand-viewer-legend .hv-real { color: #34d399; } .hand-viewer-legend .hv-pred { color: #fbbf24; } .hand-viewer-legend .hv-hint { margin-left: auto; opacity: 0.7; font-style: italic; } + +/* ---- native V4 discovery: Sources rail ---- + Discovered sources (announce beacon / cache / manual probe) with a per-source + subscribe toggle. Rendered by app.js renderDiscovery() from the WS snapshot's + `discovery` array. */ +.rail-head-sources { margin-top: 16px; display: flex; align-items: baseline; gap: 8px; } +.rail-count { font-size: 11px; font-weight: 400; color: var(--fg-faint, #8b96a8); } + +.discovery-compact .src { + border: 1px solid var(--border, #23262d); + border-radius: 8px; + padding: 8px 10px; + margin-bottom: 6px; + background: var(--panel, #14171d); +} +.discovery-compact .src.subscribed { border-left: 3px solid #34d399; } +.discovery-compact .src.err { border-left: 3px solid #f87171; } +.discovery-compact .src.known { border-left: 3px solid #6b7280; } + +.src-top { display: flex; justify-content: space-between; align-items: center; gap: 8px; } +.src-id { font-weight: 600; font-size: 13px; word-break: break-all; } +.src-state { + font-size: 10px; text-transform: uppercase; letter-spacing: 0.04em; + padding: 1px 6px; border-radius: 999px; white-space: nowrap; +} +.src-state.subscribed { color: #34d399; background: rgba(52, 211, 153, 0.12); } +.src-state.err { color: #f87171; background: rgba(248, 113, 113, 0.12); } +.src-state.known { color: #9ca3af; background: rgba(156, 163, 175, 0.12); } + +.src-meta { + display: flex; flex-wrap: wrap; gap: 6px 10px; + font-size: 11px; color: var(--fg-faint, #8b96a8); margin-top: 4px; +} +.src-err { font-size: 11px; color: #f87171; margin-top: 4px; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + +.src-btn { + margin-top: 8px; width: 100%; padding: 5px 8px; + font-size: 12px; border-radius: 6px; cursor: pointer; + border: 1px solid var(--border, #23262d); + background: var(--panel-2, #1b1f27); color: var(--fg, #e6e9ef); +} +.src-btn:hover:not(:disabled) { background: var(--panel-3, #232834); } +.src-btn:disabled { opacity: 0.6; cursor: default; } +.src.subscribed .src-btn { border-color: rgba(248, 113, 113, 0.4); } + +.discovery-probe { display: flex; gap: 6px; margin-top: 8px; } +.discovery-probe input { + flex: 1; min-width: 0; padding: 5px 8px; font-size: 12px; + border-radius: 6px; border: 1px solid var(--border, #23262d); + background: var(--panel, #14171d); color: var(--fg, #e6e9ef); +} +.discovery-probe button { + width: 32px; border-radius: 6px; cursor: pointer; font-size: 16px; + border: 1px solid var(--border, #23262d); + background: var(--panel-2, #1b1f27); color: var(--fg, #e6e9ef); +} +.discovery-probe-msg { font-size: 11px; color: var(--fg-faint, #8b96a8); margin-top: 4px; min-height: 14px; } +.discovery-probe-msg.err { color: #f87171; }