From cb0196caa760b123811c2ec76b8dff3a02b9a492 Mon Sep 17 00:00:00 2001 From: Arda Aydin Date: Sun, 11 Oct 2026 14:01:32 +0300 Subject: [PATCH 1/3] Connect: read the whole app from app.sab instead of component by component Saves the app on the device, reads app.sab and fills the Sox cache with the tree, config and links in one go: 8 s instead of 116 s for 230 components on a P-ION, 0.9 s instead of 16 s for 194 on an Orion. Sox keeps the cache current from there as before. On any failure, or when app.sab does not match the running tree, it falls back to lazy loading. A sheet's first open reads only runtime values; a tree expand with nothing to load no longer rebuilds the DOM. bench_sab_seed.py times both ways against a device. Co-Authored-By: Claude Opus 5.5 --- app_state.py | 2 +- bench_sab_seed.py | 162 ++++++++++++++++++++++++++++++++++++++++++++++ device_session.py | 110 +++++++++++++++++++++++++++++-- main.py | 11 +++- 4 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 bench_sab_seed.py diff --git a/app_state.py b/app_state.py index 716d7ed..a72c646 100644 --- a/app_state.py +++ b/app_state.py @@ -8,7 +8,7 @@ so later rebinding would only ever be seen by the module that did it. import configparser import os -VERSION = "0.0.0.169" +VERSION = "0.0.0.170" PROPERTIES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "editor.properties") DEFAULT_GRID_SIZE = 10 DEFAULT_ADD_TIMESTAMP = True diff --git a/bench_sab_seed.py b/bench_sab_seed.py new file mode 100644 index 0000000..4bdb1e7 --- /dev/null +++ b/bench_sab_seed.py @@ -0,0 +1,162 @@ +"""Benchmark: lazy Sox loading vs seeding the cache from app.sab. + + python bench_sab_seed.py HOST [--port 1876] [--user admin] [--password PW] + +Each method runs on a fresh connection, so it starts from an empty cache. + + lazy-seq every component's tree, one request at a time (as used_kit_names does) + lazy-tree the tree folder by folder, each folder's children in one window + lazy-full lazy-tree plus config and links of every component: what app.sab holds + lazy-full-seq lazy-seq plus config and links, one request at a time + sab App.save, get app.sab, decode it +""" + +import argparse +import time + +import app_state # noqa: F401 loads Sedona home +import schema +from device_session import DeviceSession +from sox import CONFIG +import app_backup +import sab_app + + +class Counter: + """Counts Sox requests sent through the client.""" + + def __init__(self, client): + self.n = 0 + orig_requests = client.requests + + def requests(reqs, *a, **k): + self.n += len(reqs) + return orig_requests(reqs, *a, **k) + + client.requests = requests + + +def connect(args): + ds = DeviceSession() + # DeviceSession.connect also loads the root's children and their values; time + # only the bare session here, so every method starts from the same place. + from sox import SoxClient + ds.sox_client = SoxClient(args.host, args.port, args.user, args.password) + ds.sox_client.checksum_resolver = schema.resolve_truncated_checksum + ds.sox_client.connect() + parts = ds.sox_client.read_schema() + ds.sox_client.read_version() + ds._load_slot_registry(parts) + return ds + + +def lazy_seq(ds): + client = ds.sox_client + pending, n = [0], 0 + while pending: + comp = client.load_ids([pending.pop()], checked=False)[0] + if comp is not None: + n += 1 + pending.extend(comp.children) + return n + + +def lazy_tree(ds): + client = ds.sox_client + client.load(0) + pending, comps = [0], [] + while pending: + cid = pending.pop() + comps.append(client.cache_get(cid)) + for child in ds.load_children(cid): + if child.children: + pending.append(child.id) + else: + comps.append(child) + return [c for c in comps if c is not None] + + +def lazy_full(ds): + client = ds.sox_client + comps = lazy_tree(ds) + # In folder-sized windows, as open_wiresheet sends them. + for i in range(0, len(comps), 16): + client.update(comps[i:i + 16], CONFIG) + for c in comps: + c.links = client.read_links(c.id) + return len(comps) + + +def lazy_full_seq(ds): + """lazy-seq plus config and links, every request alone: a P-ION's pace.""" + client = ds.sox_client + pending, n = [0], 0 + while pending: + comp = client.load_ids([pending.pop()], checked=False)[0] + if comp is None: + continue + n += 1 + pending.extend(comp.children) + client.update([comp], CONFIG) + comp.links = client.read_links(comp.id) + return n + + +def sab(ds, timing): + client = ds.sox_client + t = time.perf_counter() + app_backup.save_app(client) + timing["save"] = time.perf_counter() - t + t = time.perf_counter() + data = client.get_file(app_backup.APP_SAB) + timing["get"] = time.perf_counter() - t + timing["bytes"] = len(data) + t = time.perf_counter() + app = sab_app.SabApp(data) + timing["decode"] = time.perf_counter() - t + timing["links"] = len(app.links) + return len(app.comps) + + +def run(name, args, fn): + try: + ds = connect(args) + except Exception as e: + print(f"{name:10s} connect failed: {e}") + return + counter = Counter(ds.sox_client) + timing = {} + t = time.perf_counter() + try: + result = fn(ds, timing) if name == "sab" else fn(ds) + elapsed = time.perf_counter() - t + n = result if isinstance(result, int) else len(result) + extra = " ".join(f"{k}={v:.2f}s" if isinstance(v, float) else f"{k}={v}" + for k, v in timing.items()) + print(f"{name:10s} {elapsed:7.2f}s comps={n:4d} requests={counter.n:5d} {extra}") + except Exception as e: + print(f"{name:10s} FAILED after {time.perf_counter() - t:.2f}s " + f"({counter.n} requests): {e}") + finally: + ds.disconnect() + time.sleep(1.0) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("host") + p.add_argument("--port", type=int, default=1876) + p.add_argument("--user", default="admin") + p.add_argument("--password", default="") + p.add_argument("--only", nargs="*", + default=["lazy-seq", "lazy-tree", "lazy-full", "sab"]) + args = p.parse_args() + fns = {"lazy-seq": lazy_seq, "lazy-tree": lazy_tree, + "lazy-full": lazy_full, "lazy-full-seq": lazy_full_seq, "sab": sab} + print(f"== {args.host}:{args.port}") + for name in args.only: + run(name, args, fns[name]) + + +if __name__ == "__main__": + main() diff --git a/device_session.py b/device_session.py index e4e4996..ec5982e 100644 --- a/device_session.py +++ b/device_session.py @@ -31,6 +31,18 @@ def _slot_precision(type_name, slot_name): return 2 +def _as_sox_value(slot, val): + """An app.sab config value as a Sox config read leaves it in the cache.""" + if "s" in slot.flags: + return val + if slot.type in ("sys::Buf", "buf"): + # Sox reads every Buf as a Str: its bytes but the last, as text. + return val[:-1].decode("utf-8", errors="replace") if val else "" + if slot.type == "bool": + return val is not False # Sox reads a bool as u1 != 0, so null is true + return val + + class DeviceSession: """High-level session manager for a live Sedona device connection.""" @@ -38,6 +50,10 @@ class DeviceSession: self.sox_client = None self._event_queue = queue.Queue() self._current_subs = set() # comp_ids we subscribed for the open wiresheet + # Components whose config and links came from app.sab and have not been read + # since: their first sheet open skips reading those again. + self._seeded = set() + self.seed_info = "" # how the cache was filled, for the status bar # ── Connect / Disconnect ──────────────────────────────────── @@ -53,11 +69,18 @@ class DeviceSession: self._load_slot_registry(parts) - root_comp = self.sox_client.load(0) + self.sox_client.load(0) + try: + count = self.seed_from_sab() + self.seed_info = f"app read from app.sab: {count} components" + except Exception as e: + self.seed_info = f"app.sab not used: {e}" + + root_comp = self.sox_client.cache_get(0) if root_comp and root_comp.children: children = self.load_children(root_comp.id) if children: - self.sox_client.update(children, CONFIG | RUNTIME) + self.sox_client.update(children, self._values_to_read(children)) return root_comp def disconnect(self, unsubscribe=True): @@ -161,6 +184,74 @@ class DeviceSession: if c is not None and c.kit_id == kit_id and c.type_id == type_id_val: c.type_name = full_name + # ── Seeding from app.sab ──────────────────────────────────── + + def seed_from_sab(self): + """Fills the cache with the whole app — tree, config and links — from app.sab. + + Reading component by component costs a round trip each, and a device answers one + request per pass of its loop: 230 components took 116 s on a P-ION, where saving + the app and reading app.sab took 8 s. The app is saved first, so app.sab holds + what is running. From here on Sox keeps the cache current, as it does for a + lazily loaded one; runtime values are still read when a sheet opens. + + Nothing goes into the cache unless the whole file decodes and its tree matches the + device's, so on any failure the caller falls back to lazy loading. Returns the + number of components. + """ + import app_backup + import sab_app + from sox import Link + + client = self.sox_client + root = client.cache_get(0) or client.load(0) + app_backup.save_app(client) + app = sab_app.SabApp(client.get_file(app_backup.APP_SAB)) + + # app.sab numbers kits by the app's schema and Sox by kits.scode's; the two can + # differ, so types are matched by name, and a kit only when its checksum agrees. + device_kits = {p.name: f"{p.checksum & 0xFFFFFFFF:08x}" for p in client.schema_parts} + for name, checksum in app.kit_parts: + if device_kits.get(name) != checksum: + raise ValueError(f"kit {name} in app.sab is not the device's") + type_key = {qname: key for key, qname in client.type_name_map.items()} + + if [k["id"] for k in app.app["kids"]] != root.children: + raise ValueError("app.sab does not match the running app") + + comps = {} + for c in app.comps.values(): + key = type_key.get(c["type"]) + if key is None: + raise ValueError(f"no manifest for {c['type']}") + comp = SoxComponent(c["id"], key[0], key[1], c["type"]) + comp.name = c["name"] + comp.parent = c["parentId"] + comp.children = [k["id"] for k in c["kids"]] + comp.permissions = root.permissions + for slot, val in c["props"]: + comp.values[slot.name] = _as_sox_value(slot, val) + comps[comp.id] = comp + comps[0].subscription = root.subscription + + # Sox reads a component's links both ways, into it and out of it. + for from_c, from_slot, to_c, to_slot in app.links: + link = Link(from_c["id"], from_slot, to_c["id"], to_slot) + comps[from_c["id"]].links.append(link) + if to_c["id"] != from_c["id"]: + comps[to_c["id"]].links.append(link) + + for comp in comps.values(): + client.cache_add(comp) + self._seeded = set(comps) + return len(comps) + + def _values_to_read(self, comps): + """What to read for `comps` on a sheet open: config too, unless app.sab gave it.""" + if all(c.id in self._seeded for c in comps): + return RUNTIME + return CONFIG | RUNTIME + # ── Lazy Loading ──────────────────────────────────────────── def load_children(self, comp_id): @@ -253,9 +344,9 @@ class DeviceSession: all_link_remote_ids = set() for child in children: - links = self.sox_client.read_links(child.id) - child.links = links - for lnk in links: + if child.id not in self._seeded: + child.links = self.sox_client.read_links(child.id) + for lnk in child.links: all_link_remote_ids.add(lnk.from_comp_id) all_link_remote_ids.add(lnk.to_comp_id) @@ -267,7 +358,14 @@ class DeviceSession: if remote_ids: self.sox_client.load_ids(remote_ids, checked=False) - self.sox_client.update(children, CONFIG | RUNTIME) + seeded = [c for c in children if c.id in self._seeded] + others = [c for c in children if c.id not in self._seeded] + if seeded: + self.sox_client.update(seeded, RUNTIME) + if others: + self.sox_client.update(others, CONFIG | RUNTIME) + # Subscribed from here on, so Sox keeps them current; a later open reads again. + self._seeded -= child_ids folder_comp = self.sox_client.cache_get(folder_comp_id) to_subscribe = [folder_comp] + children if folder_comp else children diff --git a/main.py b/main.py index 7fd3540..c06e7d8 100644 --- a/main.py +++ b/main.py @@ -2953,7 +2953,8 @@ def _on_connected(ds, root_comp): menu_device.entryconfig("Download to SAX...", state="normal") menu_app_info.entryconfig("Get with Files...", state="normal") menu_app_info.entryconfig("Put with Files...", state="normal") - app_state.set_status(f"Connected to {app_state.device_host}:{app_state.device_port}") + app_state.set_status(f"Connected to {app_state.device_host}:{app_state.device_port}" + f" ({ds.seed_info})") sax_root = ds.build_dom_tree(root_comp) app_state.xml_root_element = sax_root @@ -3097,6 +3098,14 @@ def _live_tree_expand(comp_id, tree_item_id, select=True, on_loaded=None): if ds is None or not ds.is_connected: return + # Every child already in the tree (app.sab gave the whole app, or it was opened + # before): nothing to load, and rebuilding the DOM would only cost the undo steps. + elem = app_state.tree_element_map.get(tree_item_id) + if elem is not None and all(c.get("name") != "..." for c in elem.findall("comp")): + if on_loaded is not None: + on_loaded(tree_item_id) + return + app_state.set_status("Loading children...") def _bg(): From 7ff24797365f0783640030f504dbaf62a4355a65 Mon Sep 17 00:00:00 2001 From: Arda Aydin Date: Sun, 11 Oct 2026 14:13:41 +0300 Subject: [PATCH 2/3] Preferences: lazy loading on connect, on by default Off reads the whole app from app.sab when connecting. Read at each connect, so a change takes effect on the next one. Co-Authored-By: Claude Opus 5.5 --- app_state.py | 13 ++++++++++--- device_session.py | 13 ++++++++----- main.py | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/app_state.py b/app_state.py index a72c646..b1efef1 100644 --- a/app_state.py +++ b/app_state.py @@ -8,7 +8,7 @@ so later rebinding would only ever be seen by the module that did it. import configparser import os -VERSION = "0.0.0.170" +VERSION = "0.0.0.171" PROPERTIES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "editor.properties") DEFAULT_GRID_SIZE = 10 DEFAULT_ADD_TIMESTAMP = True @@ -118,6 +118,7 @@ def load_preferences(): left_pane_width = DEFAULT_LEFT_PANE_WIDTH sedona_home = "" trace_dasp = False + lazy_loading = True if os.path.exists(PROPERTIES_FILE): try: @@ -130,6 +131,7 @@ def load_preferences(): ) sedona_home = config.get("Editor", "sedona.home", fallback="").strip() trace_dasp = config.getboolean("Editor", "trace_dasp", fallback=False) + lazy_loading = config.getboolean("Editor", "lazy_loading", fallback=True) except Exception: pass else: @@ -150,7 +152,8 @@ def load_preferences(): left_pane_width = max( MIN_ALLOWED_LEFT_PANE_WIDTH, min(MAX_ALLOWED_LEFT_PANE_WIDTH, left_pane_width) ) - return sedona_home, grid_size, add_timestamp, box_width, left_pane_width, trace_dasp + return (sedona_home, grid_size, add_timestamp, box_width, left_pane_width, trace_dasp, + lazy_loading) def save_preferences(): @@ -170,6 +173,7 @@ def save_preferences(): "box_width": str(BOX_WIDTH), "left_pane_width": str(LEFT_PANE_WIDTH), "trace_dasp": str(TRACE_DASP), + "lazy_loading": str(LAZY_LOADING), } with open(PROPERTIES_FILE, "w") as configfile: config.write(configfile) @@ -259,5 +263,8 @@ def base_window_title_live(): return f"Sedona Live Editor [v{VERSION}] — {device_host}:{device_port}" -SEDONA_HOME, GRID_SIZE, ADD_TIMESTAMP, BOX_WIDTH, LEFT_PANE_WIDTH, TRACE_DASP = load_preferences() +# LAZY_LOADING: on connect, read components as folders are opened (True) or the whole +# app from app.sab first (False). Read at each connect, so a change takes effect on the next. +(SEDONA_HOME, GRID_SIZE, ADD_TIMESTAMP, BOX_WIDTH, LEFT_PANE_WIDTH, TRACE_DASP, + LAZY_LOADING) = load_preferences() LAST_PALETTE_KIT = load_last_palette_kit() diff --git a/device_session.py b/device_session.py index ec5982e..8065c00 100644 --- a/device_session.py +++ b/device_session.py @@ -70,11 +70,14 @@ class DeviceSession: self._load_slot_registry(parts) self.sox_client.load(0) - try: - count = self.seed_from_sab() - self.seed_info = f"app read from app.sab: {count} components" - except Exception as e: - self.seed_info = f"app.sab not used: {e}" + if app_state.LAZY_LOADING: + self.seed_info = "lazy loading" + else: + try: + count = self.seed_from_sab() + self.seed_info = f"app read from app.sab: {count} components" + except Exception as e: + self.seed_info = f"app.sab not used: {e}" root_comp = self.sox_client.cache_get(0) if root_comp and root_comp.children: diff --git a/main.py b/main.py index c06e7d8..76fc0f5 100644 --- a/main.py +++ b/main.py @@ -2699,6 +2699,18 @@ def open_preferences_dialog(): ) chk_trace_dasp.grid(row=5, column=1, sticky="w", pady=(0, 15)) + lbl_lazy = ttk.Label(frame_padded, text="Lazy loading on connect?", anchor="w") + lbl_lazy.grid(row=6, column=0, sticky="w", pady=(0, 15), padx=(0, 10)) + + # Off: the whole app is read from app.sab when connecting. Takes effect on the next. + var_lazy_loading = tk.BooleanVar(value=app_state.LAZY_LOADING) + chk_lazy_loading = ttk.Checkbutton( + frame_padded, + text="Yes", + variable=var_lazy_loading + ) + chk_lazy_loading.grid(row=6, column=1, sticky="w", pady=(0, 15)) + def apply_preferences(): try: val = int(entry_grid.get().strip()) @@ -2755,6 +2767,7 @@ def open_preferences_dialog(): app_state.GRID_SIZE = val app_state.ADD_TIMESTAMP = var_timestamp.get() app_state.TRACE_DASP = var_trace_dasp.get() + app_state.LAZY_LOADING = var_lazy_loading.get() app_state.SEDONA_HOME = entry_home.get().strip() app_state.save_preferences() @@ -2773,7 +2786,7 @@ def open_preferences_dialog(): messagebox.showerror("Invalid Input", message, parent=dialog) btn_save_pref = ttk.Button(frame_padded, text="Apply Changes", command=apply_preferences) - btn_save_pref.grid(row=6, column=1, sticky="e") + btn_save_pref.grid(row=7, column=1, sticky="e") if not app_state.SEDONA_HOME: entry_home.focus_set() From d905d251453eb3b13e3e5f796f48ed10d2b4dc7a Mon Sep 17 00:00:00 2001 From: Arda Aydin Date: Sun, 11 Oct 2026 14:14:41 +0300 Subject: [PATCH 3/3] DASP trace: sent lines yellow, received green on a console Plain text when stdout is a pipe or file. Also remembers localhost as the last device. Co-Authored-By: Claude Opus 5.5 --- app_state.py | 2 +- dasp.py | 38 +++++++++++++++++++++++++++++++++++++- editor.properties | 2 +- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app_state.py b/app_state.py index b1efef1..4636570 100644 --- a/app_state.py +++ b/app_state.py @@ -8,7 +8,7 @@ so later rebinding would only ever be seen by the module that did it. import configparser import os -VERSION = "0.0.0.171" +VERSION = "0.0.0.172" PROPERTIES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "editor.properties") DEFAULT_GRID_SIZE = 10 DEFAULT_ADD_TIMESTAMP = True diff --git a/dasp.py b/dasp.py index 4f62ec9..f2f0098 100644 --- a/dasp.py +++ b/dasp.py @@ -98,7 +98,43 @@ def _trace(direction, msg, datagram=None): hexdump = "" if datagram: hexdump = " [" + datagram.hex(" ") + "]" - _trace_out(f"[DASP] {ts} {direction} {name}{seq}{ack} payload={plen}{hexdump}") + line = f"[DASP] {ts} {direction} {name}{seq}{ack} payload={plen}{hexdump}" + color = _TRACE_COLORS.get(direction) + if color and _console_colors(): + line = f"{color}{line}\033[0m" + _trace_out(line) + + +# Sent lines yellow, received lines green, so the two directions read apart at a glance. +_TRACE_COLORS = {"S": "\033[93m", "R": "\033[92m"} +_colors_ok = None + + +def _console_colors(): + """True when stdout is a console that understands ANSI colour codes. + + Windows consoles need virtual-terminal processing switched on first; a pipe or + file gets plain text so captured logs stay free of escape codes. + """ + global _colors_ok + if _colors_ok is None: + import sys + stream = sys.__stdout__ + try: + _colors_ok = stream is not None and stream.isatty() + except (OSError, ValueError): + _colors_ok = False + if _colors_ok and os.name == "nt": + try: + import ctypes + kernel32 = ctypes.windll.kernel32 + handle = kernel32.GetStdHandle(-11) # STD_OUTPUT_HANDLE + mode = ctypes.c_uint32() + _colors_ok = bool(kernel32.GetConsoleMode(handle, ctypes.byref(mode)) + and kernel32.SetConsoleMode(handle, mode.value | 0x0004)) + except (OSError, AttributeError): + _colors_ok = False + return _colors_ok # Trace lines are printed by a thread of their own. The send and receive threads must diff --git a/editor.properties b/editor.properties index 693fc10..780d2ca 100644 --- a/editor.properties +++ b/editor.properties @@ -7,7 +7,7 @@ left_pane_width = 230 trace_dasp = True [Device] -host = 192.168.1.251 +host = localhost port = 1876 user = admin