Merge test/sab-seed-benchmark: optional app.sab read on connect

Preference 'Lazy loading on connect?' (on by default). Off reads the whole app
from app.sab when connecting: 8 s instead of 116 s on a P-ION, 0.9 s instead of
16 s on an Orion.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app_state.py

@@ -8,7 +8,7 @@ so later rebinding would only ever be seen by the module that did it.
 import configparser
 import os
 
-VERSION = "0.0.0.169"
+VERSION = "0.0.0.172"
 PROPERTIES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "editor.properties")
 DEFAULT_GRID_SIZE = 10
 DEFAULT_ADD_TIMESTAMP = True
@@ -118,6 +118,7 @@ def load_preferences():
     left_pane_width = DEFAULT_LEFT_PANE_WIDTH
     sedona_home = ""
     trace_dasp = False
+    lazy_loading = True
 
     if os.path.exists(PROPERTIES_FILE):
         try:
@@ -130,6 +131,7 @@ def load_preferences():
             )
             sedona_home = config.get("Editor", "sedona.home", fallback="").strip()
             trace_dasp = config.getboolean("Editor", "trace_dasp", fallback=False)
+            lazy_loading = config.getboolean("Editor", "lazy_loading", fallback=True)
         except Exception:
             pass
     else:
@@ -150,7 +152,8 @@ def load_preferences():
     left_pane_width = max(
         MIN_ALLOWED_LEFT_PANE_WIDTH, min(MAX_ALLOWED_LEFT_PANE_WIDTH, left_pane_width)
     )
-    return sedona_home, grid_size, add_timestamp, box_width, left_pane_width, trace_dasp
+    return (sedona_home, grid_size, add_timestamp, box_width, left_pane_width, trace_dasp,
+            lazy_loading)
 
 
 def save_preferences():
@@ -170,6 +173,7 @@ def save_preferences():
         "box_width": str(BOX_WIDTH),
         "left_pane_width": str(LEFT_PANE_WIDTH),
         "trace_dasp": str(TRACE_DASP),
+        "lazy_loading": str(LAZY_LOADING),
     }
     with open(PROPERTIES_FILE, "w") as configfile:
         config.write(configfile)
@@ -259,5 +263,8 @@ def base_window_title_live():
     return f"Sedona Live Editor [v{VERSION}] — {device_host}:{device_port}"
 
 
-SEDONA_HOME, GRID_SIZE, ADD_TIMESTAMP, BOX_WIDTH, LEFT_PANE_WIDTH, TRACE_DASP = load_preferences()
+# LAZY_LOADING: on connect, read components as folders are opened (True) or the whole
+# app from app.sab first (False). Read at each connect, so a change takes effect on the next.
+(SEDONA_HOME, GRID_SIZE, ADD_TIMESTAMP, BOX_WIDTH, LEFT_PANE_WIDTH, TRACE_DASP,
+ LAZY_LOADING) = load_preferences()
 LAST_PALETTE_KIT = load_last_palette_kit()

bench_sab_seed.py

@@ -0,0 +1,162 @@
+"""Benchmark: lazy Sox loading vs seeding the cache from app.sab.
+
+    python bench_sab_seed.py HOST [--port 1876] [--user admin] [--password PW]
+
+Each method runs on a fresh connection, so it starts from an empty cache.
+
+  lazy-seq    every component's tree, one request at a time (as used_kit_names does)
+  lazy-tree   the tree folder by folder, each folder's children in one window
+  lazy-full   lazy-tree plus config and links of every component: what app.sab holds
+  lazy-full-seq  lazy-seq plus config and links, one request at a time
+  sab         App.save, get app.sab, decode it
+"""
+
+import argparse
+import time
+
+import app_state  # noqa: F401  loads Sedona home
+import schema
+from device_session import DeviceSession
+from sox import CONFIG
+import app_backup
+import sab_app
+
+
+class Counter:
+    """Counts Sox requests sent through the client."""
+
+    def __init__(self, client):
+        self.n = 0
+        orig_requests = client.requests
+
+        def requests(reqs, *a, **k):
+            self.n += len(reqs)
+            return orig_requests(reqs, *a, **k)
+
+        client.requests = requests
+
+
+def connect(args):
+    ds = DeviceSession()
+    # DeviceSession.connect also loads the root's children and their values; time
+    # only the bare session here, so every method starts from the same place.
+    from sox import SoxClient
+    ds.sox_client = SoxClient(args.host, args.port, args.user, args.password)
+    ds.sox_client.checksum_resolver = schema.resolve_truncated_checksum
+    ds.sox_client.connect()
+    parts = ds.sox_client.read_schema()
+    ds.sox_client.read_version()
+    ds._load_slot_registry(parts)
+    return ds
+
+
+def lazy_seq(ds):
+    client = ds.sox_client
+    pending, n = [0], 0
+    while pending:
+        comp = client.load_ids([pending.pop()], checked=False)[0]
+        if comp is not None:
+            n += 1
+            pending.extend(comp.children)
+    return n
+
+
+def lazy_tree(ds):
+    client = ds.sox_client
+    client.load(0)
+    pending, comps = [0], []
+    while pending:
+        cid = pending.pop()
+        comps.append(client.cache_get(cid))
+        for child in ds.load_children(cid):
+            if child.children:
+                pending.append(child.id)
+            else:
+                comps.append(child)
+    return [c for c in comps if c is not None]
+
+
+def lazy_full(ds):
+    client = ds.sox_client
+    comps = lazy_tree(ds)
+    # In folder-sized windows, as open_wiresheet sends them.
+    for i in range(0, len(comps), 16):
+        client.update(comps[i:i + 16], CONFIG)
+    for c in comps:
+        c.links = client.read_links(c.id)
+    return len(comps)
+
+
+def lazy_full_seq(ds):
+    """lazy-seq plus config and links, every request alone: a P-ION's pace."""
+    client = ds.sox_client
+    pending, n = [0], 0
+    while pending:
+        comp = client.load_ids([pending.pop()], checked=False)[0]
+        if comp is None:
+            continue
+        n += 1
+        pending.extend(comp.children)
+        client.update([comp], CONFIG)
+        comp.links = client.read_links(comp.id)
+    return n
+
+
+def sab(ds, timing):
+    client = ds.sox_client
+    t = time.perf_counter()
+    app_backup.save_app(client)
+    timing["save"] = time.perf_counter() - t
+    t = time.perf_counter()
+    data = client.get_file(app_backup.APP_SAB)
+    timing["get"] = time.perf_counter() - t
+    timing["bytes"] = len(data)
+    t = time.perf_counter()
+    app = sab_app.SabApp(data)
+    timing["decode"] = time.perf_counter() - t
+    timing["links"] = len(app.links)
+    return len(app.comps)
+
+
+def run(name, args, fn):
+    try:
+        ds = connect(args)
+    except Exception as e:
+        print(f"{name:10s}  connect failed: {e}")
+        return
+    counter = Counter(ds.sox_client)
+    timing = {}
+    t = time.perf_counter()
+    try:
+        result = fn(ds, timing) if name == "sab" else fn(ds)
+        elapsed = time.perf_counter() - t
+        n = result if isinstance(result, int) else len(result)
+        extra = "  ".join(f"{k}={v:.2f}s" if isinstance(v, float) else f"{k}={v}"
+                          for k, v in timing.items())
+        print(f"{name:10s}  {elapsed:7.2f}s  comps={n:4d}  requests={counter.n:5d}  {extra}")
+    except Exception as e:
+        print(f"{name:10s}  FAILED after {time.perf_counter() - t:.2f}s "
+              f"({counter.n} requests): {e}")
+    finally:
+        ds.disconnect()
+        time.sleep(1.0)
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("host")
+    p.add_argument("--port", type=int, default=1876)
+    p.add_argument("--user", default="admin")
+    p.add_argument("--password", default="")
+    p.add_argument("--only", nargs="*",
+                   default=["lazy-seq", "lazy-tree", "lazy-full", "sab"])
+    args = p.parse_args()
+    fns = {"lazy-seq": lazy_seq, "lazy-tree": lazy_tree,
+           "lazy-full": lazy_full, "lazy-full-seq": lazy_full_seq, "sab": sab}
+    print(f"== {args.host}:{args.port}")
+    for name in args.only:
+        run(name, args, fns[name])
+
+
+if __name__ == "__main__":
+    main()

dasp.py

@@ -98,7 +98,43 @@ def _trace(direction, msg, datagram=None):
     hexdump = ""
     if datagram:
         hexdump = " [" + datagram.hex(" ") + "]"
-    _trace_out(f"[DASP] {ts} {direction} {name}{seq}{ack} payload={plen}{hexdump}")
+    line = f"[DASP] {ts} {direction} {name}{seq}{ack} payload={plen}{hexdump}"
+    color = _TRACE_COLORS.get(direction)
+    if color and _console_colors():
+        line = f"{color}{line}\033[0m"
+    _trace_out(line)
+
+
+# Sent lines yellow, received lines green, so the two directions read apart at a glance.
+_TRACE_COLORS = {"S": "\033[93m", "R": "\033[92m"}
+_colors_ok = None
+
+
+def _console_colors():
+    """True when stdout is a console that understands ANSI colour codes.
+
+    Windows consoles need virtual-terminal processing switched on first; a pipe or
+    file gets plain text so captured logs stay free of escape codes.
+    """
+    global _colors_ok
+    if _colors_ok is None:
+        import sys
+        stream = sys.__stdout__
+        try:
+            _colors_ok = stream is not None and stream.isatty()
+        except (OSError, ValueError):
+            _colors_ok = False
+        if _colors_ok and os.name == "nt":
+            try:
+                import ctypes
+                kernel32 = ctypes.windll.kernel32
+                handle = kernel32.GetStdHandle(-11)          # STD_OUTPUT_HANDLE
+                mode = ctypes.c_uint32()
+                _colors_ok = bool(kernel32.GetConsoleMode(handle, ctypes.byref(mode))
+                                  and kernel32.SetConsoleMode(handle, mode.value | 0x0004))
+            except (OSError, AttributeError):
+                _colors_ok = False
+    return _colors_ok
 
 
 # Trace lines are printed by a thread of their own. The send and receive threads must

device_session.py

@@ -31,6 +31,18 @@ def _slot_precision(type_name, slot_name):
     return 2
 
 
+def _as_sox_value(slot, val):
+    """An app.sab config value as a Sox config read leaves it in the cache."""
+    if "s" in slot.flags:
+        return val
+    if slot.type in ("sys::Buf", "buf"):
+        # Sox reads every Buf as a Str: its bytes but the last, as text.
+        return val[:-1].decode("utf-8", errors="replace") if val else ""
+    if slot.type == "bool":
+        return val is not False      # Sox reads a bool as u1 != 0, so null is true
+    return val
+
+
 class DeviceSession:
     """High-level session manager for a live Sedona device connection."""
 
@@ -38,6 +50,10 @@ class DeviceSession:
         self.sox_client = None
         self._event_queue = queue.Queue()
         self._current_subs = set()   # comp_ids we subscribed for the open wiresheet
+        # Components whose config and links came from app.sab and have not been read
+        # since: their first sheet open skips reading those again.
+        self._seeded = set()
+        self.seed_info = ""          # how the cache was filled, for the status bar
 
     # ── Connect / Disconnect ────────────────────────────────────
 
@@ -53,11 +69,21 @@ class DeviceSession:
 
         self._load_slot_registry(parts)
 
-        root_comp = self.sox_client.load(0)
+        self.sox_client.load(0)
+        if app_state.LAZY_LOADING:
+            self.seed_info = "lazy loading"
+        else:
+            try:
+                count = self.seed_from_sab()
+                self.seed_info = f"app read from app.sab: {count} components"
+            except Exception as e:
+                self.seed_info = f"app.sab not used: {e}"
+
+        root_comp = self.sox_client.cache_get(0)
         if root_comp and root_comp.children:
             children = self.load_children(root_comp.id)
             if children:
-                self.sox_client.update(children, CONFIG | RUNTIME)
+                self.sox_client.update(children, self._values_to_read(children))
         return root_comp
 
     def disconnect(self, unsubscribe=True):
@@ -161,6 +187,74 @@ class DeviceSession:
                 if c is not None and c.kit_id == kit_id and c.type_id == type_id_val:
                     c.type_name = full_name
 
+    # ── Seeding from app.sab ────────────────────────────────────
+
+    def seed_from_sab(self):
+        """Fills the cache with the whole app — tree, config and links — from app.sab.
+
+        Reading component by component costs a round trip each, and a device answers one
+        request per pass of its loop: 230 components took 116 s on a P-ION, where saving
+        the app and reading app.sab took 8 s. The app is saved first, so app.sab holds
+        what is running. From here on Sox keeps the cache current, as it does for a
+        lazily loaded one; runtime values are still read when a sheet opens.
+
+        Nothing goes into the cache unless the whole file decodes and its tree matches the
+        device's, so on any failure the caller falls back to lazy loading. Returns the
+        number of components.
+        """
+        import app_backup
+        import sab_app
+        from sox import Link
+
+        client = self.sox_client
+        root = client.cache_get(0) or client.load(0)
+        app_backup.save_app(client)
+        app = sab_app.SabApp(client.get_file(app_backup.APP_SAB))
+
+        # app.sab numbers kits by the app's schema and Sox by kits.scode's; the two can
+        # differ, so types are matched by name, and a kit only when its checksum agrees.
+        device_kits = {p.name: f"{p.checksum & 0xFFFFFFFF:08x}" for p in client.schema_parts}
+        for name, checksum in app.kit_parts:
+            if device_kits.get(name) != checksum:
+                raise ValueError(f"kit {name} in app.sab is not the device's")
+        type_key = {qname: key for key, qname in client.type_name_map.items()}
+
+        if [k["id"] for k in app.app["kids"]] != root.children:
+            raise ValueError("app.sab does not match the running app")
+
+        comps = {}
+        for c in app.comps.values():
+            key = type_key.get(c["type"])
+            if key is None:
+                raise ValueError(f"no manifest for {c['type']}")
+            comp = SoxComponent(c["id"], key[0], key[1], c["type"])
+            comp.name = c["name"]
+            comp.parent = c["parentId"]
+            comp.children = [k["id"] for k in c["kids"]]
+            comp.permissions = root.permissions
+            for slot, val in c["props"]:
+                comp.values[slot.name] = _as_sox_value(slot, val)
+            comps[comp.id] = comp
+        comps[0].subscription = root.subscription
+
+        # Sox reads a component's links both ways, into it and out of it.
+        for from_c, from_slot, to_c, to_slot in app.links:
+            link = Link(from_c["id"], from_slot, to_c["id"], to_slot)
+            comps[from_c["id"]].links.append(link)
+            if to_c["id"] != from_c["id"]:
+                comps[to_c["id"]].links.append(link)
+
+        for comp in comps.values():
+            client.cache_add(comp)
+        self._seeded = set(comps)
+        return len(comps)
+
+    def _values_to_read(self, comps):
+        """What to read for `comps` on a sheet open: config too, unless app.sab gave it."""
+        if all(c.id in self._seeded for c in comps):
+            return RUNTIME
+        return CONFIG | RUNTIME
+
     # ── Lazy Loading ────────────────────────────────────────────
 
     def load_children(self, comp_id):
@@ -253,9 +347,9 @@ class DeviceSession:
 
         all_link_remote_ids = set()
         for child in children:
-            links = self.sox_client.read_links(child.id)
-            child.links = links
-            for lnk in links:
+            if child.id not in self._seeded:
+                child.links = self.sox_client.read_links(child.id)
+            for lnk in child.links:
                 all_link_remote_ids.add(lnk.from_comp_id)
                 all_link_remote_ids.add(lnk.to_comp_id)
 
@@ -267,7 +361,14 @@ class DeviceSession:
         if remote_ids:
             self.sox_client.load_ids(remote_ids, checked=False)
 
-        self.sox_client.update(children, CONFIG | RUNTIME)
+        seeded = [c for c in children if c.id in self._seeded]
+        others = [c for c in children if c.id not in self._seeded]
+        if seeded:
+            self.sox_client.update(seeded, RUNTIME)
+        if others:
+            self.sox_client.update(others, CONFIG | RUNTIME)
+        # Subscribed from here on, so Sox keeps them current; a later open reads again.
+        self._seeded -= child_ids
 
         folder_comp = self.sox_client.cache_get(folder_comp_id)
         to_subscribe = [folder_comp] + children if folder_comp else children

editor.properties

@@ -7,7 +7,7 @@ left_pane_width = 230
 trace_dasp = True
 
 [Device]
-host = 192.168.1.251
+host = localhost
 port = 1876
 user = admin
 

main.py

@@ -2699,6 +2699,18 @@ def open_preferences_dialog():
     )
     chk_trace_dasp.grid(row=5, column=1, sticky="w", pady=(0, 15))
 
+    lbl_lazy = ttk.Label(frame_padded, text="Lazy loading on connect?", anchor="w")
+    lbl_lazy.grid(row=6, column=0, sticky="w", pady=(0, 15), padx=(0, 10))
+
+    # Off: the whole app is read from app.sab when connecting. Takes effect on the next.
+    var_lazy_loading = tk.BooleanVar(value=app_state.LAZY_LOADING)
+    chk_lazy_loading = ttk.Checkbutton(
+        frame_padded,
+        text="Yes",
+        variable=var_lazy_loading
+    )
+    chk_lazy_loading.grid(row=6, column=1, sticky="w", pady=(0, 15))
+
     def apply_preferences():
         try:
             val = int(entry_grid.get().strip())
@@ -2755,6 +2767,7 @@ def open_preferences_dialog():
             app_state.GRID_SIZE = val
             app_state.ADD_TIMESTAMP = var_timestamp.get()
             app_state.TRACE_DASP = var_trace_dasp.get()
+            app_state.LAZY_LOADING = var_lazy_loading.get()
             app_state.SEDONA_HOME = entry_home.get().strip()
 
             app_state.save_preferences()
@@ -2773,7 +2786,7 @@ def open_preferences_dialog():
             messagebox.showerror("Invalid Input", message, parent=dialog)
 
     btn_save_pref = ttk.Button(frame_padded, text="Apply Changes", command=apply_preferences)
-    btn_save_pref.grid(row=6, column=1, sticky="e")
+    btn_save_pref.grid(row=7, column=1, sticky="e")
 
     if not app_state.SEDONA_HOME:
         entry_home.focus_set()
@@ -2953,7 +2966,8 @@ def _on_connected(ds, root_comp):
     menu_device.entryconfig("Download to SAX...", state="normal")
     menu_app_info.entryconfig("Get with Files...", state="normal")
     menu_app_info.entryconfig("Put with Files...", state="normal")
-    app_state.set_status(f"Connected to {app_state.device_host}:{app_state.device_port}")
+    app_state.set_status(f"Connected to {app_state.device_host}:{app_state.device_port}"
+                         f" ({ds.seed_info})")
 
     sax_root = ds.build_dom_tree(root_comp)
     app_state.xml_root_element = sax_root
@@ -3097,6 +3111,14 @@ def _live_tree_expand(comp_id, tree_item_id, select=True, on_loaded=None):
     if ds is None or not ds.is_connected:
         return
 
+    # Every child already in the tree (app.sab gave the whole app, or it was opened
+    # before): nothing to load, and rebuilding the DOM would only cost the undo steps.
+    elem = app_state.tree_element_map.get(tree_item_id)
+    if elem is not None and all(c.get("name") != "..." for c in elem.findall("comp")):
+        if on_loaded is not None:
+            on_loaded(tree_item_id)
+        return
+
     app_state.set_status("Loading children...")
 
     def _bg():