test: operational_all: Harden to catch more

test/case/all.yaml

@@ -17,9 +17,6 @@
   infamy:
     specification: False
 
-- name: "Miscellaneous"
-  suite: misc/all.yaml
-
 
 - name: "System"
   suite: system/all.yaml
@@ -62,3 +59,7 @@
 
 - name: "Use Case Tests"
   suite: use_case/all.yaml
+
+- name: "Miscellaneous"
+  suite: misc/all.yaml
+

test/case/misc/operational_all/test.adoc

@@ -4,16 +4,27 @@ ifdef::topdoc[:imagesdir: {topdoc}../../test/case/misc/operational_all]
 
 ==== Description
 
-Basic test just to get operational from test-config without errors.
+Verify each device's operational datastore against its test-config.
+The running config is the source of truth, so the checks hold no matter
+how interfaces differ between devices:
 
-==== Topology
+  1. The operational interfaces are exactly the configured ones: none
+     missing, and none present that were never configured.
+  2. Features the test-config leaves disabled (NTP, containers, routing
+     protocols) emit no operational data.
 
-image::topology.svg[Get Operational topology, align=center, scaledwidth=75%]
+Both checks use specific-path GETs, which behave the same on NETCONF and
+RESTCONF.  A full-datastore GET is not portable: RESTCONF does not serve
+the operational datastore root, and NETCONF's operational provider errors
+when asked for an empty subtree.
+
+The test has no logical topology; it reads state from whatever DUTs the
+physical topology provides.
 
 ==== Sequence
 
-. Set up topology and attach to target DUT
-. Copy test-config to running configuration
-. Get all Operational data from 'target', verify there are no errors
+. Attach to all DUTs in the topology
+. Verify operational interfaces match the test-config
+. Verify unconfigured feature subtrees are absent
 
 

test/case/misc/operational_all/test.py

@@ -1,23 +1,112 @@
 #!/usr/bin/env python3
-
-# Test that it is possible to get all operational data
 """
 Get operational
 
-Basic test just to get operational from test-config without errors.
+Verify each device's operational datastore against its test-config.
+The running config is the source of truth, so the checks hold no matter
+how interfaces differ between devices:
+
+  1. The operational interfaces are exactly the configured ones: none
+     missing, and none present that were never configured.
+  2. Features the test-config leaves disabled (NTP, containers, routing
+     protocols) emit no operational data.
+
+Both checks use specific-path GETs, which behave the same on NETCONF and
+RESTCONF.  A full-datastore GET is not portable: RESTCONF does not serve
+the operational datastore root, and NETCONF's operational provider errors
+when asked for an empty subtree.
+
+The test has no logical topology; it reads state from whatever DUTs the
+physical topology provides.
 """
 import infamy
-import infamy.iface as iface
+from infamy.util import parallel, until
+
+# Feature subtrees that must be absent because the test-config does not
+# enable them.  /ietf-routing:routing itself is always present (its RIB
+# reflects the kernel's connected/local routes), so we target the
+# config-gated control-plane-protocols child rather than all of routing.
+ABSENT = [
+    "/ietf-ntp:ntp",
+    "/infix-containers:containers",
+    "/ietf-routing:routing/control-plane-protocols",
+]
+
+
+def configured_interfaces(dut):
+    cfg = dut.get_config_dict("/ietf-interfaces:interfaces")
+    return {i["name"] for i in cfg["interfaces"]["interface"]}
+
+
+def operational_interfaces(dut):
+    oper = dut.get_data("/ietf-interfaces:interfaces")["interfaces"]["interface"]
+    return {i["name"] for i in oper}
+
+
+def interfaces_match(name, dut, want):
+    """True once operational interfaces equal the configured set.
+
+    A preceding test (e.g. a container use_case) may leave veth endpoints
+    that are still being torn down when this test starts, so operational
+    momentarily carries interfaces that are not in the configuration.
+    Rather than asserting on that transient state, poll until the kernel,
+    and thus operational, has converged on the configured set.
+    """
+    have = operational_interfaces(dut)
+    if have != want:
+        print(f"{name}: operational {sorted(have)} != configured {sorted(want)}, waiting...")
+        return False
+    return True
+
+
+def absent(dut, xpath):
+    # RESTCONF returns nothing for an absent subtree; NETCONF's operational
+    # provider errors when asked for one.  Both mean "not present".
+    try:
+        return not dut.get_data(xpath)
+    except Exception:
+        return True
+
+
+def features_absent(name, dut):
+    """True once every unconfigured feature subtree is gone.
+
+    Like the interface set, these subtrees can momentarily linger: a
+    preceding test (containers, NTP, a routing protocol) leaves state that
+    yangerd only prunes once the underlying daemon/config is torn down.
+    Poll until operational has converged rather than asserting on the
+    transient overlap.
+    """
+    pending = [xpath for xpath in ABSENT if not absent(dut, xpath)]
+    if pending:
+        print(f"{name}: feature data still present {pending}, waiting...")
+        return False
+    return True
+
 
 with infamy.Test() as test:
-    with test.step("Set up topology and attach to target DUT"):
-        env = infamy.Env()
-        target = env.attach("target", "mgmt")
+    with test.step("Attach to all DUTs in the topology"):
+        env = infamy.Env(ltop=False)
+        infixen = env.ptop.get_infixen()
+        assert infixen, "no devices found in topology"
+        duts = dict(zip(infixen, parallel(*(lambda n=name: env.attach(n, "mgmt")
+                                            for name in infixen))))
+
+    with test.step("Verify operational interfaces match the test-config"):
+        def check(name, dut):
+            want = configured_interfaces(dut)
+            until(lambda: interfaces_match(name, dut, want), attempts=60)
+
+        parallel(*(lambda n=name, d=dut: check(n, d) for name, dut in duts.items()))
 
-    with test.step("Copy test-config to running configuration"):
-        pass
+    with test.step("Verify unconfigured feature subtrees are absent"):
+        def check_absent(name, dut):
+            # NTP is pruned by a periodic collector with a 60s poll
+            # interval, so a stale subtree can take a full cycle to clear;
+            # wait comfortably past one interval rather than racing it.
+            until(lambda: features_absent(name, dut), attempts=120)
 
-    with test.step("Get all Operational data from 'target', verify there are no errors"):
-        target.get_data(parse=False)
+        parallel(*(lambda n=name, d=dut: check_absent(n, d)
+                   for name, dut in duts.items()))
 
     test.succeed()

test/spec/generate_spec.py

@@ -140,7 +140,9 @@ def gen_spec(self, title, variables):
         if title is None:
             title = visitor.name
 
-        self.gen_topology()
+        has_topology = os.path.exists(self.topo_dot)
+        if has_topology:
+            self.gen_topology()
 
         with open(self.spec_path, "w", encoding='utf-8') as spec:
             # This is the test name/title  for the test-specification.pdf,
@@ -156,8 +158,9 @@ def gen_spec(self, title, variables):
             spec.write("==== Description\n\n")
             spec.write(description + "\n\n")
 
-            spec.write("==== Topology\n\n")
-            spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n")
+            if has_topology:
+                spec.write("==== Topology\n\n")
+                spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n")
 
             spec.write("==== Sequence\n\n")
             spec.writelines([f". {step}\n" for step in test_steps])