diff --git a/test/case/all.yaml b/test/case/all.yaml index 1fdbec3d5..79bc4cc65 100644 --- a/test/case/all.yaml +++ b/test/case/all.yaml @@ -17,9 +17,6 @@ infamy: specification: False -- name: "Miscellaneous" - suite: misc/all.yaml - - name: "System" suite: system/all.yaml @@ -62,3 +59,7 @@ - name: "Use Case Tests" suite: use_case/all.yaml + +- name: "Miscellaneous" + suite: misc/all.yaml + diff --git a/test/case/misc/operational_all/test.adoc b/test/case/misc/operational_all/test.adoc index a50ce7dfb..3cc19ae35 100644 --- a/test/case/misc/operational_all/test.adoc +++ b/test/case/misc/operational_all/test.adoc @@ -4,16 +4,27 @@ ifdef::topdoc[:imagesdir: {topdoc}../../test/case/misc/operational_all] ==== Description -Basic test just to get operational from test-config without errors. +Verify each device's operational datastore against its test-config. +The running config is the source of truth, so the checks hold no matter +how interfaces differ between devices: -==== Topology + 1. The operational interfaces are exactly the configured ones: none + missing, and none present that were never configured. + 2. Features the test-config leaves disabled (NTP, containers, routing + protocols) emit no operational data. -image::topology.svg[Get Operational topology, align=center, scaledwidth=75%] +Both checks use specific-path GETs, which behave the same on NETCONF and +RESTCONF. A full-datastore GET is not portable: RESTCONF does not serve +the operational datastore root, and NETCONF's operational provider errors +when asked for an empty subtree. + +The test has no logical topology; it reads state from whatever DUTs the +physical topology provides. ==== Sequence -. Set up topology and attach to target DUT -. Copy test-config to running configuration -. Get all Operational data from 'target', verify there are no errors +. Attach to all DUTs in the topology +. Verify operational interfaces match the test-config +. Verify unconfigured feature subtrees are absent diff --git a/test/case/misc/operational_all/test.py b/test/case/misc/operational_all/test.py index aec0f188c..d784a5ac9 100755 --- a/test/case/misc/operational_all/test.py +++ b/test/case/misc/operational_all/test.py @@ -1,23 +1,112 @@ #!/usr/bin/env python3 - -# Test that it is possible to get all operational data """ Get operational -Basic test just to get operational from test-config without errors. +Verify each device's operational datastore against its test-config. +The running config is the source of truth, so the checks hold no matter +how interfaces differ between devices: + + 1. The operational interfaces are exactly the configured ones: none + missing, and none present that were never configured. + 2. Features the test-config leaves disabled (NTP, containers, routing + protocols) emit no operational data. + +Both checks use specific-path GETs, which behave the same on NETCONF and +RESTCONF. A full-datastore GET is not portable: RESTCONF does not serve +the operational datastore root, and NETCONF's operational provider errors +when asked for an empty subtree. + +The test has no logical topology; it reads state from whatever DUTs the +physical topology provides. """ import infamy -import infamy.iface as iface +from infamy.util import parallel, until + +# Feature subtrees that must be absent because the test-config does not +# enable them. /ietf-routing:routing itself is always present (its RIB +# reflects the kernel's connected/local routes), so we target the +# config-gated control-plane-protocols child rather than all of routing. +ABSENT = [ + "/ietf-ntp:ntp", + "/infix-containers:containers", + "/ietf-routing:routing/control-plane-protocols", +] + + +def configured_interfaces(dut): + cfg = dut.get_config_dict("/ietf-interfaces:interfaces") + return {i["name"] for i in cfg["interfaces"]["interface"]} + + +def operational_interfaces(dut): + oper = dut.get_data("/ietf-interfaces:interfaces")["interfaces"]["interface"] + return {i["name"] for i in oper} + + +def interfaces_match(name, dut, want): + """True once operational interfaces equal the configured set. + + A preceding test (e.g. a container use_case) may leave veth endpoints + that are still being torn down when this test starts, so operational + momentarily carries interfaces that are not in the configuration. + Rather than asserting on that transient state, poll until the kernel, + and thus operational, has converged on the configured set. + """ + have = operational_interfaces(dut) + if have != want: + print(f"{name}: operational {sorted(have)} != configured {sorted(want)}, waiting...") + return False + return True + + +def absent(dut, xpath): + # RESTCONF returns nothing for an absent subtree; NETCONF's operational + # provider errors when asked for one. Both mean "not present". + try: + return not dut.get_data(xpath) + except Exception: + return True + + +def features_absent(name, dut): + """True once every unconfigured feature subtree is gone. + + Like the interface set, these subtrees can momentarily linger: a + preceding test (containers, NTP, a routing protocol) leaves state that + yangerd only prunes once the underlying daemon/config is torn down. + Poll until operational has converged rather than asserting on the + transient overlap. + """ + pending = [xpath for xpath in ABSENT if not absent(dut, xpath)] + if pending: + print(f"{name}: feature data still present {pending}, waiting...") + return False + return True + with infamy.Test() as test: - with test.step("Set up topology and attach to target DUT"): - env = infamy.Env() - target = env.attach("target", "mgmt") + with test.step("Attach to all DUTs in the topology"): + env = infamy.Env(ltop=False) + infixen = env.ptop.get_infixen() + assert infixen, "no devices found in topology" + duts = dict(zip(infixen, parallel(*(lambda n=name: env.attach(n, "mgmt") + for name in infixen)))) + + with test.step("Verify operational interfaces match the test-config"): + def check(name, dut): + want = configured_interfaces(dut) + until(lambda: interfaces_match(name, dut, want), attempts=60) + + parallel(*(lambda n=name, d=dut: check(n, d) for name, dut in duts.items())) - with test.step("Copy test-config to running configuration"): - pass + with test.step("Verify unconfigured feature subtrees are absent"): + def check_absent(name, dut): + # NTP is pruned by a periodic collector with a 60s poll + # interval, so a stale subtree can take a full cycle to clear; + # wait comfortably past one interval rather than racing it. + until(lambda: features_absent(name, dut), attempts=120) - with test.step("Get all Operational data from 'target', verify there are no errors"): - target.get_data(parse=False) + parallel(*(lambda n=name, d=dut: check_absent(n, d) + for name, dut in duts.items())) test.succeed() diff --git a/test/case/misc/operational_all/topology.dot b/test/case/misc/operational_all/topology.dot deleted file mode 120000 index 02b788692..000000000 --- a/test/case/misc/operational_all/topology.dot +++ /dev/null @@ -1 +0,0 @@ -../../../infamy/topologies/1x1.dot \ No newline at end of file diff --git a/test/case/misc/operational_all/topology.svg b/test/case/misc/operational_all/topology.svg deleted file mode 100644 index 6fc6f47a8..000000000 --- a/test/case/misc/operational_all/topology.svg +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - -1x1 - - - -host - -host - -mgmt - - - -target - -mgmt - -target - - - -host:mgmt--target:mgmt - - - - diff --git a/test/spec/generate_spec.py b/test/spec/generate_spec.py index 72085a6e6..ee676ba00 100755 --- a/test/spec/generate_spec.py +++ b/test/spec/generate_spec.py @@ -140,7 +140,9 @@ def gen_spec(self, title, variables): if title is None: title = visitor.name - self.gen_topology() + has_topology = os.path.exists(self.topo_dot) + if has_topology: + self.gen_topology() with open(self.spec_path, "w", encoding='utf-8') as spec: # This is the test name/title for the test-specification.pdf, @@ -156,8 +158,9 @@ def gen_spec(self, title, variables): spec.write("==== Description\n\n") spec.write(description + "\n\n") - spec.write("==== Topology\n\n") - spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n") + if has_topology: + spec.write("==== Topology\n\n") + spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n") spec.write("==== Sequence\n\n") spec.writelines([f". {step}\n" for step in test_steps])