diff --git a/docs/netlab/cli.md b/docs/netlab/cli.md index f94bad220e..d05faa06f5 100644 --- a/docs/netlab/cli.md +++ b/docs/netlab/cli.md @@ -25,6 +25,7 @@ The **netlab** command is the *netlab* CLI interface. It includes data model tra * **‌[netlab validate](validate.md)** executes tests defined in the lab topology on the lab devices * **[netlab down](down.md)** destroys the virtual lab. * **[netlab exec](exec.md)** executes a command on one or more network devices. +* **[netlab node](node.md)** starts, stops, or pauses individual *containerlab* nodes. * **[netlab api](api.md)** starts a lightweight HTTP API server for automation. ## Reports and Graphs @@ -74,6 +75,7 @@ The **netlab** command is the *netlab* CLI interface. It includes data model tra netlab inspect netlab install netlab libvirt + netlab node netlab report netlab restart netlab show diff --git a/docs/netlab/node.md b/docs/netlab/node.md new file mode 100644 index 0000000000..4cdb5d3f35 --- /dev/null +++ b/docs/netlab/node.md @@ -0,0 +1,61 @@ +(netlab-node)= +# Start, Stop, or Pause Individual Lab Nodes + +The **netlab node** command starts, stops, restarts, pauses, or unpauses individual nodes in a running lab without affecting the rest of the lab. + +```{warning} +* **netlab node** works only with *containerlab* nodes. +* A restarted container loses the device configuration deployed by _netlab_. **netlab node up** and **netlab node restart** therefore run **netlab initial** (or **netlab config --reload**) on the affected nodes unless you use the `--no-config` flag. +``` + +## Usage + +```text +usage: netlab node [-h] [-v] [-q] [--dry-run] [--no-config] [-r RELOAD] [-i INSTANCE] + {up,down,restart,pause,unpause} node [node ...] + +Start, stop, restart, pause, or unpause individual lab nodes + +positional arguments: + {up,down,restart,pause,unpause} + Action to execute on the selected nodes + node Node(s) to act on (node names, groups, or globs; separate them with spaces or commas) + +options: + -h, --help show this help message and exit + -v, --verbose Verbose logging (add multiple flags for increased verbosity) + -q, --quiet Report only major errors + --dry-run Print the commands that would be executed, but do not execute them + --no-config Do not configure the started nodes + -r RELOAD, --reload-config RELOAD + Reload saved configurations from specified directory + -i INSTANCE, --instance INSTANCE + Specify the lab instance to change nodes in +``` + +The **netlab node** command takes these parameters: + +* The action to execute: + * **down** stops the node container (`containerlab stop`) + * **up** starts a stopped node container and reconnects its links (`containerlab start`) + * **restart** stops and starts the node container (`containerlab restart`) + * **pause** freezes all processes in the node container (`docker pause`) + * **unpause** resumes a paused node container (`docker unpause`) +* A [list of nodes](netlab-inspect-node): node names, group names, or globs, separated by spaces or commas. + +Notes: + +* **netlab node** reads the transformed lab topology from the `netlab.snapshot.pickle` file created by **netlab up** or **netlab create**. Unmanaged nodes are ignored. +* After starting or restarting the nodes, **netlab node** deploys the initial device configuration with **netlab initial --limit _nodes_**. Use `--no-config` to skip this step, or `--reload-config _directory_` to reload saved configurations with **netlab config --reload** instead. +* With the `--instance` flag, you can change nodes in a lab instance running in a different directory. +* **[netlab status](status.md)** displays the state of stopped and paused containers. +* Routing protocol sessions with a paused node might time out. + +## Examples + +```bash +$ netlab node down r1 +$ netlab node up r1 +$ netlab node restart r1 r2 --no-config +$ netlab node pause r1,r2 +``` diff --git a/docs/netlab/status.md b/docs/netlab/status.md index a9867110a0..504343a998 100644 --- a/docs/netlab/status.md +++ b/docs/netlab/status.md @@ -89,7 +89,7 @@ Lab default in /home/user (netlab-status-provider)= ```{tip} -* **‌netlab status** executes **‌vagrant status --machine-readable** to get the status of Vagrant-controlled virtual machines and **‌docker ps** to get the status of running containers. The **vagrant status‌** might take a few seconds when executed on large labs and significantly longer if Vagrant cannot determine the state of a virtual machine (returning **‌inaccessible**). +* **‌netlab status** executes **‌vagrant status --machine-readable** to get the status of Vagrant-controlled virtual machines and **‌docker ps -a** to get the status of containers (including the [stopped ones](netlab-node)). The **vagrant status‌** might take a few seconds when executed on large labs and significantly longer if Vagrant cannot determine the state of a virtual machine (returning **‌inaccessible**). * **‌netlab status --memory** executes **‌virsh domstats** and **‌docker stats**. At least the **‌docker stats** command is not blazingly fast. * The **‌virsh domstats** reports the Resident Set Size (RSS) of the QEMU process running the virtual machine (the physical RAM allocation), not the amount of virtual RAM allocated to the virtual machine. * With Linux Kernel Samepage Merging (KSM) enabled, the reported memory utilization overstates the real usage. diff --git a/netsim/cli/node.py b/netsim/cli/node.py new file mode 100644 index 0000000000..2a246b2fbd --- /dev/null +++ b/netsim/cli/node.py @@ -0,0 +1,84 @@ +# +# netlab node command +# +# Start, stop, restart, pause, or unpause individual lab nodes without touching the rest of the lab +# +import argparse +import typing + +from ..providers import execute_node +from ..utils import log +from . import _nodeset, external_commands, load_snapshot, parser_add_verbose, parser_lab_location, set_dry_run + + +# +# CLI parser for 'netlab node' command +# +def node_parse(args: typing.List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="netlab node", + description='Start, stop, restart, pause, or unpause individual lab nodes') + parser_add_verbose(parser,quiet=True) + parser.add_argument( + '--dry-run', + dest='dry_run', + action='store_true', + help='Print the commands that would be executed, but do not execute them') + parser.add_argument( + '--no-config', + dest='no_config', + action='store_true', + help='Do not configure the started nodes') + parser.add_argument( + '-r','--reload-config', + dest='reload', + action='store', + help='Reload saved configurations from specified directory') + parser.add_argument( + dest='action', action='store', + choices=['up','down','restart','pause','unpause'], + help='Action to execute on the selected nodes') + parser.add_argument( + dest='node', action='store', nargs='+', + help='Node(s) to act on (node names, groups, or globs; separate them with spaces or commas)') + parser_lab_location(parser,instance=True,action='change nodes in') + + return parser.parse_args(args) + +# +# A restarted container loses its configuration: deploy the initial configuration +# (or reload the saved one) on the nodes we've just started +# +def configure_nodes(args: argparse.Namespace, node_list: typing.List[str]) -> None: + limit = ','.join(node_list) + if args.reload: + cmd = ['netlab','config','--reload',args.reload,'--limit',limit] + else: + cmd = ['netlab','initial','--no-message','--limit',limit] + + if not external_commands.run_command(external_commands.set_ansible_flags(cmd)): + log.fatal(f'netlab {cmd[1]} failed on nodes {limit}, aborting...','netlab node') + +def run(cli_args: typing.List[str]) -> None: + args = node_parse(cli_args) + log.set_logging_flags(args) + set_dry_run(args) + + if args.reload and args.no_config: + log.fatal('Cannot combine --reload-config and --no-config') + + topology = load_snapshot(args,ghosts=False) + node_list = _nodeset.parse_nodeset(','.join(args.node),topology) + log.exit_on_error() + + for node in node_list: + status = execute_node('control_node',node=topology.nodes[node],topology=topology,action=args.action) + if status is None: + log.error(f'The virtualization provider of node {node} does not support "netlab node"',module='node',skip_header=True) + elif not status: + log.error(f'Failed to {args.action} node {node}',module='node',skip_header=True) + + log.exit_on_error() + + if args.action in ('up','restart') and not args.no_config: + configure_nodes(args,node_list) diff --git a/netsim/cli/usage.txt b/netsim/cli/usage.txt index 5d8fa7acb3..3abfcda914 100644 --- a/netsim/cli/usage.txt +++ b/netsim/cli/usage.txt @@ -29,6 +29,8 @@ down Destroy the virtual lab exec Executes a command on one or more network devices +node Start, stop, restart, or pause individual containerlab nodes + tc Disable, enable, display, or modify link impairment parameters on libvirt virtual machines or containerlab-created Docker containers. diff --git a/netsim/providers/clab/__init__.py b/netsim/providers/clab/__init__.py index abb057926a..d3ed219b43 100644 --- a/netsim/providers/clab/__init__.py +++ b/netsim/providers/clab/__init__.py @@ -103,7 +103,7 @@ def health_check(self, nodeset: list, topology: Box) -> None: def get_lab_status(self,collect_status: dict) -> Box: try: status = external_commands.run_command( - 'docker ps --format json', + 'docker ps -a --format json', # -a: show stopped nodes as 'Exited' check_result=True, ignore_errors=True, return_stdout=True, @@ -222,6 +222,17 @@ def capture_command(self, node: Box, topology: Box, args: argparse.Namespace) -> node_name = self.get_node_name(node.name,topology) return strings.string_to_list(f'sudo ip netns exec {node_name}') + cmd + def control_node(self, node: Box, topology: Box, action: str) -> bool: + labops.set_clab_runtime(topology) + if action in ['pause','unpause']: # Containerlab cannot pause containers + cmd = [ 'docker', action, self.get_node_name(node.name,topology) ] + else: + c_action = { 'up': 'start', 'down': 'stop' }.get(action,action) + c_topo = 'clab.yml' if topology.provider == 'clab' else 'clab-augment.yml' + cmd = [ 'containerlab', c_action, '-t', c_topo, '--node', node.name ] + + return bool(external_commands.run_command(cmd,ignore_errors=True)) # containerlab logs to stderr, check exit code only + def set_tc(self, node: Box, topology: Box, intf: Box, error: bool = True) -> None: c_name = self.get_node_name(node.name,topology) c_intf = intf.get('clab.name',intf.ifname).replace('/','-') diff --git a/netsim/providers/clab/labops.py b/netsim/providers/clab/labops.py index f05052b9e3..0323d19394 100644 --- a/netsim/providers/clab/labops.py +++ b/netsim/providers/clab/labops.py @@ -155,11 +155,15 @@ def set_clab_runtime(topology: Box) -> None: if not runtime: return - log.info(f'Setting containerlab runtime to {runtime}') - os.environ['CLAB_RUNTIME'] = str(runtime) if runtime == 'podman' and os.geteuid() != 0: log.fatal('You must run netlab as the root user when using podman') + if os.environ.get('CLAB_RUNTIME',None) == str(runtime): + return + + log.info(f'Setting containerlab runtime to {runtime}') + os.environ['CLAB_RUNTIME'] = str(runtime) + def health_ready_setup(topology: Box) -> None: """ Modify device definitions to prepare for the container healthcheck. For every