Commit d07c8f43 authored by Zuo Xiang's avatar Zuo Xiang
Browse files

<examples>: Add a packet-based service migration example.

parent fc5e276d
......@@ -366,6 +366,9 @@ def removeContainer(self, name: str, wait: bool = True):
"""
with self._container_queue_lock:
container = self._name_container_map.get(name, None)
# The container could be already removed by the user via CLI or
# other approaches, raise the exception to let user handle this
# situation.
if not container:
raise ValueError(f"Can not find container with name: {container}")
......
FROM python:3.6-alpine3.9
COPY ./client.py /home/client.py
COPY ./server.py /home/server.py
CMD python /home/server.py
# Service Migration Example #
This example shows a simple service migration using ComNetsEmu.
The following figure depicts the setup:
```text
H1 --------- H2
|
|----- H3
```
While the client application is deployed on h1, the service application is firstly deployed on host h2.
The client application simply sends dummy UDP packets to the service address: 10.0.0.12:8888 every second and waits for
the response from the service application.
The service application simply perform a counting operation and return the number of packets received from the client.
The service application will be migrated between h2 and h3.
The state of the latest value of the counter is transmitted between h2 and h3 via their internal data network, which is
in a different subnet from the service address: 192.168.0.0/24.
So h2 and h3 has two interfaces up: one for service traffic and another one for state traffic between them.
Please check `./topology.py` for details.
## Run the emulation
Please first build the Docker image named `service_migration` with:
```bash
$ sudo ./build_docker_images.sh
```
Then the emulation script can be executed with:
```bash
$ sudo python3 ./topology.py
```
An example of the output:
```bash
- Internal IP of h2: 192.168.0.12
- Internal IP of h3: 192.168.0.13
PING 10.0.0.12 (10.0.0.12): 56 data bytes
64 bytes from 10.0.0.12: seq=0 ttl=64 time=204.414 ms
64 bytes from 10.0.0.12: seq=1 ttl=64 time=47.904 ms
64 bytes from 10.0.0.12: seq=2 ttl=64 time=40.560 ms
--- 10.0.0.12 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 40.560/97.626/204.414 ms
*** Setup1: Current log of the client:
2020-06-16T19:42:01.159265128Z Current counter: 1
2020-06-16T19:42:02.427236720Z Current counter: 2
2020-06-16T19:42:03.575836874Z Current counter: 3
2020-06-16T19:42:04.649414447Z Current counter: 4
2020-06-16T19:42:05.729810284Z Current counter: 5
2020-06-16T19:42:06.868402696Z Current counter: 6
2020-06-16T19:42:08.020400935Z Current counter: 7
2020-06-16T19:42:09.231398774Z Current counter: 8
2020-06-16T19:42:10.419174716Z Current counter: 9
*** Setup2: Current log of the client:
2020-06-16T19:42:01.159265128Z Current counter: 1
2020-06-16T19:42:02.427236720Z Current counter: 2
2020-06-16T19:42:03.575836874Z Current counter: 3
2020-06-16T19:42:04.649414447Z Current counter: 4
2020-06-16T19:42:05.729810284Z Current counter: 5
2020-06-16T19:42:06.868402696Z Current counter: 6
2020-06-16T19:42:08.020400935Z Current counter: 7
2020-06-16T19:42:09.231398774Z Current counter: 8
2020-06-16T19:42:10.419174716Z Current counter: 9
2020-06-16T19:42:11.470505370Z Current counter: 10
2020-06-16T19:42:12.832284709Z Current counter: 11
2020-06-16T19:42:13.886770602Z Current counter: 12
2020-06-16T19:42:14.931565808Z Current counter: 13
2020-06-16T19:42:15.973464801Z Current counter: 14
2020-06-16T19:42:17.015291447Z Current counter: 15
2020-06-16T19:42:18.056888966Z Current counter: 16
2020-06-16T19:42:19.098543492Z Current counter: 17
2020-06-16T19:42:20.140110212Z Current counter: 18
2020-06-16T19:42:21.181807849Z Current counter: 19
The PID of the old service: 22389
*** Setup3: Current log of the client:
2020-06-16T19:42:01.159265128Z Current counter: 1
2020-06-16T19:42:02.427236720Z Current counter: 2
2020-06-16T19:42:03.575836874Z Current counter: 3
2020-06-16T19:42:04.649414447Z Current counter: 4
2020-06-16T19:42:05.729810284Z Current counter: 5
2020-06-16T19:42:06.868402696Z Current counter: 6
2020-06-16T19:42:08.020400935Z Current counter: 7
2020-06-16T19:42:09.231398774Z Current counter: 8
2020-06-16T19:42:10.419174716Z Current counter: 9
2020-06-16T19:42:11.470505370Z Current counter: 10
2020-06-16T19:42:12.832284709Z Current counter: 11
2020-06-16T19:42:13.886770602Z Current counter: 12
2020-06-16T19:42:14.931565808Z Current counter: 13
2020-06-16T19:42:15.973464801Z Current counter: 14
2020-06-16T19:42:17.015291447Z Current counter: 15
2020-06-16T19:42:18.056888966Z Current counter: 16
2020-06-16T19:42:19.098543492Z Current counter: 17
2020-06-16T19:42:20.140110212Z Current counter: 18
2020-06-16T19:42:21.181807849Z Current counter: 19
2020-06-16T19:42:22.223770331Z Current counter: 20
2020-06-16T19:42:23.431841620Z Current counter: 21
2020-06-16T19:42:24.500341806Z Current counter: 22
2020-06-16T19:42:25.618693327Z Current counter: 23
2020-06-16T19:42:26.741078337Z Current counter: 24
2020-06-16T19:42:27.835002743Z Current counter: 25
2020-06-16T19:42:28.966631359Z Current counter: 26
2020-06-16T19:42:30.216317879Z Current counter: 27
2020-06-16T19:42:31.481871814Z Current counter: 28
2020-06-16T19:42:32.625274454Z Current counter: 29
mininet>
```
#!/bin/bash
echo "Build docker image for the service migration."
docker build -t service_migration --file ./Dockerfile .
docker image prune
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
About: Simple client.
"""
import socket
import time
SERVICE_IP = "10.0.0.12"
SERVICE_PORT = 8888
if __name__ == "__main__":
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
data = b"Show me the counter, please!"
while True:
sock.sendto(data, (SERVICE_IP, SERVICE_PORT))
counter, _ = sock.recvfrom(1024)
print("Current counter: {}".format(counter.decode("utf-8")))
time.sleep(1)
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
About: Simple server for counting.
"""
import argparse
import os
import signal
import socket
import time
INTERNAL_IP_H2 = "192.168.0.12"
INTERNAL_IP_H3 = "192.168.0.13"
INTERNAL_PORT = 9999
SERVICE_IP = "10.0.0.12"
SERVICE_PORT = 8888
HOST_NAME = None
def recv_state(host_name):
"""Get the latest counter state from the internal
network between h2 and h3.
"""
if host_name == "h2":
recv_ip = INTERNAL_IP_H2
else:
recv_ip = INTERNAL_IP_H3
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((recv_ip, INTERNAL_PORT))
state, _ = sock.recvfrom(1024)
state = int(state.decode("utf-8"))
return state
def run(host_name, get_state=False):
"""Run the couting service and handle sigterm signal."""
counter = 0
if get_state:
counter = recv_state(host_name)
print("Get the init counter state: {}".format(counter))
# Use closure to avoid using a global variable for state.
def term_signal_handler(signum, frame):
# Check if the server is running on the host 2.
if host_name == "h2":
dest_ip = INTERNAL_IP_H3
else:
dest_ip = INTERNAL_IP_H2
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# Send 3 duplicated packets to avoid losses.
for _ in range(3):
sock.sendto(str(counter).encode("utf-8"), (dest_ip, INTERNAL_PORT))
sock.close()
signal.signal(signal.SIGTERM, term_signal_handler)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((SERVICE_IP, SERVICE_PORT))
while True:
# Block here waiting for data input.
_, addr = sock.recvfrom(1024)
counter += 1
sock.sendto(str(counter).encode("utf-8"), addr)
time.sleep(0.5)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple counting server.")
parser.add_argument(
"hostname",
type=str,
help="The name of the host on which the server is deployed.",
)
parser.add_argument(
"--get_state", action="store_true", help="Get state from network.",
)
args = parser.parse_args()
run(args.hostname, args.get_state)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
About: Basic example of service (running inside a APPContainer) migration.
"""
import os
import shlex
import time
from subprocess import check_output
from comnetsemu.cli import CLI, spawnXtermDocker
from comnetsemu.net import Containernet, VNFManager
from mininet.link import TCLink
from mininet.log import info, setLogLevel
from mininet.node import Controller
def get_ofport(ifce: str):
"""Get the openflow port based on the iterface name.
:param ifce (str): Name of the interface.
"""
return (
check_output(shlex.split("ovs-vsctl get Interface {} ofport".format(ifce)))
.decode("utf-8")
.strip()
)
if __name__ == "__main__":
# Only used for auto-testing.
AUTOTEST_MODE = os.environ.get("COMNETSEMU_AUTOTEST_MODE", 0)
setLogLevel("info")
net = Containernet(controller=Controller, link=TCLink, xterms=False)
mgr = VNFManager(net)
info("*** Add the default controller\n")
net.addController("c0")
info("*** Creating the client and hosts\n")
h1 = net.addDockerHost(
"h1", dimage="dev_test", ip="10.0.0.11/24", docker_args={"hostname": "h1"},
)
# INFO: In order to make the host
h2 = net.addDockerHost(
"h2",
dimage="dev_test",
ip="10.0.0.12/24",
docker_args={"hostname": "h2", "pid_mode": "host"},
)
h3 = net.addDockerHost(
"h3",
dimage="dev_test",
ip="10.0.0.12/24",
docker_args={"hostname": "h3", "pid_mode": "host"},
)
info("*** Adding switch and links\n")
s1 = net.addSwitch("s1")
net.addLinkNamedIfce(s1, h1, bw=100, delay="10ms")
# Add the interfaces for service traffic.
net.addLinkNamedIfce(s1, h2, bw=100, delay="10ms")
net.addLinkNamedIfce(s1, h3, bw=100, delay="10ms")
# Add the interface for host internal traffic.
net.addLink(
s1, h2, bw=100, delay="1ms", intfName1="s1-h2-int", intfName2="h2-s1-int",
)
net.addLink(
s1, h3, bw=100, delay="1ms", intfName1="s1-h3-int", intfName2="h3-s1-int",
)
info("\n*** Starting network\n")
net.start()
s1_h1_port_num = get_ofport("s1-h1")
s1_h2_port_num = get_ofport("s1-h2")
s1_h3_port_num = get_ofport("s1-h3")
h2_mac = h2.MAC(intf="h2-s1")
h3_mac = h3.MAC(intf="h3-s1")
h2.setMAC("00:00:00:00:00:12", intf="h2-s1")
h3.setMAC("00:00:00:00:00:12", intf="h3-s1")
info("*** Use the subnet 192.168.0.0/24 for internal traffic between h2 and h3.\n")
print("- Internal IP of h2: 192.168.0.12")
print("- Internal IP of h3: 192.168.0.13")
h2.cmd("ip addr add 192.168.0.12/24 dev h2-s1-int")
h3.cmd("ip addr add 192.168.0.13/24 dev h3-s1-int")
h2.cmd("ping -c 3 192.168.0.13")
# INFO: For the simplicity, OpenFlow rules are managed directly via
# `ovs-ofctl` utility provided by the OvS.
# For realistic setup, switches should be managed by a remote controller.
info("*** Add flow to forward traffic from h1 to h2 to switch s1.\n")
check_output(
shlex.split(
'ovs-ofctl add-flow s1 "in_port={}, actions=output:{}"'.format(
s1_h1_port_num, s1_h2_port_num
)
)
)
check_output(
shlex.split(
'ovs-ofctl add-flow s1 "in_port={}, actions=output:{}"'.format(
s1_h2_port_num, s1_h1_port_num
)
)
)
check_output(
shlex.split(
'ovs-ofctl add-flow s1 "in_port={}, actions=output:{}"'.format(
s1_h3_port_num, s1_h1_port_num
)
)
)
info("*** h1 ping 10.0.0.12 with 3 packets: \n")
ret = h1.cmd("ping -c 3 10.0.0.12")
print(ret)
info("*** Deploy counter service on h2.\n")
counter_server_h2 = mgr.addContainer(
"counter_server_h2", "h2", "service_migration", "python /home/server.py h2",
)
info("*** Deploy client app on h1.\n")
client_app = mgr.addContainer(
"client", "h1", "service_migration", "python /home/client.py"
)
time.sleep(10)
client_log = client_app.getLogs()
print("\n*** Setup1: Current log of the client: \n{}".format(client_log))
info("*** Migrate (Re-deploy) the couter service to h3.\n")
counter_server_h3 = mgr.addContainer(
"counter_server_h3",
"h3",
"service_migration",
"python /home/server.py h3 --get_state",
)
info(
"*** Send SEGTERM signal to the service running on the h2.\n"
"Let it transfer its state through the internal network.\n"
)
pid_old_service = (
check_output(shlex.split("pgrep -f '^python /home/server.py h2$'"))
.decode("utf-8")
.strip()
)
for _ in range(3):
check_output(shlex.split("kill {}".format(pid_old_service)))
# Wait a little bit to let the signal work.
time.sleep(0.1)
# INFO: Used for testing.
# service_log = counter_server_h2.getLogs()
# print("\n*** Current log of the service on h2: \n{}".format(service_log))
# service_log = counter_server_h3.getLogs()
# print("\n*** Current log of the service on h3: \n{}".format(service_log))
mgr.removeContainer("counter_server_h2")
info("*** Mod the added flow to forward traffic from h1 to h3 to switch s1.\n")
check_output(
shlex.split(
'ovs-ofctl mod-flows s1 "in_port={}, actions=output:{}"'.format(
s1_h1_port_num, s1_h3_port_num
)
)
)
time.sleep(10)
client_log = client_app.getLogs()
print("\n*** Setup2: Current log of the client: \n{}".format(client_log))
info("*** Migrate (Re-deploy) the couter service back to h2.\n")
counter_server_h2 = mgr.addContainer(
"counter_server_h2",
"h2",
"service_migration",
"python /home/server.py h2 --get_state",
)
pid_old_service = (
check_output(shlex.split("pgrep -f '^python /home/server.py h3 --get_state$'"))
.decode("utf-8")
.strip()
)
print(f"The PID of the old service: {pid_old_service}")
for _ in range(3):
check_output(shlex.split("kill {}".format(pid_old_service)))
# Wait a little bit to let the signal work.
time.sleep(0.1)
mgr.removeContainer("counter_server_h3")
info("*** Mod the added flow to forward traffic from h1 back to h2 to switch s1.\n")
check_output(
shlex.split(
'ovs-ofctl mod-flows s1 "in_port={}, actions=output:{}"'.format(
s1_h1_port_num, s1_h2_port_num
)
)
)
time.sleep(10)
client_log = client_app.getLogs()
print("\n*** Setup3: Current log of the client: \n{}".format(client_log))
if not AUTOTEST_MODE:
# Cannot spawn xterm for srv1 since BASH is not installed in the image:
# echo_server.
# spawnXtermDocker("srv2")
CLI(net)
try:
mgr.removeContainer("counter_server_h2")
mgr.removeContainer("counter_server_h3")
except Exception as e:
print(e)
finally:
net.stop()
mgr.stop()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment