P4 Developer Day 2018 Spring (#159)

* Repository reorganization for 2018 Spring P4 Developer Day.

* Port tutorial exercises to P4Runtime with static controller (#156)

* Switch VM to a minimal Ubuntu 16.04 desktop image

* Add commands to install Protobuf Python bindings to user_bootstrap.sh

* Implement P4Runtime static controller for use in exercises

From the exercise perspective, the main difference is that control plane
rules are now specified using JSON files instead of CLI commands. Such
JSON files define rules that use the same name for tables, keys, etc. as
in the P4Info file.

All P4Runtime requests generated as part of the make run process are
logged in the exercise's “logs” directory, making it easier for students
to see the actual P4Runtime messages sent to the switch.

Only the "basic" exercise has been ported to use P4Runtime.
The "p4runtime" exercise has been updated to work with P4Runtime
protocol changes.

Known issues:
- make run hangs in case of errors when running the P4Runtime controller
    (probably due to gRPC stream channel threads not terminated properly)
- missing support for inserting table entries with default action
    (can specify in P4 program as a workaround)

* Force install protobuf python module

* Fixing Ctrl-C hang by shutdown switches

* Moving gRPC error print to function for readability

Unforuntately, if this gets moved out of the file, the process hangs.
We'll need to figure out how why later.

* Renaming ShutdownAllSwitches -> ShutdownAllSwitchConnections

* Reverting counter index change

* Porting the ECN exercise to use P4 Runtime Static Controller

* updating the README in the ecn exercise to reflect the change in rule files

* Allow set table default action in P4Runtime static controller

* Fixed undefined match string when printing P4Runtime table entry

* Updated basic_tunnel exercise to use P4Runtime controller.

* Changed default action in the basic exercise's ipv4_lpm table to drop

* Porting the MRI exercise to use P4runtime with static controller

* Updating readme to reflect the change of controller for mri

* Update calc exercise for P4Runtime static controller

* Port source_routing to P4 Runtime static controller (#157)

* Port Load Balance to P4 Runtime Static Controller (#158)
This commit is contained in:
Nate Foster
2018-06-01 02:54:33 -04:00
committed by GitHub
parent e7e6899d5c
commit dc08948a34
503 changed files with 1432 additions and 30666 deletions

View File

View File

@@ -0,0 +1,30 @@
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from switch import SwitchConnection
from p4.tmp import p4config_pb2
def buildDeviceConfig(bmv2_json_file_path=None):
"Builds the device config for BMv2"
device_config = p4config_pb2.P4DeviceConfig()
device_config.reassign = True
with open(bmv2_json_file_path) as f:
device_config.device_data = f.read()
return device_config
class Bmv2SwitchConnection(SwitchConnection):
def buildDeviceConfig(self, **kwargs):
return buildDeviceConfig(**kwargs)

View File

@@ -0,0 +1,119 @@
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
import socket
import math
'''
This package contains several helper functions for encoding to and decoding from byte strings:
- integers
- IPv4 address strings
- Ethernet address strings
'''
mac_pattern = re.compile('^([\da-fA-F]{2}:){5}([\da-fA-F]{2})$')
def matchesMac(mac_addr_string):
return mac_pattern.match(mac_addr_string) is not None
def encodeMac(mac_addr_string):
return mac_addr_string.replace(':', '').decode('hex')
def decodeMac(encoded_mac_addr):
return ':'.join(s.encode('hex') for s in encoded_mac_addr)
ip_pattern = re.compile('^(\d{1,3}\.){3}(\d{1,3})$')
def matchesIPv4(ip_addr_string):
return ip_pattern.match(ip_addr_string) is not None
def encodeIPv4(ip_addr_string):
return socket.inet_aton(ip_addr_string)
def decodeIPv4(encoded_ip_addr):
return socket.inet_ntoa(encoded_ip_addr)
def bitwidthToBytes(bitwidth):
return int(math.ceil(bitwidth / 8.0))
def encodeNum(number, bitwidth):
byte_len = bitwidthToBytes(bitwidth)
num_str = '%x' % number
if number >= 2 ** bitwidth:
raise Exception("Number, %d, does not fit in %d bits" % (number, bitwidth))
return ('0' * (byte_len * 2 - len(num_str)) + num_str).decode('hex')
def decodeNum(encoded_number):
return int(encoded_number.encode('hex'), 16)
def encode(x, bitwidth):
'Tries to infer the type of `x` and encode it'
byte_len = bitwidthToBytes(bitwidth)
if (type(x) == list or type(x) == tuple) and len(x) == 1:
x = x[0]
encoded_bytes = None
if type(x) == str:
if matchesMac(x):
encoded_bytes = encodeMac(x)
elif matchesIPv4(x):
encoded_bytes = encodeIPv4(x)
else:
# Assume that the string is already encoded
encoded_bytes = x
elif type(x) == int:
encoded_bytes = encodeNum(x, bitwidth)
else:
raise Exception("Encoding objects of %r is not supported" % type(x))
assert(len(encoded_bytes) == byte_len)
return encoded_bytes
if __name__ == '__main__':
# TODO These tests should be moved out of main eventually
mac = "aa:bb:cc:dd:ee:ff"
enc_mac = encodeMac(mac)
assert(enc_mac == '\xaa\xbb\xcc\xdd\xee\xff')
dec_mac = decodeMac(enc_mac)
assert(mac == dec_mac)
ip = "10.0.0.1"
enc_ip = encodeIPv4(ip)
assert(enc_ip == '\x0a\x00\x00\x01')
dec_ip = decodeIPv4(enc_ip)
assert(ip == dec_ip)
num = 1337
byte_len = 5
enc_num = encodeNum(num, byte_len * 8)
assert(enc_num == '\x00\x00\x00\x05\x39')
dec_num = decodeNum(enc_num)
assert(num == dec_num)
assert(matchesIPv4('10.0.0.1'))
assert(not matchesIPv4('10.0.0.1.5'))
assert(not matchesIPv4('1000.0.0.1'))
assert(not matchesIPv4('10001'))
assert(encode(mac, 6 * 8) == enc_mac)
assert(encode(ip, 4 * 8) == enc_ip)
assert(encode(num, 5 * 8) == enc_num)
assert(encode((num,), 5 * 8) == enc_num)
assert(encode([num], 5 * 8) == enc_num)
num = 256
byte_len = 2
try:
enc_num = encodeNum(num, 8)
raise Exception("expected exception")
except Exception as e:
print e

View File

@@ -0,0 +1,193 @@
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
import google.protobuf.text_format
from p4 import p4runtime_pb2
from p4.config import p4info_pb2
from convert import encode
class P4InfoHelper(object):
def __init__(self, p4_info_filepath):
p4info = p4info_pb2.P4Info()
# Load the p4info file into a skeleton P4Info object
with open(p4_info_filepath) as p4info_f:
google.protobuf.text_format.Merge(p4info_f.read(), p4info)
self.p4info = p4info
def get(self, entity_type, name=None, id=None):
if name is not None and id is not None:
raise AssertionError("name or id must be None")
for o in getattr(self.p4info, entity_type):
pre = o.preamble
if name:
if (pre.name == name or pre.alias == name):
return o
else:
if pre.id == id:
return o
if name:
raise AttributeError("Could not find %r of type %s" % (name, entity_type))
else:
raise AttributeError("Could not find id %r of type %s" % (id, entity_type))
def get_id(self, entity_type, name):
return self.get(entity_type, name=name).preamble.id
def get_name(self, entity_type, id):
return self.get(entity_type, id=id).preamble.name
def get_alias(self, entity_type, id):
return self.get(entity_type, id=id).preamble.alias
def __getattr__(self, attr):
# Synthesize convenience functions for name to id lookups for top-level entities
# e.g. get_tables_id(name_string) or get_actions_id(name_string)
m = re.search("^get_(\w+)_id$", attr)
if m:
primitive = m.group(1)
return lambda name: self.get_id(primitive, name)
# Synthesize convenience functions for id to name lookups
# e.g. get_tables_name(id) or get_actions_name(id)
m = re.search("^get_(\w+)_name$", attr)
if m:
primitive = m.group(1)
return lambda id: self.get_name(primitive, id)
raise AttributeError("%r object has no attribute %r" % (self.__class__, attr))
def get_match_field(self, table_name, name=None, id=None):
for t in self.p4info.tables:
pre = t.preamble
if pre.name == table_name:
for mf in t.match_fields:
if name is not None:
if mf.name == name:
return mf
elif id is not None:
if mf.id == id:
return mf
raise AttributeError("%r has no attribute %r" % (table_name, name if name is not None else id))
def get_match_field_id(self, table_name, match_field_name):
return self.get_match_field(table_name, name=match_field_name).id
def get_match_field_name(self, table_name, match_field_id):
return self.get_match_field(table_name, id=match_field_id).name
def get_match_field_pb(self, table_name, match_field_name, value):
p4info_match = self.get_match_field(table_name, match_field_name)
bitwidth = p4info_match.bitwidth
p4runtime_match = p4runtime_pb2.FieldMatch()
p4runtime_match.field_id = p4info_match.id
match_type = p4info_match.match_type
if match_type == p4info_pb2.MatchField.VALID:
valid = p4runtime_match.valid
valid.value = bool(value)
elif match_type == p4info_pb2.MatchField.EXACT:
exact = p4runtime_match.exact
exact.value = encode(value, bitwidth)
elif match_type == p4info_pb2.MatchField.LPM:
lpm = p4runtime_match.lpm
lpm.value = encode(value[0], bitwidth)
lpm.prefix_len = value[1]
elif match_type == p4info_pb2.MatchField.TERNARY:
lpm = p4runtime_match.ternary
lpm.value = encode(value[0], bitwidth)
lpm.mask = encode(value[1], bitwidth)
elif match_type == p4info_pb2.MatchField.RANGE:
lpm = p4runtime_match.range
lpm.low = encode(value[0], bitwidth)
lpm.high = encode(value[1], bitwidth)
else:
raise Exception("Unsupported match type with type %r" % match_type)
return p4runtime_match
def get_match_field_value(self, match_field):
match_type = match_field.WhichOneof("field_match_type")
if match_type == 'valid':
return match_field.valid.value
elif match_type == 'exact':
return match_field.exact.value
elif match_type == 'lpm':
return (match_field.lpm.value, match_field.lpm.prefix_len)
elif match_type == 'ternary':
return (match_field.ternary.value, match_field.ternary.mask)
elif match_type == 'range':
return (match_field.range.low, match_field.range.high)
else:
raise Exception("Unsupported match type with type %r" % match_type)
def get_action_param(self, action_name, name=None, id=None):
for a in self.p4info.actions:
pre = a.preamble
if pre.name == action_name:
for p in a.params:
if name is not None:
if p.name == name:
return p
elif id is not None:
if p.id == id:
return p
raise AttributeError("action %r has no param %r, (has: %r)" % (action_name, name if name is not None else id, a.params))
def get_action_param_id(self, action_name, param_name):
return self.get_action_param(action_name, name=param_name).id
def get_action_param_name(self, action_name, param_id):
return self.get_action_param(action_name, id=param_id).name
def get_action_param_pb(self, action_name, param_name, value):
p4info_param = self.get_action_param(action_name, param_name)
p4runtime_param = p4runtime_pb2.Action.Param()
p4runtime_param.param_id = p4info_param.id
p4runtime_param.value = encode(value, p4info_param.bitwidth)
return p4runtime_param
def buildTableEntry(self,
table_name,
match_fields=None,
default_action=False,
action_name=None,
action_params=None,
priority=None):
table_entry = p4runtime_pb2.TableEntry()
table_entry.table_id = self.get_tables_id(table_name)
if priority is not None:
table_entry.priority = priority
if match_fields:
table_entry.match.extend([
self.get_match_field_pb(table_name, match_field_name, value)
for match_field_name, value in match_fields.iteritems()
])
if default_action:
table_entry.is_default_action = True
if action_name:
action = table_entry.action.action
action.action_id = self.get_actions_id(action_name)
if action_params:
action.params.extend([
self.get_action_param_pb(action_name, field_name, value)
for field_name, value in action_params.iteritems()
])
return table_entry

View File

@@ -0,0 +1,195 @@
#!/usr/bin/env python2
#
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import json
import os
import sys
import bmv2
import helper
def error(msg):
print >> sys.stderr, ' - ERROR! ' + msg
def info(msg):
print >> sys.stdout, ' - ' + msg
class ConfException(Exception):
pass
def main():
parser = argparse.ArgumentParser(description='P4Runtime Simple Controller')
parser.add_argument('-a', '--p4runtime-server-addr',
help='address and port of the switch\'s P4Runtime server (e.g. 192.168.0.1:50051)',
type=str, action="store", required=True)
parser.add_argument('-d', '--device-id',
help='Internal device ID to use in P4Runtime messages',
type=int, action="store", required=True)
parser.add_argument('-p', '--proto-dump-file',
help='path to file where to dump protobuf messages sent to the switch',
type=str, action="store", required=True)
parser.add_argument("-c", '--runtime-conf-file',
help="path to input runtime configuration file (JSON)",
type=str, action="store", required=True)
args = parser.parse_args()
if not os.path.exists(args.runtime_conf_file):
parser.error("File %s does not exist!" % args.runtime_conf_file)
workdir = os.path.dirname(os.path.abspath(args.runtime_conf_file))
with open(args.runtime_conf_file, 'r') as sw_conf_file:
program_switch(addr=args.p4runtime_server_addr,
device_id=args.device_id,
sw_conf_file=sw_conf_file,
workdir=workdir,
proto_dump_fpath=args.proto_dump_file)
def check_switch_conf(sw_conf, workdir):
required_keys = ["p4info"]
files_to_check = ["p4info"]
target_choices = ["bmv2"]
if "target" not in sw_conf:
raise ConfException("missing key 'target'")
target = sw_conf['target']
if target not in target_choices:
raise ConfException("unknown target '%s'" % target)
if target == 'bmv2':
required_keys.append("bmv2_json")
files_to_check.append("bmv2_json")
for conf_key in required_keys:
if conf_key not in sw_conf or len(sw_conf[conf_key]) == 0:
raise ConfException("missing key '%s' or empty value" % conf_key)
for conf_key in files_to_check:
real_path = os.path.join(workdir, sw_conf[conf_key])
if not os.path.exists(real_path):
raise ConfException("file does not exist %s" % real_path)
def program_switch(addr, device_id, sw_conf_file, workdir, proto_dump_fpath):
sw_conf = json_load_byteified(sw_conf_file)
try:
check_switch_conf(sw_conf=sw_conf, workdir=workdir)
except ConfException as e:
error("While parsing input runtime configuration: %s" % str(e))
return
info('Using P4Info file %s...' % sw_conf['p4info'])
p4info_fpath = os.path.join(workdir, sw_conf['p4info'])
p4info_helper = helper.P4InfoHelper(p4info_fpath)
target = sw_conf['target']
info("Connecting to P4Runtime server on %s (%s)..." % (addr, target))
if target == "bmv2":
sw = bmv2.Bmv2SwitchConnection(address=addr, device_id=device_id,
proto_dump_file=proto_dump_fpath)
else:
raise Exception("Don't know how to connect to target %s" % target)
try:
sw.MasterArbitrationUpdate()
if target == "bmv2":
info("Setting pipeline config (%s)..." % sw_conf['bmv2_json'])
bmv2_json_fpath = os.path.join(workdir, sw_conf['bmv2_json'])
sw.SetForwardingPipelineConfig(p4info=p4info_helper.p4info,
bmv2_json_file_path=bmv2_json_fpath)
else:
raise Exception("Should not be here")
if 'table_entries' in sw_conf:
table_entries = sw_conf['table_entries']
info("Inserting %d table entries..." % len(table_entries))
for entry in table_entries:
info(tableEntryToString(entry))
insertTableEntry(sw, entry, p4info_helper)
finally:
sw.shutdown()
def insertTableEntry(sw, flow, p4info_helper):
table_name = flow['table']
match_fields = flow.get('match') # None if not found
action_name = flow['action_name']
default_action = flow.get('default_action') # None if not found
action_params = flow['action_params']
priority = flow.get('priority') # None if not found
table_entry = p4info_helper.buildTableEntry(
table_name=table_name,
match_fields=match_fields,
default_action=default_action,
action_name=action_name,
action_params=action_params,
priority=priority)
sw.WriteTableEntry(table_entry)
# object hook for josn library, use str instead of unicode object
# https://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-of-unicode-from-json
def json_load_byteified(file_handle):
return _byteify(json.load(file_handle, object_hook=_byteify),
ignore_dicts=True)
def _byteify(data, ignore_dicts=False):
# if this is a unicode string, return its string representation
if isinstance(data, unicode):
return data.encode('utf-8')
# if this is a list of values, return list of byteified values
if isinstance(data, list):
return [_byteify(item, ignore_dicts=True) for item in data]
# if this is a dictionary, return dictionary of byteified keys and values
# but only if we haven't already byteified it
if isinstance(data, dict) and not ignore_dicts:
return {
_byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
for key, value in data.iteritems()
}
# if it's anything else, return it in its original form
return data
def tableEntryToString(flow):
if 'match' in flow:
match_str = ['%s=%s' % (match_name, str(flow['match'][match_name])) for match_name in
flow['match']]
match_str = ', '.join(match_str)
elif 'default_action' in flow and flow['default_action']:
match_str = '(default action)'
else:
match_str = '(any)'
params = ['%s=%s' % (param_name, str(flow['action_params'][param_name])) for param_name in
flow['action_params']]
params = ', '.join(params)
return "%s: %s => %s(%s)" % (
flow['table'], match_str, flow['action_name'], params)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,166 @@
# Copyright 2017-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from Queue import Queue
from abc import abstractmethod
from datetime import datetime
import grpc
from p4 import p4runtime_pb2
from p4.tmp import p4config_pb2
MSG_LOG_MAX_LEN = 1024
# List of all active connections
connections = []
def ShutdownAllSwitchConnections():
for c in connections:
c.shutdown()
class SwitchConnection(object):
def __init__(self, name=None, address='127.0.0.1:50051', device_id=0,
proto_dump_file=None):
self.name = name
self.address = address
self.device_id = device_id
self.p4info = None
self.channel = grpc.insecure_channel(self.address)
if proto_dump_file is not None:
interceptor = GrpcRequestLogger(proto_dump_file)
self.channel = grpc.intercept_channel(self.channel, interceptor)
self.client_stub = p4runtime_pb2.P4RuntimeStub(self.channel)
self.requests_stream = IterableQueue()
self.stream_msg_resp = self.client_stub.StreamChannel(iter(self.requests_stream))
self.proto_dump_file = proto_dump_file
connections.append(self)
@abstractmethod
def buildDeviceConfig(self, **kwargs):
return p4config_pb2.P4DeviceConfig()
def shutdown(self):
self.requests_stream.close()
self.stream_msg_resp.cancel()
def MasterArbitrationUpdate(self, dry_run=False, **kwargs):
request = p4runtime_pb2.StreamMessageRequest()
request.arbitration.device_id = self.device_id
request.arbitration.election_id.high = 0
request.arbitration.election_id.low = 1
if dry_run:
print "P4 Runtime MasterArbitrationUpdate: ", request
else:
self.requests_stream.put(request)
def SetForwardingPipelineConfig(self, p4info, dry_run=False, **kwargs):
device_config = self.buildDeviceConfig(**kwargs)
request = p4runtime_pb2.SetForwardingPipelineConfigRequest()
request.election_id.low = 1
request.device_id = self.device_id
config = request.config
config.p4info.CopyFrom(p4info)
config.p4_device_config = device_config.SerializeToString()
request.action = p4runtime_pb2.SetForwardingPipelineConfigRequest.VERIFY_AND_COMMIT
if dry_run:
print "P4 Runtime SetForwardingPipelineConfig:", request
else:
self.client_stub.SetForwardingPipelineConfig(request)
def WriteTableEntry(self, table_entry, dry_run=False):
request = p4runtime_pb2.WriteRequest()
request.device_id = self.device_id
request.election_id.low = 1
update = request.updates.add()
update.type = p4runtime_pb2.Update.INSERT
update.entity.table_entry.CopyFrom(table_entry)
if dry_run:
print "P4 Runtime Write:", request
else:
self.client_stub.Write(request)
def ReadTableEntries(self, table_id=None, dry_run=False):
request = p4runtime_pb2.ReadRequest()
request.device_id = self.device_id
entity = request.entities.add()
table_entry = entity.table_entry
if table_id is not None:
table_entry.table_id = table_id
else:
table_entry.table_id = 0
if dry_run:
print "P4 Runtime Read:", request
else:
for response in self.client_stub.Read(request):
yield response
def ReadCounters(self, counter_id=None, index=None, dry_run=False):
request = p4runtime_pb2.ReadRequest()
request.device_id = self.device_id
entity = request.entities.add()
counter_entry = entity.counter_entry
if counter_id is not None:
counter_entry.counter_id = counter_id
else:
counter_entry.counter_id = 0
if index is not None:
counter_entry.index.index = index
if dry_run:
print "P4 Runtime Read:", request
else:
for response in self.client_stub.Read(request):
yield response
class GrpcRequestLogger(grpc.UnaryUnaryClientInterceptor,
grpc.UnaryStreamClientInterceptor):
"""Implementation of a gRPC interceptor that logs request to a file"""
def __init__(self, log_file):
self.log_file = log_file
with open(self.log_file, 'w') as f:
# Clear content if it exists.
f.write("")
def log_message(self, method_name, body):
with open(self.log_file, 'a') as f:
ts = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
msg = str(body)
f.write("\n[%s] %s\n---\n" % (ts, method_name))
if len(msg) < MSG_LOG_MAX_LEN:
f.write(str(body))
else:
f.write("Message too long (%d bytes)! Skipping log...\n" % len(msg))
f.write('---\n')
def intercept_unary_unary(self, continuation, client_call_details, request):
self.log_message(client_call_details.method, request)
return continuation(client_call_details, request)
def intercept_unary_stream(self, continuation, client_call_details, request):
self.log_message(client_call_details.method, request)
return continuation(client_call_details, request)
class IterableQueue(Queue):
_sentinel = object()
def __iter__(self):
return iter(self.get, self._sentinel)
def close(self):
self.put(self._sentinel)