[6320] | 1 | # -*- coding: utf-8 -*- |
---|
| 2 | """ |
---|
| 3 | ------------------------------------------------------------------------------ |
---|
| 4 | Mango 802.11 Reference Design Experiments Framework - Messages |
---|
| 5 | ------------------------------------------------------------------------------ |
---|
| 6 | License: Copyright 2019 Mango Communications, Inc. All rights reserved. |
---|
| 7 | Use and distribution subject to terms in LICENSE.txt |
---|
| 8 | ------------------------------------------------------------------------------ |
---|
| 9 | |
---|
| 10 | This module provides class definitions for all over-the-wire messages. |
---|
| 11 | |
---|
| 12 | Functions (see below for more information): |
---|
| 13 | TransportHeader() -- Transport Header to describe message contents |
---|
| 14 | Cmd() -- Base class for commands the require responses |
---|
| 15 | BufferCmd() -- Base class for commands that require buffer responses |
---|
| 16 | Resp() -- Base class for responses (single packet) |
---|
| 17 | Buffer() -- Base class for responses (multiple packets) |
---|
| 18 | |
---|
| 19 | """ |
---|
| 20 | |
---|
| 21 | import struct |
---|
| 22 | |
---|
| 23 | from . import transport |
---|
| 24 | |
---|
| 25 | __all__ = ['TransportHeader', 'Cmd', 'BufferCmd', 'Resp', 'Buffer'] |
---|
| 26 | |
---|
| 27 | # Buffer Command defines |
---|
| 28 | CMD_BUFFER_GET_SIZE_FROM_DATA = 0xFFFFFFFF |
---|
| 29 | |
---|
| 30 | class Message(object): |
---|
| 31 | """Base class for messages to / from 802.11 ref design nodes. |
---|
| 32 | |
---|
| 33 | Attributes: |
---|
| 34 | const -- Dictionary of constants |
---|
| 35 | """ |
---|
| 36 | const = None |
---|
| 37 | |
---|
| 38 | def __init__(self): |
---|
| 39 | self.const = dict() |
---|
| 40 | |
---|
| 41 | def add_const(self, name, value): |
---|
| 42 | """Add a constant to the Message.""" |
---|
| 43 | if (name in self.const.keys()): |
---|
| 44 | print("WARNING: Updating value of constant: {0}".format(name)) |
---|
| 45 | self.const[name] = value |
---|
| 46 | |
---|
| 47 | def get_const(self, name): |
---|
| 48 | """Get a constant from the Message.""" |
---|
| 49 | if (name in self.const.keys()): |
---|
| 50 | return self.const[name] |
---|
| 51 | else: |
---|
| 52 | msg = "Constant {0} does not exist ".format(name) |
---|
| 53 | msg += "in {0}".format(self.__class__.__name__) |
---|
| 54 | raise AttributeError(msg) |
---|
| 55 | |
---|
| 56 | def serialize(self,): raise NotImplementedError |
---|
| 57 | def deserialize(self, data_buffer): raise NotImplementedError |
---|
| 58 | def sizeof(self,): raise NotImplementedError |
---|
| 59 | |
---|
| 60 | # End Class |
---|
| 61 | |
---|
| 62 | |
---|
| 63 | class TransportHeader(Message): |
---|
| 64 | """Class for Transport Header describing message contents. |
---|
| 65 | |
---|
| 66 | Attributes: |
---|
| 67 | dest_id -- (uint16) Destination ID of the message |
---|
| 68 | src_id -- (uint16) Source ID of the message |
---|
| 69 | length -- (uint16) Length of the payload in bytes |
---|
| 70 | seq_num -- (uint16) Sequence number of the message |
---|
| 71 | flags -- (uint16) Flags of the message |
---|
| 72 | """ |
---|
| 73 | |
---|
| 74 | def __init__(self, dest_id=0, src_id=0, reserved=0, length=0, seq_num=0, flags=0): |
---|
| 75 | super(TransportHeader, self).__init__() |
---|
| 76 | self.dest_id = dest_id |
---|
| 77 | self.src_id = src_id |
---|
| 78 | self.reserved = reserved |
---|
| 79 | self.length = length |
---|
| 80 | self.seq_num = seq_num |
---|
| 81 | self.flags = flags |
---|
| 82 | |
---|
| 83 | def serialize(self): |
---|
| 84 | """Return a bytes object of a packed transport header.""" |
---|
| 85 | return struct.pack('!5H', |
---|
| 86 | self.dest_id, self.src_id, |
---|
| 87 | self.length, self.seq_num, self.flags) |
---|
| 88 | |
---|
| 89 | def deserialize(self, data_buffer): |
---|
| 90 | """Not used for Transport headers""" |
---|
| 91 | pass |
---|
| 92 | |
---|
| 93 | def sizeof(self): |
---|
| 94 | """Return the size of the transport header.""" |
---|
| 95 | return struct.calcsize('!5H') |
---|
| 96 | |
---|
| 97 | def increment(self, step=1): |
---|
| 98 | """Increment the sequence number of the header by a given step.""" |
---|
| 99 | self.seq_num = (self.seq_num + step) % 0xFFFF |
---|
| 100 | |
---|
| 101 | def set_length(self, value): self.length = value |
---|
| 102 | def set_src_id(self, value): self.src_id = value |
---|
| 103 | def set_dest_id(self, value): self.dest_id = value |
---|
| 104 | |
---|
| 105 | def get_length(self): return self.length |
---|
| 106 | def get_src_id(self): return self.src_id |
---|
| 107 | def get_dest_id(self): return self.dest_id |
---|
| 108 | |
---|
| 109 | def response_required(self): |
---|
| 110 | """Sets bit 0 of the flags since a response is required.""" |
---|
| 111 | self.flags = self.flags | 0x1 |
---|
| 112 | |
---|
| 113 | def response_not_required(self): |
---|
| 114 | """Clears bit 0 of the flags since a response is not required.""" |
---|
| 115 | self.flags = self.flags & 0xFFFE |
---|
| 116 | |
---|
| 117 | def reset(self): |
---|
| 118 | """Reset the sequence number of the transport header.""" |
---|
| 119 | self.seq_num = 1 |
---|
| 120 | |
---|
| 121 | def is_reply(self, input_data): |
---|
| 122 | """Checks input_data to see if it is a valid reply to the last |
---|
| 123 | outgoing packet. |
---|
| 124 | |
---|
| 125 | Checks: |
---|
| 126 | input_data.dest_id == self.src_id |
---|
| 127 | input_data.src_id == self.dest_id |
---|
| 128 | input_data.seq_num == self.seq_num |
---|
| 129 | |
---|
| 130 | Raises a TypeError excpetion if input data is not the correct size. |
---|
| 131 | """ |
---|
| 132 | if len(input_data) == self.sizeof(): |
---|
| 133 | # FIXME: embedded TransportHeader format, here instead of deserialize |
---|
| 134 | dataTuple = struct.unpack('!5H', input_data[0:10]) |
---|
| 135 | |
---|
| 136 | if ((self.dest_id != dataTuple[1]) or |
---|
| 137 | (self.src_id != dataTuple[0]) or |
---|
| 138 | (self.seq_num != dataTuple[3])): |
---|
| 139 | msg = "WARNING: transport header mismatch:" |
---|
| 140 | msg += "[{0:d} {1:d}]".format(self.dest_id, dataTuple[1]) |
---|
| 141 | msg += "[{0:d} {1:d}]".format(self.src_id, dataTuple[0]) |
---|
| 142 | msg += "[{0:d} {1:d}]".format(self.seq_num, dataTuple[3]) |
---|
| 143 | print(msg) |
---|
| 144 | return False |
---|
| 145 | else: |
---|
| 146 | return True |
---|
| 147 | else: |
---|
| 148 | raise TypeError(str("TransportHeader: length of header " + |
---|
| 149 | "did not match size of transport header")) |
---|
| 150 | |
---|
| 151 | # End Class |
---|
| 152 | |
---|
| 153 | |
---|
| 154 | class CmdRespMessage(Message): |
---|
| 155 | """Base class for command / response messages. |
---|
| 156 | |
---|
| 157 | Attributes: |
---|
| 158 | command -- (uint32) ID of the command / response |
---|
| 159 | length -- (uint16) Length of the cmd / resp arguments (args) in bytes |
---|
| 160 | num_args -- (uint16) Number of uint32 arguments |
---|
| 161 | args -- (list of uint32) Arguments of the command / reponse |
---|
| 162 | """ |
---|
| 163 | command = None |
---|
| 164 | length = None |
---|
| 165 | num_args = None |
---|
| 166 | args = None |
---|
| 167 | payload = None |
---|
| 168 | raw_data = None |
---|
| 169 | |
---|
| 170 | def __init__(self, command=0, length=0, num_args=0, args=None, payload=None): |
---|
| 171 | super(CmdRespMessage, self).__init__() |
---|
| 172 | self.command = command |
---|
| 173 | self.length = length |
---|
| 174 | self.num_args = num_args |
---|
| 175 | self.args = args or [] |
---|
| 176 | self.payload = payload or [] |
---|
| 177 | self.raw_data = bytearray(self.serialize()) |
---|
| 178 | |
---|
| 179 | def serialize(self): |
---|
| 180 | """Return a bytes object of a packed command / response.""" |
---|
| 181 | ret_val = b'' |
---|
| 182 | |
---|
| 183 | # Add the command header |
---|
| 184 | ret_val += struct.pack('!I 2H', self.command, self.length, self.num_args) |
---|
| 185 | |
---|
| 186 | # Add the command arguments, if any |
---|
| 187 | if self.num_args > 0: |
---|
| 188 | ret_val += struct.pack('!{0}I'.format(self.num_args), *self.args) |
---|
| 189 | |
---|
| 190 | # Add the command payload, if any |
---|
| 191 | if self.payload: |
---|
| 192 | ret_val += self.payload |
---|
| 193 | |
---|
| 194 | self.raw_data = bytearray(ret_val) |
---|
| 195 | return ret_val |
---|
| 196 | |
---|
| 197 | def deserialize(self, data_buffer): |
---|
| 198 | """Populate the fields of a CmdRespMessage from a buffer.""" |
---|
| 199 | try: |
---|
| 200 | dataTuple = struct.unpack('!I 2H', data_buffer[0:8]) |
---|
| 201 | self.command = dataTuple[0] |
---|
| 202 | self.length = dataTuple[1] |
---|
| 203 | self.num_args = dataTuple[2] |
---|
| 204 | self.args = list(struct.unpack_from('!%dI' % self.num_args, |
---|
| 205 | data_buffer, offset=8)) |
---|
| 206 | self.raw_data = bytearray(data_buffer) |
---|
| 207 | |
---|
| 208 | # Store any raw payload following the response arguments |
---|
| 209 | # Payload (if any) starts at first byte after last resp argument |
---|
| 210 | payload_offset = 8 + 4*self.num_args |
---|
| 211 | |
---|
| 212 | if payload_offset < len(self.raw_data): |
---|
| 213 | self.resp_payload = self.raw_data[payload_offset:] |
---|
| 214 | else: |
---|
| 215 | self.resp_payload = bytearray() |
---|
| 216 | |
---|
| 217 | except struct.error as err: |
---|
| 218 | # Reset Cmd/Resp. We want predictable behavior on error |
---|
| 219 | self.reset() |
---|
| 220 | print("Error unpacking cmd/resp: {0}".format(err)) |
---|
| 221 | |
---|
| 222 | def get_bytes(self): |
---|
| 223 | """Returns the data buffer as bytes.""" |
---|
| 224 | return self.raw_data |
---|
| 225 | |
---|
| 226 | def sizeof(self): |
---|
| 227 | """Return the size of the cmd/resp including all attributes.""" |
---|
| 228 | if self.num_args == 0: |
---|
| 229 | return struct.calcsize('!I 2H') |
---|
| 230 | else: |
---|
| 231 | return struct.calcsize('!I 2H %dI' % self.num_args) |
---|
| 232 | |
---|
| 233 | def reset(self): |
---|
| 234 | """Reset the CmdRespMessage object to a default state (all zeros)""" |
---|
| 235 | self.command = 0 |
---|
| 236 | self.length = 0 |
---|
| 237 | self.num_args = 0 |
---|
| 238 | self.args = [] |
---|
| 239 | |
---|
| 240 | # End Class |
---|
| 241 | |
---|
| 242 | |
---|
| 243 | class Cmd(CmdRespMessage): |
---|
| 244 | """Base class for commands. |
---|
| 245 | |
---|
| 246 | Attributes: |
---|
| 247 | resp_type -- Response type of the command. See transport documentation |
---|
| 248 | for defined response types. By default, a Cmd will require a Resp |
---|
| 249 | as a response. |
---|
| 250 | |
---|
| 251 | See documentation of CmdRespMessage for additional attributes |
---|
| 252 | """ |
---|
| 253 | resp_type = None |
---|
| 254 | |
---|
| 255 | def __init__(self, command=0, length=0, num_args=0, |
---|
| 256 | args=None, resp_type=None): |
---|
| 257 | super(Cmd, self).__init__(command, length, num_args, args) |
---|
| 258 | self.resp_type = resp_type or transport.TRANSPORT_RESP |
---|
| 259 | |
---|
| 260 | def set_args(self, *args): |
---|
| 261 | """Set the command arguments.""" |
---|
| 262 | self.args = args |
---|
| 263 | self.num_args = len(args) |
---|
| 264 | self.length = self.num_args * 4 |
---|
| 265 | |
---|
| 266 | def add_args(self, *args): |
---|
| 267 | """Append arguments to current command argument list. |
---|
| 268 | |
---|
| 269 | Since the transport only operates on unsigned 32 bit integers, the |
---|
| 270 | command will convert all values to 32 bit unsigned integers. |
---|
| 271 | """ |
---|
| 272 | import ctypes |
---|
| 273 | |
---|
| 274 | for arg in args: |
---|
| 275 | if (arg < 0): |
---|
| 276 | arg = ctypes.c_uint32(arg).value |
---|
| 277 | self.args.append(arg) |
---|
| 278 | |
---|
| 279 | self.num_args += len(args) |
---|
| 280 | self.length += len(args) * 4 |
---|
| 281 | |
---|
| 282 | def set_payload(self, payload): |
---|
| 283 | """Adds an arbitrary payload to the message, typically a bytearray. The |
---|
| 284 | payload is appended after any arguments when the message is serialized |
---|
| 285 | for transmission over-the-wire. |
---|
| 286 | """ |
---|
| 287 | # This method sets the entire payload, possibly replacing an existing payload |
---|
| 288 | # First subtract the length of the existing payload, then set the new payload |
---|
| 289 | # and length values |
---|
| 290 | if self.payload is not None: |
---|
| 291 | self.length -= len(self.payload) |
---|
| 292 | |
---|
| 293 | self.payload = payload |
---|
| 294 | self.length += len(self.payload) |
---|
| 295 | |
---|
| 296 | def get_resp_type(self): |
---|
| 297 | return self.resp_type |
---|
| 298 | |
---|
| 299 | def process_resp(self, resp): |
---|
| 300 | """Process the response of the command.""" |
---|
| 301 | raise NotImplementedError |
---|
| 302 | |
---|
| 303 | def __str__(self): |
---|
| 304 | """Pretty print the Command""" |
---|
| 305 | msg = "" |
---|
| 306 | if self.command is not None: |
---|
| 307 | msg += "Command [{0:d}] ".format(self.command) |
---|
| 308 | msg += "({0:d} bytes): \n".format(self.length) |
---|
| 309 | |
---|
| 310 | if (self.num_args > 0): |
---|
| 311 | msg += " Args [0:{0:d}] : \n".format(self.num_args) |
---|
| 312 | for i in range(len(self.args)): |
---|
| 313 | msg += " 0x{0:08x}\n".format(self.args[i]) |
---|
| 314 | return msg |
---|
| 315 | |
---|
| 316 | # End Class |
---|
| 317 | |
---|
| 318 | |
---|
| 319 | class BufferCmd(CmdRespMessage): |
---|
| 320 | """Base Class for Buffer commands. |
---|
| 321 | |
---|
| 322 | Arguments: |
---|
| 323 | start_byte -- (uint32) Starting address of the buffer for this message. |
---|
| 324 | size -- (uint32) Size of the buffer in bytes |
---|
| 325 | - Reserved value: CMD_BUFFER_GET_SIZE_FROM_DATA |
---|
| 326 | |
---|
| 327 | Attributes: |
---|
| 328 | resp_type -- Response type of the command. See transport documentation |
---|
| 329 | for defined response types. By default, a BufferCmd will require |
---|
| 330 | a Buffer as a repsonse. |
---|
| 331 | |
---|
| 332 | Note: The wire format of a Buffer command is: |
---|
| 333 | Word[0] -- start_address of transfer |
---|
| 334 | Word[1] -- size of transfer (in bytes) |
---|
| 335 | Word[2 - N] -- Additional arguments |
---|
| 336 | |
---|
| 337 | To add additional arguments to a BufferCmd, use the add_args() method. |
---|
| 338 | |
---|
| 339 | See documentation of CmdRespMessage for additional attributes |
---|
| 340 | """ |
---|
| 341 | resp_type = None |
---|
| 342 | start_byte = None |
---|
| 343 | size = None |
---|
| 344 | |
---|
| 345 | def __init__(self, command=0, start_byte=0, size=0): |
---|
| 346 | super(BufferCmd, self).__init__(command=command, length=16, num_args=2, |
---|
| 347 | args=[start_byte, size]) |
---|
| 348 | |
---|
| 349 | self.resp_type = transport.TRANSPORT_BUFFER |
---|
| 350 | self.start_byte = start_byte |
---|
| 351 | self.timestamp_in_hdr = False |
---|
| 352 | |
---|
| 353 | if (size == CMD_BUFFER_GET_SIZE_FROM_DATA): |
---|
| 354 | self.size = 0 |
---|
| 355 | else: |
---|
| 356 | self.size = size |
---|
| 357 | |
---|
| 358 | def get_resp_type(self): return self.resp_type |
---|
| 359 | def get_buffer_start_byte(self): return self.start_byte |
---|
| 360 | def get_buffer_size(self): return self.size |
---|
| 361 | |
---|
| 362 | def update_start_byte(self, value): |
---|
| 363 | self.start_byte = value |
---|
| 364 | self.args[0] = value # FIXME: why self.args[0] not self.start_byte? |
---|
| 365 | |
---|
| 366 | def update_size(self, value): |
---|
| 367 | self.size = value |
---|
| 368 | self.args[1] = value # FIXME: why self.args[1] not self.size? |
---|
| 369 | |
---|
| 370 | def add_args(self, *args): |
---|
| 371 | """Append arguments to current command argument list. |
---|
| 372 | |
---|
| 373 | Since the transport only operates on unsigned 32 bit integers, the |
---|
| 374 | command will convert all values to 32 bit unsigned integers. |
---|
| 375 | """ |
---|
| 376 | import ctypes |
---|
| 377 | |
---|
| 378 | for arg in args: |
---|
| 379 | if (arg < 0): |
---|
| 380 | arg = ctypes.c_uint32(arg).value |
---|
| 381 | self.args.append(arg) |
---|
| 382 | |
---|
| 383 | self.num_args += len(args) |
---|
| 384 | self.length += len(args) * 4 |
---|
| 385 | |
---|
| 386 | def process_resp(self, resp): |
---|
| 387 | """Process the response of the command.""" |
---|
| 388 | raise NotImplementedError |
---|
| 389 | |
---|
| 390 | def __str__(self): |
---|
| 391 | """Pretty print the Command""" |
---|
| 392 | msg = "" |
---|
| 393 | if self.command is not None: |
---|
| 394 | msg += "Buffer Command [{0:d}] ".format(self.command) |
---|
| 395 | msg += "({0:d} bytes): \n".format(self.length) |
---|
| 396 | |
---|
| 397 | if (self.num_args > 0): |
---|
| 398 | msg += " Args [0:{0:d}] : \n".format(self.num_args) |
---|
| 399 | for i in range(len(self.args)): |
---|
| 400 | msg += " 0x{0:08x}\n".format(self.args[i]) |
---|
| 401 | return msg |
---|
| 402 | |
---|
| 403 | # End Class |
---|
| 404 | |
---|
| 405 | |
---|
| 406 | class Resp(CmdRespMessage): |
---|
| 407 | """Class for responses. |
---|
| 408 | |
---|
| 409 | This class is used if a command will return at most a single Ethernet |
---|
| 410 | packet of data from the node. |
---|
| 411 | |
---|
| 412 | See documentation of CmdRespMessage for attributes |
---|
| 413 | """ |
---|
| 414 | def get_args(self): |
---|
| 415 | """Return the response arguments.""" |
---|
| 416 | return self.args |
---|
| 417 | |
---|
| 418 | |
---|
| 419 | def resp_is_valid(self, num_args=None, status_errors=None, name=None): |
---|
| 420 | """Checks if the response is valid |
---|
| 421 | |
---|
| 422 | Args: |
---|
| 423 | num_args (int, optional): Check that the number of arguments matches the |
---|
| 424 | provided value |
---|
| 425 | status_errors (dict, optional): Dictionary of status errors for the form: |
---|
| 426 | { <status error value (int)> : <Error message (str)>} |
---|
| 427 | name (str, optional): Name of the command |
---|
| 428 | |
---|
| 429 | Returns: |
---|
| 430 | is_valid (bool): Is the response valid? |
---|
| 431 | """ |
---|
| 432 | error = False |
---|
| 433 | msg = "ERROR:\n" |
---|
| 434 | |
---|
| 435 | if num_args is not None: |
---|
| 436 | if len(self.args) != num_args: |
---|
| 437 | msg += " Number of arguments in response ({0}) does not match\n".format(len(self.args)) |
---|
| 438 | msg += " number of expected arguments ({0})\n".format(num_args) |
---|
| 439 | error = True |
---|
| 440 | |
---|
| 441 | if status_errors is not None: |
---|
| 442 | if self.args: |
---|
| 443 | if (self.args[0] in status_errors.keys()): |
---|
| 444 | if name is not None: |
---|
| 445 | msg += " Received status error {0}\n".format(name) |
---|
| 446 | else: |
---|
| 447 | msg += " Received status error\n" |
---|
| 448 | msg += " {0}\n".format(status_errors[self.args[0]]) |
---|
| 449 | error = True |
---|
| 450 | else: |
---|
| 451 | if name is not None: |
---|
| 452 | msg += " No status in response {0}\n".format(name) |
---|
| 453 | else: |
---|
| 454 | msg += " No status in response\n" |
---|
| 455 | error = True |
---|
| 456 | |
---|
| 457 | if error: |
---|
| 458 | print(msg) |
---|
| 459 | print(self) |
---|
| 460 | return False |
---|
| 461 | else: |
---|
| 462 | return True |
---|
| 463 | |
---|
| 464 | |
---|
| 465 | def __str__(self): |
---|
| 466 | """Pretty print the Response""" |
---|
| 467 | msg = "" |
---|
| 468 | if self.command is not None: |
---|
| 469 | msg += "Response [CMDID = 0x{0:08x}] ".format(self.command) |
---|
| 470 | msg += "({0:d} bytes): \n".format(self.length) |
---|
| 471 | |
---|
| 472 | if (self.num_args > 0): |
---|
| 473 | msg += " Args [0:{0:d}] : \n".format(self.num_args) |
---|
| 474 | for i in range(len(self.args)): |
---|
| 475 | msg += " 0x{0:08x}\n".format(self.args[i]) |
---|
| 476 | return msg |
---|
| 477 | |
---|
| 478 | # End Class |
---|
| 479 | |
---|
| 480 | |
---|
| 481 | class Buffer(Message): |
---|
| 482 | """Class for buffer for transferring generic information. |
---|
| 483 | |
---|
| 484 | This class is used if a command will return one or more Ethernet packets |
---|
| 485 | of data from the node. This object provides a container to transfer |
---|
| 486 | information that will be decoded by higher level functions. |
---|
| 487 | |
---|
| 488 | Attributes: |
---|
| 489 | complete -- Flag to indicate if buffer contains all of the bytes |
---|
| 490 | indicated by the size parameter |
---|
| 491 | start_byte -- Start byte of the buffer |
---|
| 492 | num_bytes -- Number of bytes currently contained within the buffer |
---|
| 493 | |
---|
| 494 | Wire Data Format: |
---|
| 495 | command -- (uint32) command / response ID |
---|
| 496 | length -- (uint16) Length of the cmd / resp args in bytes |
---|
| 497 | num_args -- (uint16) Number of uint32 arguments |
---|
| 498 | bytes_remainig -- (uint32) Number of bytes remain in the current request |
---|
| 499 | start_byte -- (uint32) Address of start byte for the transfer |
---|
| 500 | size -- (uint32) Size of the buffer in bytes |
---|
| 501 | buffer -- (list of uint8) Content of the buffer |
---|
| 502 | """ |
---|
| 503 | complete = None |
---|
| 504 | start_byte = None |
---|
| 505 | num_bytes = None |
---|
| 506 | tracker = None |
---|
| 507 | |
---|
| 508 | size = None |
---|
| 509 | buffer = None |
---|
| 510 | |
---|
| 511 | def __init__(self, start_byte=0, size=0, data_buffer=None): |
---|
| 512 | self.start_byte = start_byte |
---|
| 513 | self.size = size |
---|
| 514 | self.timestamp_in_hdr = False |
---|
| 515 | self.retrieval_timestamp = None |
---|
| 516 | |
---|
| 517 | self.tracker = [{0:start_byte, 1:start_byte, 2:0}] |
---|
| 518 | |
---|
| 519 | if data_buffer is not None: |
---|
| 520 | self._add_buffer_data(start_byte, data_buffer) |
---|
| 521 | else: |
---|
| 522 | # Create an empty buffer of the specified size |
---|
| 523 | self.complete = False |
---|
| 524 | self.num_bytes = 0 |
---|
| 525 | self.buffer = bytearray(self.size) |
---|
| 526 | |
---|
| 527 | |
---|
| 528 | def serialize(self): |
---|
| 529 | """Return a bytes object of a packed buffer.""" |
---|
| 530 | return self.serialize_cmd() |
---|
| 531 | |
---|
| 532 | |
---|
| 533 | def serialize_cmd(self, command=None, start_byte=None): |
---|
| 534 | """Return a bytes object of a packed buffer.""" |
---|
| 535 | if command is None: command = 0 |
---|
| 536 | if start_byte is None: start_byte = self.start_byte |
---|
| 537 | |
---|
| 538 | # FIXME: who calls this method? And why does this header |
---|
| 539 | # use the response header format (with 5I) instead of |
---|
| 540 | # the command header format (with 4I)? |
---|
| 541 | # FIXME 2: Updated format to remove buffer_id and flags, still |
---|
| 542 | # don't know who calls this method to construct a response header? |
---|
| 543 | return struct.pack('!I 2H 3I %dB' % self.size, |
---|
| 544 | command, 20, 3, # length = Num_args * 4 bytes / arg; Num_args = 3; |
---|
| 545 | 0, start_byte, |
---|
| 546 | self.size, *self.buffer) |
---|
| 547 | |
---|
| 548 | |
---|
| 549 | def deserialize(self, data_buffer): |
---|
| 550 | """Populate the fields of a Buffer with a message raw_data.""" |
---|
| 551 | (args, data) = self._unpack_data(data_buffer) |
---|
| 552 | |
---|
| 553 | bytes_remaining = args[0] |
---|
| 554 | start_byte = args[1] |
---|
| 555 | |
---|
| 556 | |
---|
| 557 | offset = (start_byte - self.start_byte) |
---|
| 558 | |
---|
| 559 | self._update_buffer_size(bytes_remaining) |
---|
| 560 | self._add_buffer_data(offset, data) |
---|
| 561 | self._set_buffer_complete() |
---|
| 562 | |
---|
| 563 | |
---|
| 564 | def add_data_to_buffer(self, raw_data): |
---|
| 565 | """Add the raw data (with the format of a Buffer) to the current |
---|
| 566 | Buffer. |
---|
| 567 | |
---|
| 568 | Note: This will check to make sure that data is for the given buffer |
---|
| 569 | as well as place it in the appropriate place indicated by the |
---|
| 570 | start_byte. |
---|
| 571 | """ |
---|
| 572 | (args, data) = self._unpack_data(raw_data) |
---|
| 573 | |
---|
| 574 | bytes_remaining = args[0] |
---|
| 575 | start_byte = args[1] |
---|
| 576 | |
---|
| 577 | # Old code checked args.buffer_id == self.buffer_id |
---|
| 578 | # This check never failed in old code |
---|
| 579 | # This is where new code could verify req_id? |
---|
| 580 | offset = (start_byte - self.start_byte) |
---|
| 581 | |
---|
| 582 | self._update_buffer_size(bytes_remaining) |
---|
| 583 | self._add_buffer_data(offset, data) |
---|
| 584 | self._set_buffer_complete() |
---|
| 585 | |
---|
| 586 | |
---|
| 587 | def append(self, buffer): |
---|
| 588 | """Append the contents of the provided Buffer to the current Buffer.""" |
---|
| 589 | curr_size = self.size |
---|
| 590 | new_size = curr_size + buffer.get_buffer_size() |
---|
| 591 | |
---|
| 592 | self._update_buffer_size(new_size, force=1) |
---|
| 593 | self._add_buffer_data(curr_size, buffer.get_bytes()) |
---|
| 594 | self._set_buffer_complete() |
---|
| 595 | |
---|
| 596 | |
---|
| 597 | def merge(self, buffer): |
---|
| 598 | """Merge the contents of the provided Buffer to the current Buffer.""" |
---|
| 599 | start_byte = buffer.get_start_byte() |
---|
| 600 | offset = (start_byte - self.start_byte) |
---|
| 601 | size = buffer.get_buffer_size() |
---|
| 602 | end_byte = offset + size |
---|
| 603 | |
---|
| 604 | if (end_byte > self.size): |
---|
| 605 | # Need to update the buffer to allocate more memory first |
---|
| 606 | self._update_buffer_size(end_byte, force=1) |
---|
| 607 | |
---|
| 608 | self._add_buffer_data(offset, buffer.get_bytes()) |
---|
| 609 | self._set_buffer_complete() |
---|
| 610 | |
---|
| 611 | |
---|
| 612 | |
---|
| 613 | def trim(self): |
---|
| 614 | """Trim the Buffer to the largest contiguous number of bytes received.""" |
---|
| 615 | locations = self.get_missing_byte_locations() |
---|
| 616 | |
---|
| 617 | if locations: |
---|
| 618 | # This assumes that the missing byte locations are in order |
---|
| 619 | # with the first missing byte after the start byte in the first |
---|
| 620 | # item in the list. |
---|
| 621 | missing_start = locations[0][0] |
---|
| 622 | contiguous_size = missing_start - self.start_byte |
---|
| 623 | |
---|
| 624 | self.num_bytes = contiguous_size |
---|
| 625 | self._update_buffer_size(contiguous_size, force=1) |
---|
| 626 | self._set_buffer_complete() |
---|
| 627 | |
---|
| 628 | |
---|
| 629 | def sizeof(self): |
---|
| 630 | """Return the size of the Buffer including all attributes.""" |
---|
| 631 | # Do not calculate the size of the buffer just using calcsize. |
---|
| 632 | # This is extremely memory inefficient for large buffers and can |
---|
| 633 | # cause memory issues. |
---|
| 634 | # 3I is format of list/log retrieval header (3 u32 args) |
---|
| 635 | return (struct.calcsize('!3I') + self.size) |
---|
| 636 | |
---|
| 637 | def get_start_byte(self): return self.start_byte |
---|
| 638 | def get_header_size(self): return struct.calcsize('!3I') |
---|
| 639 | def get_buffer_size(self): return self.size |
---|
| 640 | def get_occupancy(self): return self.num_bytes |
---|
| 641 | |
---|
| 642 | def set_bytes(self, data_buffer): |
---|
| 643 | """Set the message bytes of the Buffer.""" |
---|
| 644 | self._update_buffer_size(len(data_buffer), force=1) |
---|
| 645 | self._add_buffer_data(0, data_buffer) |
---|
| 646 | self._set_buffer_complete() |
---|
| 647 | |
---|
| 648 | def get_bytes(self): |
---|
| 649 | """Return the message bytes of the Buffer.""" |
---|
| 650 | return self.buffer |
---|
| 651 | |
---|
| 652 | def get_missing_byte_locations(self): |
---|
| 653 | """Returns a list of tuples (start_index, end_index, size) that |
---|
| 654 | contain the missing byte locations. |
---|
| 655 | """ |
---|
| 656 | if not self.complete: |
---|
| 657 | return self._find_missing_bytes() |
---|
| 658 | else: |
---|
| 659 | return [] |
---|
| 660 | |
---|
| 661 | def is_buffer_complete(self): |
---|
| 662 | """Return if the Buffer is complete.""" |
---|
| 663 | return self.complete |
---|
| 664 | |
---|
| 665 | def reset(self): |
---|
| 666 | """Reset the Buffer object to a default state (all zeros)""" |
---|
| 667 | self.buffer_id = 0 |
---|
| 668 | self.flags = 0 |
---|
| 669 | self.size = 0 |
---|
| 670 | self.buffer = bytearray(self.size) |
---|
| 671 | |
---|
| 672 | def __str__(self): |
---|
| 673 | """Pretty print the Buffer""" |
---|
| 674 | msg = "" |
---|
| 675 | if self.buffer is not None: |
---|
| 676 | msg += "Buffer [{0:d}] ".format(self.buffer_id) |
---|
| 677 | msg += "({0:d} bytes): \n".format(self.size) |
---|
| 678 | msg += " Flags : 0x{0:08x} \n".format(self.flags) |
---|
| 679 | msg += " Start : {0:d}\n".format(self.start_byte) |
---|
| 680 | msg += " Num bytes: {0:d}\n".format(self.num_bytes) |
---|
| 681 | msg += " Complete : {0}\n".format(self.complete) |
---|
| 682 | |
---|
| 683 | if (False): |
---|
| 684 | msg += " Data : " |
---|
| 685 | for i in range(len(self.buffer)): |
---|
| 686 | if (i % 16) == 0: |
---|
| 687 | msg += "\n {0:02x} ".format(self.buffer[i]) |
---|
| 688 | else: |
---|
| 689 | msg += "{0:02x} ".format(self.buffer[i]) |
---|
| 690 | return msg |
---|
| 691 | |
---|
| 692 | |
---|
| 693 | # ------------------------------------------------------------------------- |
---|
| 694 | # Internal helper methods |
---|
| 695 | # ------------------------------------------------------------------------- |
---|
| 696 | def _unpack_data(self, raw_data): |
---|
| 697 | """Internal method to unpack a data buffer.""" |
---|
| 698 | args = [] |
---|
| 699 | data = [] |
---|
| 700 | if 1: |
---|
| 701 | # First unpack just the cmd_resp header |
---|
| 702 | |
---|
| 703 | cmd_resp_hdr = CmdRespMessage() |
---|
| 704 | cmd_resp_hdr.deserialize(raw_data) |
---|
| 705 | args = cmd_resp_hdr.args |
---|
| 706 | |
---|
| 707 | # List and log retrieval responses set num_args=0 when returning |
---|
| 708 | # an empty list/payload |
---|
| 709 | if args: |
---|
| 710 | if self.timestamp_in_hdr: |
---|
| 711 | # List retrieval responses include timestamp |
---|
| 712 | list_ret_args = struct.unpack('!3I Q', raw_data[8:28]) |
---|
| 713 | size = args[2] |
---|
| 714 | self.retrieval_timestamp = args[3] |
---|
| 715 | data = struct.unpack_from('!%dB' % size, raw_data, offset=28) |
---|
| 716 | args += list_ret_args |
---|
| 717 | |
---|
| 718 | else: |
---|
| 719 | # Log retrieval response does not include timestamp |
---|
| 720 | log_ret_args = struct.unpack('!3I', raw_data[8:20]) |
---|
| 721 | size = args[2] |
---|
| 722 | data = struct.unpack_from('!%dB' % size, raw_data, offset=20) |
---|
| 723 | args += log_ret_args |
---|
| 724 | else: |
---|
| 725 | # Node returned empty list/log buffer |
---|
| 726 | # Construct "empty" response args to return |
---|
| 727 | # 0: bytes_remaining |
---|
| 728 | # 1: start_byte |
---|
| 729 | # 2: size |
---|
| 730 | args = (0, 0, 0) |
---|
| 731 | data = [] |
---|
| 732 | |
---|
| 733 | #except struct.error as err: |
---|
| 734 | # # Ignore the data. We want predictable behavior on error |
---|
| 735 | # print("Error unpacking buffer:") |
---|
| 736 | # print(" {0}".format(err)) |
---|
| 737 | # print(" Ignorning data. This error could be caused by a mismatch between\n") |
---|
| 738 | # print(" the maximum packet size between the node and the host.\n") |
---|
| 739 | |
---|
| 740 | return (args, data) |
---|
| 741 | |
---|
| 742 | def _update_buffer_size(self, size, force=0): |
---|
| 743 | """Internal method to update the size of the transfer.""" |
---|
| 744 | if (self.size == 0): |
---|
| 745 | self.size = size |
---|
| 746 | self.buffer = bytearray(self.size) |
---|
| 747 | elif (force == 1): |
---|
| 748 | # Update the size of the buffer |
---|
| 749 | old_size = self.size |
---|
| 750 | self.size = size |
---|
| 751 | |
---|
| 752 | # Update the buffer allocation |
---|
| 753 | if (size > old_size): |
---|
| 754 | self.buffer.extend(bytearray(size - old_size)) |
---|
| 755 | else: |
---|
| 756 | self.buffer = self.buffer[:size] |
---|
| 757 | |
---|
| 758 | |
---|
| 759 | def _add_buffer_data(self, buffer_offset, data_buffer): |
---|
| 760 | """Internal method to add data to the Buffer |
---|
| 761 | |
---|
| 762 | Only self.size bytes were allocated for the Buffer. Therefore, |
---|
| 763 | only take an offset from the start_byte (ie a relative address) |
---|
| 764 | for where to store the data in the Buffer. |
---|
| 765 | |
---|
| 766 | If the provided data is greater than specified Buffer size, then the |
---|
| 767 | data will be truncated. |
---|
| 768 | """ |
---|
| 769 | data_to_add_size = len(data_buffer) |
---|
| 770 | buffer_end_byte = buffer_offset + data_to_add_size |
---|
| 771 | |
---|
| 772 | # If the buffer size will be exceeded, then truncate the add |
---|
| 773 | if (buffer_end_byte > self.size): |
---|
| 774 | buffer_end_byte = self.size |
---|
| 775 | data_to_add_size = buffer_end_byte - buffer_offset |
---|
| 776 | |
---|
| 777 | # Update the tracker with the information |
---|
| 778 | # - Need to convert back to absolute addresses for tracker |
---|
| 779 | self._update_tracker((buffer_offset + self.start_byte), (buffer_end_byte + self.start_byte), data_to_add_size) |
---|
| 780 | |
---|
| 781 | # Add the data to the buffer |
---|
| 782 | if (data_to_add_size > 0): |
---|
| 783 | self.buffer[buffer_offset:buffer_end_byte] = data_buffer[:data_to_add_size] |
---|
| 784 | |
---|
| 785 | # Update the ocupancy of the buffer |
---|
| 786 | self.num_bytes = self._tracker_size() |
---|
| 787 | |
---|
| 788 | # Set the buffer complete flag |
---|
| 789 | self._set_buffer_complete() |
---|
| 790 | |
---|
| 791 | |
---|
| 792 | def _set_buffer_complete(self): |
---|
| 793 | """Internal method to set the complete flag on the Buffer.""" |
---|
| 794 | if (self.num_bytes == self.size): |
---|
| 795 | self.complete = True |
---|
| 796 | elif (self.num_bytes < self.size): |
---|
| 797 | self.complete = False |
---|
| 798 | else: |
---|
| 799 | print("WARNING: Buffer out of sync. Should never reach here.") |
---|
| 800 | |
---|
| 801 | |
---|
| 802 | def _tracker_size(self): |
---|
| 803 | """Internal method to get the Buffer size via the tracker.""" |
---|
| 804 | size = 0 |
---|
| 805 | for item in self.tracker: |
---|
| 806 | size += item[2] |
---|
| 807 | |
---|
| 808 | return size |
---|
| 809 | |
---|
| 810 | |
---|
| 811 | def _update_tracker(self, start_byte, end_byte, size): |
---|
| 812 | """Internal method to update the tracker.""" |
---|
| 813 | done = False |
---|
| 814 | |
---|
| 815 | # Don't add duplicate entries |
---|
| 816 | for item in self.tracker: |
---|
| 817 | if (start_byte == item[0]) and (end_byte == item[1]) and (size == item[2]): |
---|
| 818 | return |
---|
| 819 | |
---|
| 820 | # Can this update be added to one of the current tracker entries |
---|
| 821 | for item in self.tracker: |
---|
| 822 | if (start_byte == item[1]): |
---|
| 823 | item[1] += size |
---|
| 824 | item[2] += size |
---|
| 825 | done = True |
---|
| 826 | |
---|
| 827 | if not done: |
---|
| 828 | self.tracker.append({0:start_byte, 1:end_byte, 2:size}) |
---|
| 829 | |
---|
| 830 | # Try to compress the tracker |
---|
| 831 | self._compress_tracker() |
---|
| 832 | |
---|
| 833 | |
---|
| 834 | def _compress_tracker(self): |
---|
| 835 | """Internal method to compress the tracker.""" |
---|
| 836 | # If there is more than one item, try to compress them |
---|
| 837 | if (len(self.tracker) > 1): |
---|
| 838 | tracker = [] |
---|
| 839 | tmp_tracker = sorted(self.tracker, key=lambda k: k[0]) |
---|
| 840 | |
---|
| 841 | tmp_item = tmp_tracker[0] |
---|
| 842 | tracker.append(tmp_item) |
---|
| 843 | |
---|
| 844 | # For each remaining item if the start_byte equals the end_byte |
---|
| 845 | # of the start_item, then merge the items |
---|
| 846 | for item in tmp_tracker[1:]: |
---|
| 847 | if (item[0] == tmp_item[1]): |
---|
| 848 | tmp_item[1] = item[1] |
---|
| 849 | tmp_item[2] += item[2] |
---|
| 850 | else: |
---|
| 851 | tmp_item = item |
---|
| 852 | tracker.append(tmp_item) |
---|
| 853 | |
---|
| 854 | self.tracker = tracker |
---|
| 855 | |
---|
| 856 | |
---|
| 857 | def _find_missing_bytes(self): |
---|
| 858 | """Internal method to find the missing bytes using the tracker.""" |
---|
| 859 | ret_val = [] |
---|
| 860 | missing_bytes = self.size - self.num_bytes |
---|
| 861 | start = self.start_byte |
---|
| 862 | end = self.start_byte + self.size |
---|
| 863 | tmp_tracker = sorted(self.tracker, key=lambda k: k[0]) |
---|
| 864 | tracker_count = list(tmp_tracker) |
---|
| 865 | error = False |
---|
| 866 | |
---|
| 867 | if (missing_bytes != 0): |
---|
| 868 | # Iterate through all the items in the list and remove |
---|
| 869 | # them to build up the holes |
---|
| 870 | for item in tmp_tracker: |
---|
| 871 | |
---|
| 872 | # Process item but don't add a hole |
---|
| 873 | if (start == item[0]): |
---|
| 874 | start = item[1] |
---|
| 875 | tracker_count.remove(item) |
---|
| 876 | |
---|
| 877 | # There is a missing piece of the buffer to request |
---|
| 878 | elif ((start + missing_bytes) >= item[0]): |
---|
| 879 | tmp_size = item[0] - start |
---|
| 880 | |
---|
| 881 | if tmp_size < 0: |
---|
| 882 | print("WARNING: Issue with finding missing bytes.") |
---|
| 883 | print(" Size between items is negative.") |
---|
| 884 | print(" item : ({0}, {1}, {2})".format(item[0], item[1], item[2])) |
---|
| 885 | print(" start : {0}".format(start)) |
---|
| 886 | error = True |
---|
| 887 | else: |
---|
| 888 | missing_bytes -= tmp_size |
---|
| 889 | ret_val.append((start, item[0], tmp_size)) |
---|
| 890 | start = item[1] |
---|
| 891 | |
---|
| 892 | tracker_count.remove(item) |
---|
| 893 | |
---|
| 894 | # There was an error in the tracker |
---|
| 895 | else: |
---|
| 896 | print("WARNING: Issue with tracking missing bytes.") |
---|
| 897 | print(" Number of missing bytes does not cover hole between tracker items.") |
---|
| 898 | print(" Missing Bytes : {0}".format(missing_bytes)) |
---|
| 899 | |
---|
| 900 | tmp_size = item[0] - start |
---|
| 901 | missing_bytes = 0 |
---|
| 902 | ret_val.append((start, item[0], tmp_size)) |
---|
| 903 | start = item[1] |
---|
| 904 | tracker_count.remove(item) |
---|
| 905 | error = True |
---|
| 906 | |
---|
| 907 | if tracker_count: |
---|
| 908 | print("WARNING: Issue with finding missing bytes.") |
---|
| 909 | print(" Not all tracker items processed.") |
---|
| 910 | error = True |
---|
| 911 | |
---|
| 912 | if error: |
---|
| 913 | print(" Tracker : {0}".format(self.tracker)) |
---|
| 914 | print(" Tmp Tracker : {0}".format(tmp_tracker)) |
---|
| 915 | print(" Remaining Items : {0}".format(tracker_count)) |
---|
| 916 | print(" Locations : {0}".format(ret_val)) |
---|
| 917 | |
---|
| 918 | # Find any holes at the end of the buffer |
---|
| 919 | if (missing_bytes != 0): |
---|
| 920 | if (end != start): |
---|
| 921 | tmp_size = end - start |
---|
| 922 | ret_val.append((start, end, tmp_size)) |
---|
| 923 | missing_bytes -= tmp_size |
---|
| 924 | |
---|
| 925 | # Missing bytes, so print a warning |
---|
| 926 | if (missing_bytes != 0): |
---|
| 927 | print("WARNING: Could not find all missing bytes: {0}".format(missing_bytes)) |
---|
| 928 | |
---|
| 929 | return ret_val |
---|
| 930 | |
---|
| 931 | |
---|
| 932 | # End Class |
---|
| 933 | |
---|
| 934 | |
---|
| 935 | |
---|
| 936 | |
---|