[chirp_devel] [PATCH] [New Model] Support for the BTECH Mobile Radios, update 1 for #3015
# HG changeset patch # User Jim Unroe rock.unroe@gmail.com # Date 1459036642 14400 # Node ID be962502a6c19e3a09de92ff7bdcded96e2c90a2 # Parent 1dea27fb59f7adefe00398d8e5648120cddc8e12 [New Model] Support for the BTECH Mobile Radios, update 1 for #3015
This patch includes the following changes:
Changes related to developer mailing list discussions - Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one - Removed our software serial timeout in favor of the system serial timeout - Removed the reference of the second magic for the alpha radio - Renamed the _do_magic() def - Restructured the "super nested" code to a less nested one - Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout - Removed/changed some of the LOG.xxx() - Removed the exception that goes to the UI about maxtime - Renamed the base class as BTech (from btech)
Bug fixes - Fixed the bug found by minuteman1120 about the duplex not working in some cases - New "split" algorithm for the duplex bug above - Fixed a bug with the scode about 0x*F reseted to 0x*0
New - Added support for the QYT KT-UV980 Radio
also related to #2673
diff -r 1dea27fb59f7 -r be962502a6c1 chirp/drivers/btech.py --- a/chirp/drivers/btech.py Tue Mar 22 18:50:50 2016 -0700 +++ b/chirp/drivers/btech.py Sat Mar 26 19:57:22 2016 -0400 @@ -18,6 +18,7 @@ import time import struct import logging +import sys
LOG = logging.getLogger(__name__)
@@ -160,6 +161,9 @@ MINI8900_fp = "M28854"
+# QYT KT UV-890 +KTUV890_fp = "H28854" + #### MAGICS # for the Waccom Mini-8900 MSTRING_MINI8900 = "\x55\xA5\xB5\x45\x55\x45\x4d\x02" @@ -169,37 +173,41 @@ MSTRING = "\x55\x20\x15\x09\x20\x45\x4d\x02"
+def _clean_buffer(radio): + """Cleaning the read serial buffer""" + radio.pipe.setTimeout(0.1) + + try: + dump = radio.pipe.read(100) + + except Exception: + raise errors.RadioError("Unknown error cleaning the serial buffer") + + def _rawrecv(radio, amount): - """Raw read from the radio device, new approach, this time a byte at - a time as the original driver, the receive data has to be atomic""" + """Raw read from the radio device, less intensive way""" + data = ""
try: - tdiff = 0 - start = time.time() - maxtime = amount * 0.020 - - while len(data) < amount and tdiff < maxtime: - d = radio.pipe.read(1) - if len(d) == 1: - data += d - - # Delta time - tdiff = time.time() - start - - # DEBUG - if debug is True: - LOG.debug("time diff %.04f maxtime %.04f, data: %d" % - (tdiff, maxtime, len(data))) + # Getting the data with a dynamic timeout in the serial deppending on + # the amount of data we need + timeout = amount * 0.06 + radio.pipe.setTimeout(timeout) + data = radio.pipe.read(amount)
# DEBUG if debug is True: LOG.debug("<== (%d) bytes:\n\n%s" % (len(data), util.hexprint(data)))
+ # fail if no data is received + if len(data) == 0: + raise errors.RadioError("No data received from radio") + if len(data) < amount: - LOG.error("Short reading %d bytes from the %d requested." % - (len(data), amount)) + LOG.warn("Short reading %d bytes from the %d requested." % + (len(data), amount))
except: raise errors.RadioError("Error reading data from radio") @@ -212,7 +220,6 @@ try: for byte in data: radio.pipe.write(byte) - time.sleep(0.003)
# DEBUG if debug is True: @@ -242,123 +249,102 @@
def _recv(radio, addr): - """Get data from the radio """ + """Get data from the radio all at once to lower syscalls load""" + # Get the full 69 bytes at a time to reduce load + # # 1 byte ACK + # 4 bytes header + - # data of length of data (as I see always 0x40 = 64 bytes) + # 64 bytes of data (BLOCK_SIZE)
- # catching ack - ack = _rawrecv(radio, 1) + # get the whole block + block = _rawrecv(radio, BLOCK_SIZE + 5)
- # checking for a response - if len(ack) != 1: - msg = "No response in the read of the block #0x%04x" % addr - LOG.error(msg) - raise errors.RadioError(msg) + # basic check + if len(block) < (BLOCK_SIZE + 5): + raise errors.RadioError("Short read of the block 0x%04x" % addr)
- # valid data + # checking for the ack + ack = block[0] if ack != ACK_CMD: - msg = "Bad ack received from radio in block 0x%04x" % addr - LOG.error(msg) - LOG.debug("Bad ACK was 0x%02x" % ord(ack)) - raise errors.RadioError(msg) + raise errors.RadioError("Bad ack from radio in block 0x%04x" % addr)
- # Get the header + basic sanitize - hdr = _rawrecv(radio, 4) - if len(hdr) != 4: - msg = "Short header for block: 0x%04x" % addr - LOG.error(msg) - raise errors.RadioError(msg) - - # receive and validate the header + # header validation + hdr = block[1:5] c, a, l = struct.unpack(">BHB", hdr) if a != addr or l != BLOCK_SIZE or c != ord("X"): - msg = "Invalid answer for block 0x%04x:" % addr - LOG.error(msg) + LOG.debug("Invalid header for block 0x%04x" % addr) LOG.debug("CMD: %s ADDR: %04x SIZE: %02x" % (c, a, l)) - raise errors.RadioError(msg) + raise errors.RadioError("Invalid header for block 0x%04x:" % addr)
# Get the data - data = _rawrecv(radio, l) - - # basic validation - if len(data) != l: - msg = "Short block of data in block #0x%04x" % addr - LOG.error(msg) - raise errors.RadioError(msg) - + data = block[5:] return data
-def _do_magic(radio, status): - """Try to put the radio in program mode and get the ident string - it will make multiple tries""" +def _start_clone_mode(radio, status): + """Put the radio in clone mode and get the ident string + using multiple tries"""
# how many tries - tries = 5 + tries = 3
# prep the data to show in the UI status.cur = 0 status.msg = "Identifying the radio..." - status.max = len(radio._magic) * tries + status.max = tries * len(radio._magic) radio.status_fn(status) mc = 0 + radio.pipe.setTimeout(0.6) + + def send_magic_word(m, magic): + """Send the magic word to the radio, catching the answer""" + + # cleaning the buffer and set timeout + _clean_buffer(radio) + + for a in range(tries): + # Update the UI + status.cur = m * a + a + radio.status_fn(status) + + # send the magic word + _send(radio, magic) + + # Now you get a x06 of ACK if all goes well + ack = radio.pipe.read(1) + + if ack == "\x06": + # DEBUG + LOG.info("Magic ACK received") + status.msg = "Positive Ident!" + status.cur = status.max + radio.status_fn(status) + + return True + + # if the magic word don't open the gate we must inform it + status.cur = status.max + radio.status_fn(status) + return False
try: - # do the magic + # try to put the radio in clone mode for magic in radio._magic: - # we try a few times - for a in range(0, tries): - # Update the UI - status.cur = (mc * tries) + a - radio.status_fn(status) - - # cleaning the serial buffer, try wrapped - try: - radio.pipe.flushInput() - except: - msg = "Error with a serial rx buffer flush at _do_magic" - LOG.error(msg) - raise errors.RadioError(msg) - - # send the magic a byte at a time - for byte in magic: - ack = _rawrecv(radio, 1) - _send(radio, byte) - - # A explicit time delay, with a longer one for the UV-5001 - if "5001" in radio.MODEL: - time.sleep(0.5) - else: - time.sleep(0.1) - - # Now you get a x06 of ACK if all goes well - ack = _rawrecv(radio, 1) - - if ack == "\x06": - # DEBUG - LOG.info("Magic ACK received") - status.msg = "Positive Ident!" - status.cur = status.max - radio.status_fn(status) - - return True + # send the magic word and check for a valid answer + r = send_magic_word(mc, magic) + if r is True: + return True
# increment the count of magics to send, this is for the UI status mc += 1
- # wait between tries for different MAGICs to allow the radio to - # timeout, this is an experimental fature for the 5001 alpha that - # has the same ident as the MINI8900, raise it if it don't work - time.sleep(5) + # if you get here if that it don't worked + return False
except errors.RadioError: raise except Exception, e: - msg = "Unknown error sending Magic to radio:\n%s" % e - raise errors.RadioError(msg) - - return False + raise errors.RadioError("Error sending Magic to radio:\n%s" % e)
def _do_ident(radio, status): @@ -366,28 +352,21 @@ # set the serial discipline radio.pipe.setBaudrate(9600) radio.pipe.setParity("N") - radio.pipe.setTimeout(0.005) - # cleaning the serial buffer, try wrapped - try: - radio.pipe.flushInput() - except: - msg = "Error with a serial rx buffer flush at _do_ident" - LOG.error(msg) - raise errors.RadioError(msg) + + # cleaning the buffers + _clean_buffer(radio)
# do the magic trick - if _do_magic(radio, status) is False: + if _start_clone_mode(radio, status) is False: msg = "Radio did not respond to magic string, check your cable." - LOG.error(msg) raise errors.RadioError(msg)
# Ok, get the ident string ident = _rawrecv(radio, 49)
# basic check for the ident - if len(ident) != 49: - msg = "Radio send a sort ident block, you need to increase maxtime." - LOG.error(msg) + if len(ident) < 49: + msg = "Radio sent a sort ident block, aborting" raise errors.RadioError(msg)
# check if ident is OK @@ -402,7 +381,9 @@ msg = "Incorrect model ID, got this:\n\n" msg += util.hexprint(ident) LOG.debug(msg) - raise errors.RadioError("Radio identification failed.") + error = "Radio identification failed, this is not the correct model, " + error += "or it's a not supported variant yet." + raise errors.RadioError()
# DEBUG LOG.info("Positive ident, this is a %s" % radio.MODEL) @@ -424,11 +405,9 @@ # we just care about the first 16, our magic string is in there if len(id2) < 16: msg = "The extra UV-2501+220 ID is short, aborting." - # DEBUG - LOG.error(msg) raise errors.RadioError(msg)
- # ok, check for it, any of the correct ID must be in the received data + # ok, check for it, any of the correct IDs must be in the received data itis = False for eid in radio._id2: if eid in id2: @@ -442,7 +421,6 @@ if itis is False: msg = "The extra UV-2501+220 ID is wrong, aborting." # DEBUG - LOG.error(msg) LOG.debug("Full extra ID on the 2501+220 is: \n%s" % util.hexprint(id2)) raise errors.RadioError(msg) @@ -476,16 +454,11 @@ status.cur = 0 radio.status_fn(status)
+ # clean serial buffers + _clean_buffer(radio) + data = "" for addr in range(0, MEM_SIZE, BLOCK_SIZE): - # flush input, as per the original driver behavior, try wrapped - try: - radio.pipe.flushInput() - except: - msg = "Error with a serial rx buffer flush at _download" - LOG.error(msg) - raise errors.RadioError(msg) - # sending the read request _send(radio, _make_frame("S", addr, BLOCK_SIZE), 0.1)
@@ -525,19 +498,22 @@ status.msg = "Cloning to radio..." radio.status_fn(status)
+ # clean buffers + _clean_buffer(radio) + + # set a delay based on the OS + if sys.platform in ["win32", "cygwin"]: + # we are in windows + delay = 0.015 + else: + # we are in linux/mac + delay = 0.033 + # the fun start here for addr in range(0, MEM_SIZE, TX_BLOCK_SIZE): - # flush input, as per the original driver behavior, try wrapped - try: - radio.pipe.flushInput() - except: - msg = "Error with a serial rx buffer flush at _upload" - LOG.error(msg) - raise errors.RadioError(msg) - # sending the data d = data[addr:addr + TX_BLOCK_SIZE] - _send(radio, _make_frame("X", addr, TX_BLOCK_SIZE, d), 0.015) + _send(radio, _make_frame("X", addr, TX_BLOCK_SIZE, d), delay)
# receiving the response ack = _rawrecv(radio, 1) @@ -580,7 +556,22 @@ return (ilow, ihigh)
-class btech(chirp_common.CloneModeRadio, chirp_common.ExperimentalRadio): +def _split(rf, f1, f2): + """Returns False if the two freqs are in the same band (no split) + or True otherwise""" + + # determine if the two freqs are in the same band + for low, high in rf.valid_bands: + if f1 >= low and f1 <= high and \ + f2 >= low and f2 <= high: + # if the two freqs are on the same Band this is not a split + return False + + # if you get here is because the freq pairs are split + return False + + +class BTech(chirp_common.CloneModeRadio, chirp_common.ExperimentalRadio): """BTECH's UV-5001 and alike radios""" VENDOR = "BTECH" MODEL = "" @@ -697,7 +688,7 @@ ranges"""
# setting the correct ranges for each radio type - if self.MODEL == "UV-2501+220": + if "+220" in self.MODEL: # the model 2501+220 has a segment in 220 # and a different position in the memmap ranges = self._memobj.ranges220 @@ -713,7 +704,7 @@ LOG.info("Radio ranges: UHF %d to %d" % uhf)
# 220Mhz case - if self.MODEL == "UV-2501+220": + if "+220" in self.MODEL: vhf2 = _decode_ranges(ranges.vhf2_low, ranges.vhf2_high) LOG.info("Radio ranges: VHF(220) %d to %d" % vhf2) self._220_range = vhf2 @@ -805,7 +796,7 @@ # TX freq set offset = (int(_mem.txfreq) * 10) - mem.freq if offset != 0: - if offset > 70000000: # 70 Mhz + if _split(self.get_features(), mem.freq, int(_mem.txfreq) * 10): mem.duplex = "split" mem.offset = int(_mem.txfreq) * 10 elif offset < 0: @@ -855,10 +846,12 @@ PTTID_LIST[_mem.pttid])) mem.extra.append(pttid)
+ # validating scode + scode = _mem.scode if _mem.scode != 15 else 0 pttidcode = RadioSetting("scode", "PTT ID signal code", RadioSettingValueList( PTTIDCODE_LIST, - PTTIDCODE_LIST[_mem.scode])) + PTTIDCODE_LIST[scode])) mem.extra.append(pttidcode)
optsig = RadioSetting("optsig", "Optional signaling", @@ -963,16 +956,18 @@ return False
-# Note: -# the order in the lists in the _magic, IDENT and _fileid is important -# we put the most common units first, the policy is as follows: - -# - First latest (newer) units, as they will be the most common -# - Second the former latest version, and recursively... -# - At the end the pre-production units (pp) as this will be unique +# =========================================================================== +# The order in the _magic list, is important if it's not bigger than two. +# If we found a situation where the count of magics for a single model is +# more than two we need to make a new radio model definition as "Gen 2 v2" +# or something like that. +# +# This has a time impact on the _start_clone_mode() with 5 secs of pause for +# each magic to sent, so if this list get bigger than two the user has to wait +# for at least 15 seconds to the start of the real clone mode.
@directory.register -class UV2501(btech): +class UV2501(BTech): """Baofeng Tech UV2501""" MODEL = "UV-2501" _magic = [MSTRING, ] @@ -980,7 +975,7 @@
@directory.register -class UV2501_220(btech): +class UV2501_220(BTech): """Baofeng Tech UV2501+220""" MODEL = "UV-2501+220" _magic = [MSTRING_220, ] @@ -989,17 +984,28 @@
@directory.register -class UV5001(btech): +class UV5001(BTech): """Baofeng Tech UV5001""" MODEL = "UV-5001" - _magic = [MSTRING, MSTRING_MINI8900] + _magic = [MSTRING, ] _fileid = [UV5001G22_fp, UV5001G2_fp, UV5001alpha_fp, UV5001pp_fp]
@directory.register -class MINI8900(btech): +class MINI8900(BTech): """WACCOM MINI-8900""" VENDOR = "WACCOM" MODEL = "MINI-8900" _magic = [MSTRING_MINI8900, ] _fileid = [MINI8900_fp, ] + + +@directory.register +class KTUV980(BTech): + """QYT KT-UV980""" + VENDOR = "QYT" + MODEL = "KT-UV980" + _vhf_range = (136000000, 175000000) + _uhf_range = (400000000, 481000000) + _magic = [MSTRING_MINI8900, ] + _fileid = [KTUV890_fp, ]
This patch includes the following changes:
Changes related to developer mailing list discussions
- Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one
- Removed our software serial timeout in favor of the system serial timeout
- Removed the reference of the second magic for the alpha radio
- Renamed the _do_magic() def
- Restructured the "super nested" code to a less nested one
- Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout
- Removed/changed some of the LOG.xxx()
- Removed the exception that goes to the UI about maxtime
- Renamed the base class as BTech (from btech)
Bug fixes
- Fixed the bug found by minuteman1120 about the duplex not working in some cases
- New "split" algorithm for the duplex bug above
- Fixed a bug with the scode about 0x*F reseted to 0x*0
New
- Added support for the QYT KT-UV980 Radio
This is a lot to digest in a single patch. It'd be great to separate some of the changes so we don't have to block some of the critical/obvious stuff on the other that needs some discussion.
+def _clean_buffer(radio):
- """Cleaning the read serial buffer"""
- radio.pipe.setTimeout(0.1)
- try:
dump = radio.pipe.read(100)
- except Exception:
raise errors.RadioError("Unknown error cleaning the serial buffer")
This won't really flush the inbound reliably. You need to loop until you read at least once with nothing returned (i.e. hit the timeout). If there are 101 bytes in the pipe, this will not read them all out. Similarly, if there are ten bytes and the first read returns five of those, you won't clear the rest. Remember that read() does not have to return all the bytes you asked for.
def _rawrecv(radio, amount):
- """Raw read from the radio device, new approach, this time a byte at
- a time as the original driver, the receive data has to be atomic"""
"""Raw read from the radio device, less intensive way"""
data = ""
try:
tdiff = 0
start = time.time()
maxtime = amount * 0.020
while len(data) < amount and tdiff < maxtime:
d = radio.pipe.read(1)
if len(d) == 1:
data += d
# Delta time
tdiff = time.time() - start
# DEBUG
if debug is True:
LOG.debug("time diff %.04f maxtime %.04f, data: %d" %
(tdiff, maxtime, len(data)))
# Getting the data with a dynamic timeout in the serial deppending on
# the amount of data we need
timeout = amount * 0.06
This really doesn't make any sense: the value of timeout doesn't need to be greater if you're reading more data. The select() call that pyserial is making doesn't know how much data you're asking for, it only reports whether there is *any* data to be read from the descriptor. Further, pyserial cuts down the timeout each time through the read loop, if it has to go again. That means that if you're a little late on the first read loop, your timeout is cut down even further in the second and subsequent loops. Eventually that will end up cutting your timeout down to very small (i.e. less than a byte interval) value, which will make this unreliable. It might be good to review the implementation of pyserial's read code:
https://github.com/pyserial/pyserial/blob/master/serial/serialposix.py#L470-...
This is why you need to be processing incoming data into a queue if you're having trouble with the timing bits. Relying on the timeouts like you are is only going to work in lucky situations where you start your read with enough timeout remaining to process the full block.
+def _start_clone_mode(radio, status):
"""Put the radio in clone mode and get the ident string
using multiple tries"""
# how many tries
- tries = 5
tries = 3
# prep the data to show in the UI status.cur = 0 status.msg = "Identifying the radio..."
- status.max = len(radio._magic) * tries
- status.max = tries * len(radio._magic) radio.status_fn(status) mc = 0
- radio.pipe.setTimeout(0.6)
- def send_magic_word(m, magic):
Can you break this out to module level instead of defining it here? It's a little long and deep as a closure -- I didn't realize it was separate during my first read through the patch.
"""Send the magic word to the radio, catching the answer"""
# cleaning the buffer and set timeout
_clean_buffer(radio)
You do the buffer purge here...
def _do_ident(radio, status): @@ -366,28 +352,21 @@ # set the serial discipline radio.pipe.setBaudrate(9600) radio.pipe.setParity("N")
- radio.pipe.setTimeout(0.005)
- # cleaning the serial buffer, try wrapped
- try:
radio.pipe.flushInput()
- except:
msg = "Error with a serial rx buffer flush at _do_ident"
LOG.error(msg)
raise errors.RadioError(msg)
- # cleaning the buffers
- _clean_buffer(radio)
But you've already done it here, just a line before you call the above method which does it again:
# do the magic trick
- if _do_magic(radio, status) is False:
- if _start_clone_mode(radio, status) is False:
Once should be enough if it's properly implemented :)
# basic check for the ident
- if len(ident) != 49:
msg = "Radio send a sort ident block, you need to increase maxtime."
LOG.error(msg)
- if len(ident) < 49:
msg = "Radio sent a sort ident block, aborting"
s/sort/short/
raise errors.RadioError(msg) # check if ident is OK
@@ -402,7 +381,9 @@ msg = "Incorrect model ID, got this:\n\n" msg += util.hexprint(ident) LOG.debug(msg)
raise errors.RadioError("Radio identification failed.")
error = "Radio identification failed, this is not the correct model, "
error += "or it's a not supported variant yet."
raise errors.RadioError()
Did you mean to include error in this RadioError() ?
- # set a delay based on the OS
- if sys.platform in ["win32", "cygwin"]:
# we are in windows
delay = 0.015
- else:
# we are in linux/mac
delay = 0.033
Where does this come from? Nowhere else in the CHIRP source do we have to adjust timing like this based on the platform. I strongly object to this without real justification.
_send(radio, _make_frame("X", addr, TX_BLOCK_SIZE, d), delay)
This just makes _send() delay by 15 or 33ms *after* the write, which makes no sense. If you need to delay, do it here before you do the read.
# receiving the response ack = _rawrecv(radio, 1)
Jim, I get the impression that you are largely working on the memory decode bits and Pavel is working on the cloning part? If so, it'd be cool if you could just send your smaller tweaks (if possible) as separate patches so we can get them landed and fixed for people while we discuss the cloning bits separately.
Pavel, does your internet connection prevent you from sending patches to the list for some reason?
Thanks!
--Dan
Jim, I get the impression that you are largely working on the memory decode bits and Pavel is working on the cloning part? If so, it'd be cool if you could just send your smaller tweaks (if possible) as separate patches so we can get them landed and fixed for people while we discuss the cloning bits separately.
Pavel, does your internet connection prevent you from sending patches to the list for some reason?
Thanks!
--Dan
I almost did. And was just just about to tell Pavel we need to start processing this in smaller chunks.
Jim
- # set a delay based on the OS
- if sys.platform in ["win32", "cygwin"]:
# we are in windows
delay = 0.015
- else:
# we are in linux/mac
delay = 0.033
Where does this come from? Nowhere else in the CHIRP source do we have to adjust timing like this based on the platform. I strongly object to this without real justification.
_send(radio, _make_frame("X", addr, TX_BLOCK_SIZE, d), delay)
This just makes _send() delay by 15 or 33ms *after* the write, which makes no sense. If you need to delay, do it here before you do the read.
There may be a better way of doing this. But if this isn't done, Linux sends too fast and the radio doesn't respond after uploading the 3rd block. If Windows get the same dealy, then it take 2.5 to 4.5 minutes to upload to the radio.
Jim
On Sun, Mar 27, 2016 at 5:42 PM, Dan Smith via chirp_devel chirp_devel@intrepid.danplanet.com wrote:
This patch includes the following changes:
Changes related to developer mailing list discussions
- Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one
- Removed our software serial timeout in favor of the system serial timeout
- Removed the reference of the second magic for the alpha radio
- Renamed the _do_magic() def
- Restructured the "super nested" code to a less nested one
- Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout
- Removed/changed some of the LOG.xxx()
- Removed the exception that goes to the UI about maxtime
- Renamed the base class as BTech (from btech)
Bug fixes
- Fixed the bug found by minuteman1120 about the duplex not working in some cases
- New "split" algorithm for the duplex bug above
- Fixed a bug with the scode about 0x*F reseted to 0x*0
New
- Added support for the QYT KT-UV980 Radio
This is a lot to digest in a single patch. It'd be great to separate some of the changes so we don't have to block some of the critical/obvious stuff on the other that needs some discussion.
+def _clean_buffer(radio):
- """Cleaning the read serial buffer"""
- radio.pipe.setTimeout(0.1)
- try:
dump = radio.pipe.read(100)
- except Exception:
raise errors.RadioError("Unknown error cleaning the serial buffer")
This won't really flush the inbound reliably. You need to loop until you read at least once with nothing returned (i.e. hit the timeout). If there are 101 bytes in the pipe, this will not read them all out. Similarly, if there are ten bytes and the first read returns five of those, you won't clear the rest. Remember that read() does not have to return all the bytes you asked for.
def _rawrecv(radio, amount):
- """Raw read from the radio device, new approach, this time a byte at
- a time as the original driver, the receive data has to be atomic"""
"""Raw read from the radio device, less intensive way"""
data = ""
try:
tdiff = 0
start = time.time()
maxtime = amount * 0.020
while len(data) < amount and tdiff < maxtime:
d = radio.pipe.read(1)
if len(d) == 1:
data += d
# Delta time
tdiff = time.time() - start
# DEBUG
if debug is True:
LOG.debug("time diff %.04f maxtime %.04f, data: %d" %
(tdiff, maxtime, len(data)))
# Getting the data with a dynamic timeout in the serial deppending on
# the amount of data we need
timeout = amount * 0.06
This really doesn't make any sense: the value of timeout doesn't need to be greater if you're reading more data. The select() call that pyserial is making doesn't know how much data you're asking for, it only reports whether there is *any* data to be read from the descriptor. Further, pyserial cuts down the timeout each time through the read loop, if it has to go again. That means that if you're a little late on the first read loop, your timeout is cut down even further in the second and subsequent loops. Eventually that will end up cutting your timeout down to very small (i.e. less than a byte interval) value, which will make this unreliable. It might be good to review the implementation of pyserial's read code:
https://github.com/pyserial/pyserial/blob/master/serial/serialposix.py#L470-...
This is why you need to be processing incoming data into a queue if you're having trouble with the timing bits. Relying on the timeouts like you are is only going to work in lucky situations where you start your read with enough timeout remaining to process the full block.
+def _start_clone_mode(radio, status):
"""Put the radio in clone mode and get the ident string
using multiple tries"""
# how many tries
- tries = 5
tries = 3
# prep the data to show in the UI status.cur = 0 status.msg = "Identifying the radio..."
- status.max = len(radio._magic) * tries
- status.max = tries * len(radio._magic) radio.status_fn(status) mc = 0
- radio.pipe.setTimeout(0.6)
- def send_magic_word(m, magic):
Can you break this out to module level instead of defining it here? It's a little long and deep as a closure -- I didn't realize it was separate during my first read through the patch.
"""Send the magic word to the radio, catching the answer"""
# cleaning the buffer and set timeout
_clean_buffer(radio)
You do the buffer purge here...
def _do_ident(radio, status): @@ -366,28 +352,21 @@ # set the serial discipline radio.pipe.setBaudrate(9600) radio.pipe.setParity("N")
- radio.pipe.setTimeout(0.005)
- # cleaning the serial buffer, try wrapped
- try:
radio.pipe.flushInput()
- except:
msg = "Error with a serial rx buffer flush at _do_ident"
LOG.error(msg)
raise errors.RadioError(msg)
- # cleaning the buffers
- _clean_buffer(radio)
But you've already done it here, just a line before you call the above method which does it again:
# do the magic trick
- if _do_magic(radio, status) is False:
- if _start_clone_mode(radio, status) is False:
Once should be enough if it's properly implemented :)
# basic check for the ident
- if len(ident) != 49:
msg = "Radio send a sort ident block, you need to increase maxtime."
LOG.error(msg)
- if len(ident) < 49:
msg = "Radio sent a sort ident block, aborting"
s/sort/short/
raise errors.RadioError(msg) # check if ident is OK
@@ -402,7 +381,9 @@ msg = "Incorrect model ID, got this:\n\n" msg += util.hexprint(ident) LOG.debug(msg)
raise errors.RadioError("Radio identification failed.")
error = "Radio identification failed, this is not the correct model, "
error += "or it's a not supported variant yet."
raise errors.RadioError()
Did you mean to include error in this RadioError() ?
- # set a delay based on the OS
- if sys.platform in ["win32", "cygwin"]:
# we are in windows
delay = 0.015
- else:
# we are in linux/mac
delay = 0.033
Where does this come from? Nowhere else in the CHIRP source do we have to adjust timing like this based on the platform. I strongly object to this without real justification.
_send(radio, _make_frame("X", addr, TX_BLOCK_SIZE, d), delay)
This just makes _send() delay by 15 or 33ms *after* the write, which makes no sense. If you need to delay, do it here before you do the read.
# receiving the response ack = _rawrecv(radio, 1)
Jim, I get the impression that you are largely working on the memory decode bits and Pavel is working on the cloning part? If so, it'd be cool if you could just send your smaller tweaks (if possible) as separate patches so we can get them landed and fixed for people while we discuss the cloning bits separately.
Pavel, does your internet connection prevent you from sending patches to the list for some reason?
Thanks!
--Dan _______________________________________________ chirp_devel mailing list chirp_devel@intrepid.danplanet.com http://intrepid.danplanet.com/mailman/listinfo/chirp_devel Developer docs: http://chirp.danplanet.com/projects/chirp/wiki/Developers
On Sun, Mar 27, 2016 at 5:42 PM, Dan Smith via chirp_devel chirp_devel@intrepid.danplanet.com wrote:
This patch includes the following changes:
Changes related to developer mailing list discussions
- Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one
- Removed our software serial timeout in favor of the system serial timeout
- Removed the reference of the second magic for the alpha radio
- Renamed the _do_magic() def
- Restructured the "super nested" code to a less nested one
- Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout
- Removed/changed some of the LOG.xxx()
- Removed the exception that goes to the UI about maxtime
- Renamed the base class as BTech (from btech)
Bug fixes
- Fixed the bug found by minuteman1120 about the duplex not working in some cases
- New "split" algorithm for the duplex bug above
- Fixed a bug with the scode about 0x*F reseted to 0x*0
New
- Added support for the QYT KT-UV980 Radio
This is a lot to digest in a single patch. It'd be great to separate some of the changes so we don't have to block some of the critical/obvious stuff on the other that needs some discussion.
Pavel,
I will try to send the non-IO related items in separated patches tonight or tomorrow evening.
Jim
On Sun, Mar 27, 2016 at 7:43 PM, Jim Unroe rock.unroe@gmail.com wrote:
On Sun, Mar 27, 2016 at 5:42 PM, Dan Smith via chirp_devel chirp_devel@intrepid.danplanet.com wrote:
This patch includes the following changes:
Changes related to developer mailing list discussions
- Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one
- Removed our software serial timeout in favor of the system serial timeout
- Removed the reference of the second magic for the alpha radio
- Renamed the _do_magic() def
- Restructured the "super nested" code to a less nested one
- Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout
- Removed/changed some of the LOG.xxx()
- Removed the exception that goes to the UI about maxtime
- Renamed the base class as BTech (from btech)
Bug fixes
- Fixed the bug found by minuteman1120 about the duplex not working in some cases
- New "split" algorithm for the duplex bug above
- Fixed a bug with the scode about 0x*F reseted to 0x*0
New
- Added support for the QYT KT-UV980 Radio
This is a lot to digest in a single patch. It'd be great to separate some of the changes so we don't have to block some of the critical/obvious stuff on the other that needs some discussion.
Pavel,
I will try to send the non-IO related items in separated patches tonight or tomorrow evening.
Jim
Dan and Pavel,
I believe that I have patches submitted for everything that can be done.
Thank you both for you assistance with this.
Jim
El 27/03/16 a las 22:13, Jim Unroe via chirp_devel escribió:
Dan and Pavel,
I believe that I have patches submitted for everything that can be done.
Thank you both for you assistance with this.
Jim
Dan & Jim,
I'm working on the IO part,
I have the first patch ready but mine has to do with parts of the code Jim already changed (but are not applied), it's safe to send it on top of Jim's changes before they are applied?
Or I have to wait for this to be applied?
73.
I have the first patch ready but mine has to do with parts of the code Jim already changed (but are not applied), it's safe to send it on top of Jim's changes before they are applied?
Or I have to wait for this to be applied?
I just applied Jim's so you should be able to rebase now.
Thanks!
--Dan
El 27/03/16 a las 17:42, Dan Smith via chirp_devel escribió:
This patch includes the following changes:
Changes related to developer mailing list discussions
- Less intensive strategy to get big chunks of data instead the former more fine grained but resource intensive one
- Removed our software serial timeout in favor of the system serial timeout
- Removed the reference of the second magic for the alpha radio
- Renamed the _do_magic() def
- Restructured the "super nested" code to a less nested one
- Removed the "not needed" time.sleep() before the read of just one ACK, in favor of a serial system timeout
- Removed/changed some of the LOG.xxx()
- Removed the exception that goes to the UI about maxtime
- Renamed the base class as BTech (from btech)
Bug fixes
- Fixed the bug found by minuteman1120 about the duplex not working in some cases
- New "split" algorithm for the duplex bug above
- Fixed a bug with the scode about 0x*F reseted to 0x*0
New
- Added support for the QYT KT-UV980 Radio
This is a lot to digest in a single patch. It'd be great to separate some of the changes so we don't have to block some of the critical/obvious stuff on the other that needs some discussion.
Ok, don't apply it, I will work to separate it on issues with the improvements that will rise from the discussion on this thread.
73
El 27/03/16 a las 17:42, Dan Smith via chirp_devel escribió:
def _rawrecv(radio, amount):
- """Raw read from the radio device, new approach, this time a byte at
- a time as the original driver, the receive data has to be atomic"""
"""Raw read from the radio device, less intensive way"""
data = ""
try:
tdiff = 0
start = time.time()
maxtime = amount * 0.020
while len(data) < amount and tdiff < maxtime:
d = radio.pipe.read(1)
if len(d) == 1:
data += d
# Delta time
tdiff = time.time() - start
# DEBUG
if debug is True:
LOG.debug("time diff %.04f maxtime %.04f, data: %d" %
(tdiff, maxtime, len(data)))
# Getting the data with a dynamic timeout in the serial deppending on
# the amount of data we need
timeout = amount * 0.06
This really doesn't make any sense: the value of timeout doesn't need to be greater if you're reading more data. The select() call that pyserial is making doesn't know how much data you're asking for, it only reports whether there is*any* data to be read from the descriptor. Further, pyserial cuts down the timeout each time through the read loop, if it has to go again. That means that if you're a little late on the first read loop, your timeout is cut down even further in the second and subsequent loops. Eventually that will end up cutting your timeout down to very small (i.e. less than a byte interval) value, which will make this unreliable. It might be good to review the implementation of pyserial's read code:
https://github.com/pyserial/pyserial/blob/master/serial/serialposix.py#L470-...
This is why you need to be processing incoming data into a queue if you're having trouble with the timing bits. Relying on the timeouts like you are is only going to work in lucky situations where you start your read with enough timeout remaining to process the full block.
Good to know about the shrinking of the timeout by using this. I will review the pyserial code now to better understand the issue.
Can you point me to a example of the queue schema you mention here?
73
Can you point me to a example of the queue schema you mention here?
I don't think I've worked on any radio driver in chirp that has required such a thing. Many other amateur radio work has required stuff like this, usually because of slow over-the-air transmission rates, etc. So, not sure what to point you at. Just implementing a queue based on a list structure should be fine and straightforward.
--Dan
El 27/03/16 a las 17:42, Dan Smith via chirp_devel escribió:
Pavel, does your internet connection prevent you from sending patches to the list for some reason?
Yes, I'm mainly via a half duplex 9600 baud Ham Packet Radio Link in the 2m band (mean BW is 0.5 kb/s), then a (very restrictive) proxy, etc. This is from home and anything bigger than 20-50k will timeout either way (down/up)
Now I get access to a real internet (1Mbps) connection via WIFI but I have to travel to a public park and connect there, that's why Jim has been on the task of patch sending.
We will split the task, don't worry.
73.
El 27/03/16 a las 17:42, Dan Smith via chirp_devel escribió:
+def _clean_buffer(radio):
- """Cleaning the read serial buffer"""
- radio.pipe.setTimeout(0.1)
- try:
dump = radio.pipe.read(100)
- except Exception:
raise errors.RadioError("Unknown error cleaning the serial buffer")
This won't really flush the inbound reliably. You need to loop until you read at least once with nothing returned (i.e. hit the timeout). If there are 101 bytes in the pipe, this will not read them all out. Similarly, if there are ten bytes and the first read returns five of those, you won't clear the rest. Remember that read() does not have to return all the bytes you asked for.
You mean something like this:
===========================================
def _clean_buffer(radio): # I omitted the non essential code radio.pipe.setTimeout(0.1) # more or less 100 bytes on a call out = "1"
while len(out) > 0: out = radio.pipe.read(100)
===========================================
I have tested it and it will work as you mention, but the script testbed will hang testing it, with the experience we had reading we found just a few bytes on the serial queue never reaching 100, so this seems to be a good number for our case.
Can you mod the testbed to allow for a code like that to pass it?
Thinking bigger: can be useful to have a call like this at the radio.pipe level? I mean to code it at the radio class to simply call from any driver like this
radio.pipe.flushInput() or .cleanBuffer() or anything alike.
73
You mean something like this:
===========================================
def _clean_buffer(radio): # I omitted the non essential code radio.pipe.setTimeout(0.1) # more or less 100 bytes on a call out = "1"
while len(out) > 0: out = radio.pipe.read(100)
===========================================
Yes.
I have tested it and it will work as you mention, but the script testbed will hang testing it, with the experience we had reading we found just a few bytes on the serial queue never reaching 100, so this seems to be a good number for our case.
Because of the random influx test that generates garbage for every read, right? That test is proving that if you literally just do the above, then you would hang in a real-world case as well. Imagine if someone chose the wrong serial port and instead chose an NMEA GPS with a constant stream of data. You'd hang.
So, put a limit on it and discard data, failing if you read 100 or 1024 bytes before you hit the "end" or something like that. Just saying, read(100) can't be sure to clear the buffer.
--Dan
participants (3)
-
Dan Smith
-
Jim Unroe
-
M.Sc. Pavel Milanes Costa