# HG changeset patch
# User DanClemmensen <DanClemmensen@gmail.com>
# Date 1550894107 28800
#      Fri Feb 22 19:55:07 2019 -0800
# Node ID 41efaf746855eaa768603b5d29ee2d0d29ac4a5d
# Parent  2513b6da29c39aa92e8df3c7468949d479e8a984
bug fixes for ft4.py.
1) upload to radio now works (tip and py3)
2) radio version is now checked on upload in addition to download
3) version check is more tolerant of different version formats
4) Tone handling is now correct and tested

diff -r 2513b6da29c3 -r 41efaf746855 chirp/drivers/ft4.py
--- a/chirp/drivers/ft4.py	Thu Feb 21 13:47:30 2019 -0800
+++ b/chirp/drivers/ft4.py	Fri Feb 22 19:55:07 2019 -0800
@@ -23,11 +23,9 @@
 """
 import logging
 import struct
-from chirp import chirp_common, directory, memmap, bitwise, errors,  util
+from chirp import chirp_common, directory, memmap, bitwise, errors, util
 from chirp.settings import RadioSetting, RadioSettingGroup, \
-    RadioSettingValueList, \
-    RadioSettingValueString, \
-    RadioSettings
+    RadioSettingValueList, RadioSettingValueString, RadioSettings
 
 LOG = logging.getLogger(__name__)
 
@@ -39,7 +37,7 @@
 # each with a different purpose and format. Five groups consist of slots.
 # A slot describes a radio channel, and all slots have the same internal
 # format. Three of the groups consist of bitmaps, which all have the same
-# internal mapping. Name group, misc group, and DTMF digit group,
+# internal mapping. also groups for Name, misc, DTMF digits, and prog,
 # plus some unused groups.
 
 # Define the structures for each type of group here, but do not associate them
@@ -76,7 +74,8 @@
 };
 
 //miscellaneous params. One 4-block group. (could be treated as 4 separate.)
-//"SMI": "Set Mode Index" of the radio keypad function used to set a parameter.
+//"SMI": "Set Mode Index" of the FT-4 radio keypad function to set parameter.
+//"SMI numbers on the FT-65 are difrferent but the names an mem are the same.
 struct misc {
   u8  apo;        //SMI 01. 0==off, (1-24) is the number of half-hours.
   u8  arts_beep;  //SMI 02. 0==off, 1==inrange, 2==always
@@ -126,12 +125,13 @@
 };
 
 struct dtmfset {
- u8 digit[16];    //ASCII (*,#,0-9,A-D). (null terminated??)
+ u8 digit[16];    //ASCII (*,#,-,0-9,A-D). (dash-filled??)
 };
 
 //one block with stuff for the programmable keys
+//supports 4 keys. FT-4 has only 2 keys. FT-65 has all 4.
 struct progs {
- u8 modes[8];     //should be array of 2-byte structs, but bitwise.py objects
+ u8 modes[8];     //should be array of 2-byte structs, but bitwise.py refuses
  u8 ndx[4];
  u8 unused[8];
 };
@@ -192,6 +192,20 @@
 # the serial port. The code runs on either Python 2 or Python3, so some
 # constructs could be better optimized for one or the other, but not both.
 
+def get_mmap_data(radio):
+    """
+    horrible kludge needed until we convert entirely to Python 3 OR we add a
+    slightly less horrible kludge to the Py2 or Py3 versions of memmap.py.
+    The minimal change have Py3 code return a bytestring instead of a string.
+    This is the only function in this module that must explicitly test for the
+    data string type. It is used only in the do_upload function.
+    returns the memobj data as a byte-like object.
+    """
+    data = radio.get_mmap().get_packed()
+    if isinstance(data, bytes):
+        return data
+    return bytearray(radio.get_mmap()._data)
+
 
 def checkSum8(data):
     """
@@ -202,13 +216,34 @@
     return sum(x for x in bytearray(data)) & 0xFF
 
 
+def variable_len_resp(pipe):
+    """
+    when length of expected reply is not known, read byte at a time
+    until the ack character is found.
+    """
+    response = b""
+    i = 0
+    toolong = 256        # arbitrary
+    while True:
+        b = pipe.read(1)
+        if b == b'\x06':
+            break
+        else:
+            response += b
+            i += 1
+            if i > toolong:
+                LOG.debug("Response too long. got" + util.hexprint(response))
+                raise errors.RadioError("Response too long.")
+    return(response)
+
+
 def sendcmd(pipe, cmd, response_len):
     """
-    send a command bytelist to radio,receive and return the resulting bytelist.
+    send a command bytes to radio,receive and return the resulting bytes.
     Input: pipe         - serial port object to use
            cmd          - bytes to send
            response_len - number of bytes of expected response,
-                           not including the ACK.
+                           not including the ACK. (if None, read until ack)
     This cable is "two-wire": The TxD and RxD are "or'ed" so we receive
     whatever we send and then whatever response the radio sends. We check the
     echo and strip it, returning only the radio's response.
@@ -221,6 +256,8 @@
         msg += "Received:" + util.hexprint(echo)
         LOG.debug(msg)
         raise errors.RadioError("Incorrect echo on serial port.")
+    if response_len is None:
+        return variable_len_resp(pipe)
     if response_len > 0:
         response = pipe.read(response_len)
     else:
@@ -232,11 +269,37 @@
     return response
 
 
-def getblock(pipe, addr, _mmap):
+expected_id = b'IFT-35R\x00\x00V100\x00\x00'
+
+
+def startcomms(radio):
+    """
+    For either upload or download, put the radio into PROGRAM mode
+    and check the radio's id. In this preliminary version of the driver,
+    the exact nature of the ID has been inferred from a single test case.
+        send "PROGRAM" to command the radio into clone mode
+        read the initial string (version?)
     """
-    read a single 16-byte block from the radio
+    if b"QX" != sendcmd(radio.pipe, b"PROGRAM", 2):
+        raise errors.RadioError("expected QX from radio.")
+    id_response = sendcmd(radio.pipe, b'\x02', None)
+    if id_response != expected_id:
+        if id_response[0:8] != expected_id[0:8]:
+            msg = "ID mismatch. Expected" + util.hexprint(expected_id)
+            msg += ", Received:" + util.hexprint(id_response)
+            LOG.debug(msg)
+            raise errors.RadioError("Incorrect ID.")
+        else:
+            msg = "ID suspect. Expected" + util.hexprint(expected_id)
+            msg += ", Received:" + util.hexprint(id_response)
+            LOG.debug(msg)
+
+
+def getblock(pipe, addr, image):
+    """
+    read a single 16-byte block from the radio.
     send the command and check the response
-    returns the 16-byte bytearray
+    places the response into the correct offset in the supplied bytearray
     """
     cmd = struct.pack(">cHb", b"R", addr, 16)
     response = sendcmd(pipe, cmd, 21)
@@ -248,41 +311,24 @@
     if checkSum8(response[1:20]) != bytearray(response)[20]:
         LOG.debug(b"Bad checksum: " + util.hexprint(response))
         raise errors.RadioError("bad block checksum.")
-    _mmap[addr:addr+16] = response[4:20]
-
-
-expected_id = b'IFT-35R\x00\x00V100\x00\x00'
+    image[addr:addr+16] = response[4:20]
 
 
 def do_download(radio):
     """
     Read memory from the radio.
-      send "PROGRAM" to command the radio into clone mode,
-      read the initial string (version?)
+      call startcomms to go into program mode and check version
       create an mmap
       read the memory blocks and place the data into the mmap
       send "END"
     """
-    _mmap = bytearray(radio.get_memsize())
+    image = bytearray(radio.get_memsize())
     pipe = radio.pipe  # Get the serial port connection
-
-    if b"QX" != sendcmd(pipe, b"PROGRAM", 2):
-        raise errors.RadioError("expected QX from radio.")
-    id_response = sendcmd(pipe, b'\x02', 15)
-    if id_response != expected_id:
-        if id_response[0:8] != expected_id[0:8]:
-            msg = "ID mismatch. Expected" + util.hexprint(expected_id)
-            msg += ", Received:" + util.hexprint(id_response)
-            LOG.debug(msg)
-            raise errors.RadioError("Incorrect ID.")
-        else:
-            msg = "ID suspect. Expected" + util.hexprint(expected_id)
-            msg += ", Received:" + util.hexprint(id_response)
-            LOG.debug(msg)
+    startcomms(radio)
     for _i in range(radio.numblocks):
-        getblock(pipe, 16 * _i, _mmap)
+        getblock(pipe, 16 * _i, image)
     sendcmd(pipe, b"END", 0)
-    return memmap.MemoryMap(bytes(_mmap))
+    return memmap.MemoryMap(bytes(image))
 
 
 def putblock(pipe, addr, data):
@@ -298,16 +344,13 @@
 def do_upload(radio):
     """
     Write memory image to radio
-      send "PROGRAM" to command the radio into clone mode,
+      call startcomms to go into program mode and check version
       write the memory blocks. Skip the first block
       send "END"
     """
     pipe = radio.pipe  # Get the serial port connection
-
-    if b"QX" != sendcmd(pipe, b"PROGRAM", 2):
-        raise errors.RadioError("expected QX from radio.")
-    data = radio.get_mmap()
-    sendcmd(pipe, b'\x02', 15)
+    startcomms(radio)
+    data = get_mmap_data(radio)
     for _i in range(1, radio.numblocks):
         putblock(pipe, 16*_i, data[16*_i:16*(_i+1)])
     sendcmd(pipe, b"END", 0)
@@ -401,12 +444,53 @@
                 chirp_common.PowerLevel("Mid", watts=2.5),
                 chirp_common.PowerLevel("Low", watts=0.5)]
 STEPS = [0, 5.0, 6.25, 10.0, 12.5, 15.0, 20.0, 25.0, 50.0, 100.0]
-TONE_MODES = ["", "Tone", "TSQL",  "DTCS",  "DTCS-R",  "TSQL-R",   "Cross"]
-CROSS_MODES = ["DTCS->",  "DTCS->DTCS"]   # only the extras we need
-# The radio and the code support the additional cross modes, but
-# they are redundant with the extended tone modes, and they cause
-# the "BruteForce" unit test to fail.
-# CROSS_MODES += ["Tone->Tone", "->DTCS", "->Tone", "DTCS->DTCS", "Tone->"]
+# Yaesu sql_type field codes
+SQL_TYPE = ["off", "R-TONE", "T-TONE", "TSQL",  "REV-TN", "DCS", "PAGER"]
+
+# map a  CHIRP tone mode to a FT-4 sql and which if any code to set to 0.
+# The keys are provided to RadioFeatures as a list
+MODES_TONE = {
+    "":       ("off",  None),
+    "Tone":   ("T-TONE", None),        # chirp means "xmit, not rcv"
+    "TSQL":   ("TSQL", None),          # chirp means "xmit and rcv, x==r"
+    "DTCS":   ("DCS",  None),          # chirp means "xmt and rcv
+    "TSQL-R": ("REV-TN", None),        # not documented. reverse R-Tone
+    "Cross":  ()                       # not used in lookup
+    }
+
+# map a CHIRP Cross type if the CHIRP sql type is "cross". The keys
+# are provided to RadioFeatures as a list.
+MODES_CROSS = {
+    "DTCS->": ("DCS", "rx_dcs"),
+    "->DTCS": ("DCS", "tx_dcs"),
+    "DTCS->DTCS": ("DCS", None),
+    "->Tone": ("R-TONE", None),
+    "Tone->Tone": ("TSQL", None)
+    }
+
+# Map the radio image tone/dtcs settings back to the CHIRP settings
+RADIO_TMODES = [
+        ("(None)", ["", ""]),           # sql_type= 0. off
+        ("(None)", ["Cross", "->Tone"]),       # sql_type= 1. R-TONE
+        ("(None)", ["Tone", ""]),       # sql_type= 2. T-TONE
+        ("(None)", None, "tx_ctcss", "rx_ctcss", [    # sql_type= 3. TSQL
+          ["", None],                   # x==0, r==0 : not valid
+          ["TSQL", ""],                 # x==0
+          ["Tone", ""],                 # r==0
+          ["Cross", "Tone->Tone"],      # x!=r
+          ["TSQL", ""]                  # x==r
+         ]),
+        ("(None)", ["TSQL-R", ""]),     # sql_type= 4. REV TN
+        ("(None)", None, "tx_dcs", "rx_dcs", [    # sql_type= 5.DCS
+          ["", None],                   # x==0, r==0 : not valid
+          ["Cross", "->DTCS"],          # x==0
+          ["Cross", "DTCS->"],          # r==0
+          ["Cross", "DTCS->DTCS"],      # x!=r
+          ["DTCS", ""]                  # x==r
+         ]),
+        ("PAGER", ["", None])  # sql_type= 6. handled as a CHIRP "extra"
+        ]
+
 
 DTMF_CHARS = "0123456789ABCD*#- "
 CW_ID_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ "
@@ -451,7 +535,7 @@
     ]
 EPCS_CODES = [format(flt) for flt in [0] + TONE_MAP[1:]]
 
-# names for the setmode function for the programmable keys.  Mode zero means
+# names for the setmode function for the programmable keys. Mode zero means
 # that the key is programmed for a memory not a setmode.
 SETMODES = [
     "mem", "apo", "ar bep", "ar int", "beclo",
@@ -507,8 +591,8 @@
         rf.valid_special_chans = specials
         rf.memory_bounds = (1, self.MAX_MEM_SLOT)
         rf.valid_duplexes = DUPLEX
-        rf.valid_tmodes = TONE_MODES
-        rf.valid_cross_modes = CROSS_MODES
+        rf.valid_tmodes = list(MODES_TONE.keys())
+        rf.valid_cross_modes = list(MODES_CROSS.keys())
         rf.valid_power_levels = POWER_LEVELS
         rf.valid_tuning_steps = STEPS
         rf.valid_skips = SKIPS
@@ -532,6 +616,8 @@
     def sync_in(self):
         try:
             self._mmap = do_download(self)
+        except errors.RadioError:
+            raise
         except Exception as e:
             raise errors.RadioError("Failed to communicate with radio: %s" % e)
         self.process_mmap()
@@ -548,8 +634,10 @@
     def process_mmap(self):
         self._memobj = bitwise.parse(MEM_FORMAT, self._mmap)
 
-    # functions to handle complicated settings.
-    # callback for settng  byte arrays (DTMF[0-9], passwd, and CW_ID)
+    # There are about 40 settings and most are handled generically below.
+    # The few that are more complicated use these handlers instead.
+
+    # callback for setting  byte arrays (DTMF[0-9], passwd, and CW_ID)
     def apply_str_to_bytearray(self,  element, obj):
         lng = len(obj)
         strng = (element.value.get_value() + "                ")[:lng]
@@ -585,16 +673,13 @@
                 "dtmf_%i" % i, "DTMF Autodialer Memory %i" % i, group)
 
     def apply_P(self, element, pnum):
-        value = element.value
-        self.memobj.progkeys.modes[pnum * 2] = [0, 2][value]
+        self.memobj.progkeys.modes[pnum * 2] = [0, 2][element.value]
 
     def apply_Pmode(self, element, pnum):
-        value = element.value
-        self.memobj.progkeys.modes[pnum * 2 + 1] = value
+        self.memobj.progkeys.modes[pnum * 2 + 1] = element.value
 
     def apply_Pmem(self, element, pnum):
-        value = element.value
-        self.memobj.progkeys.ndx[pnum].func = value
+        self.memobj.progkeys.ndx[pnum].func = element.value
 
     MEMLIST = ["%d" % i for i in range(1, MAX_MEM_SLOT)] + PMSNAMES
 
@@ -612,9 +697,10 @@
             get_prog(i + 1, ["unused",  "in use"],  _progkeys.modes[i * 2],
                      "P", "Programmable key ",  self.apply_P)
             get_prog(i + 1, SETMODES, _progkeys.modes[i * 2 + 1], "modeP",
-                     "mode for Programmable key",  self.apply_Pmode)
+                     "mode for Programmable key ",  self.apply_Pmode)
             get_prog(i + 1, self.MEMLIST, _progkeys.ndx[i], "memP",
-                     "mem for Programmable key",  self.apply_Pmem)
+                     "mem for Programmable key ",  self.apply_Pmem)
+    # ------------ End of special settings handlers.
 
     # list of group description tuples: (groupame,group title, [param list]).
     # A param is a tuple:
@@ -681,9 +767,9 @@
     # ----------------end of group_descriptions
 
     # returns the current values of all the settings in the radio memory image,
-    # in the form of a RadioSettings list. First, use the group_descriptions
-    # list to create the groups and most of the params. Then, add params that
-    # require extra stuff.
+    # in the form of a RadioSettings list. Uses the group_descriptions
+    # list to create the groups and params. Simple valuelist params are handled
+    # inline. More complex params are built by calling the special handlers.
     def get_settings(self):
         _settings = self._memobj.settings
         groups = RadioSettings()
@@ -736,35 +822,14 @@
                 LOG.debug(element.get_name())
                 raise
 
-    RADIO_TMODES = [
-        ("(None)", ["", ""]),         # off
-        ("(None)", ["TSQL-R", ""]),  # R-TONE
-        ("(None)", ["Tone", ""]),   # T-TONE
-        ("(None)", None, "tx_ctcss", "rx_ctcss", [    # TSQL
-          ["", None],                 # x==0, r==0 : not valid
-          ["TSQL-R", ""],      # x==0
-          ["Tone", ""],             # r==0
-          ["TSQL", ""],        # x!=r
-          ["TSQL", ""]               # x==r
-         ]),
-        ("REV-TN", ["TSQL-R", ""]),
-        ("(None)", None, "tx_dcs", "rx_dcs", [    # DCS
-          ["", None],                 # x==0, r==0 : not valid
-          ["DTCS-R", ""],           # x==0
-          ["Cross", "DTCS->"],      # r==0
-          ["Cross", "DTCS->DTCS"],  # x!=r
-          ["DTCS", ""]      # x==r
-         ]),
-        ("PAGER", ["", None])  # handled as a CHIRP "extra"
-        ]
     LOOKUP = [[True, True], [True, False], [False, True], [False, False]]
 
     def decode_sql(self, mem, chan):
         """
         examine the radio channel fields and determine the correct
-        CHIRP CSV values for tmode, cross_mode, and dcts_polarity
+        CHIRP CSV values for tmode, cross_mode, and sql_override
         """
-        mode = self.RADIO_TMODES[chan.sql_type]
+        mode = RADIO_TMODES[chan.sql_type]
         chirpvals = mode[1]
         if not chirpvals:
             x = getattr(chan, mode[2])
@@ -786,36 +851,14 @@
             mem.rx_dtcs = DTCS_MAP[chan.rx_dcs]
         LOG.debug(" setting sql_override to <%s>" % mode[0])
         mem.extra = RadioSettingGroup("Extra", "extra")
-        extra_modes = ["(None)", "REV-TN", "PAGER"]
+        extra_modes = ["(None)", "PAGER"]
         valuelist = RadioSettingValueList(extra_modes,  mode[0])
         rs = RadioSetting("sql_override", "Squelch override", valuelist)
         mem.extra.append(rs)
-    # Yaesu sql_type field codes
-    SQL_TYPE = ["off", "R-TONE", "T-TONE", "TSQL",  "REV-TN", "DCS", "PAGER"]
-    # map a  CHIRP tone mode to a FT-4 sql and which if any code to set to 0.
-    MODE_TONE = {
-        "":       ("off",  None),
-        "Tone":   ("T-TONE", "rx_ctcss"),
-        "TSQL":   ("TSQL", None),
-        "DTCS":   ("DCS",  None),          # must set rx_dcs to tx_dcs?
-        "DTCS-R": ("DCS",  "tx_dcs"),
-        "TSQL-R": ("R-TONE", "tx_ctcss"),    # not documented on wiki
-        "Cross":  ()                       # not used in lookup
-        }
-
-    # map a CHIRP Cross type if the CHIRP sql type is "cross"
-    MODE_CROSS = {
-        "DTCS->": ("DCS", "rx_dcs"),
-        "DTCS->DTCS": ("DCS", None)
-        # "Tone->Tone": ("TSQL", None),
-        # "->DTCS": ("DCS", "tx_dcs"),
-        # "->Tone": ("R-TONE", None),
-        # "Tone->": ("T-Tone", None)
-        }
 
     def encode_sql(self, mem, chan):
         """
-        examine CHIRP CSV columns tmode, cross_mode, and dcts_polarity
+        examine CHIRP CSV columns tmode and cross_mode
         and set the correct values for the radio sql_type, dcs codes,
         and ctcss codes. We set all four codes, and then zero out
         a code if needed when Tone or DCS is one-way
@@ -824,22 +867,23 @@
         chan.tx_dcs = DTCS_MAP.index(mem.dtcs)
         chan.rx_ctcss = TONE_MAP.index(mem.ctone)
         chan.rx_dcs = DTCS_MAP.index(mem.rx_dtcs)
+        if mem.tmode == "TSQL":
+            chan.tx_ctcss = chan.rx_ctcss   # CHIRP uses ctone for TSQL
+        if mem.tmode == "DTCS":
+            chan.tx_dcs = chan.rx_dcs     # CHIRP uses rx_dtcs for DTCS
         tbl, ndx = [
-            (self.MODE_TONE, mem.tmode),
-            (self.MODE_CROSS, mem.cross_mode)
+            (MODES_TONE, mem.tmode),
+            (MODES_CROSS, mem.cross_mode)
             ][mem.tmode == "Cross"]
-        row = tbl[ndx]
-        if ndx == "DTCS":
-            chan.rx_dcs = chan.tx_dcs
-        chan.sql_type = self.SQL_TYPE.index(row[0])
-        if row[1]:
-            setattr(chan, row[1], 0)
+        sql_type, suppress = tbl[ndx]
+        chan.sql_type = SQL_TYPE.index(sql_type)
+        if suppress:
+            setattr(chan, suppress, 0)
         for setting in mem.extra:
             if (setting.get_name() == 'sql_override'):
                 value = str(setting.value)
                 if value != "(None)":
-                    chan.sql_type = self.SQL_TYPE.index(value)
-
+                    chan.sql_type = SQL_TYPE.index(value)
         return
 
     # given a CHIRP memory ref, get the radio memobj for it.
@@ -928,7 +972,6 @@
             mem.empty = False
             mem.extd_number = sname
             mem.immutable = ["number", "extd_number", "name", "skip"]
-
         return mem
 
     # modify a radio channel in memobj based on info in CHIRP canonical form
@@ -961,7 +1004,6 @@
                 duplex = "off"    # radio ignores when tx != rx
             self._memobj.txfreqs[num-1].freq = txfreq
         _mem.duplex = DUPLEX.index(duplex)
-
         return
 
 
