[chirp_devel] [PATCH 0 of 2] Initial RadioReference.com import support
Here is my initial attempt at RadioReference.com import support. It adds a menu for RadioReference.com import which prompts for a zipcode. It determines the county for that zipcode and downloads all frequencies in the RR database for that county. There is no means to request state or federal frequencies with this UI.
This feature requires a RadioReference.com Premium Subscription.
If there is interest, I'll improve the UI so data can be queried with other parameters (zipcode is very limiting).
Tom KD7LXL
# HG changeset patch # User Tom Hayward tom@tomh.us # Date 1334073618 21600 # Node ID 806b80ebe58e7d62f62ee9d6e9ab0fc929236e49 # Parent 86b87e37879200f579a4edee085b3238ea0800c1 Rename chirp/xml.py to chirp/generic_xml.py so it doesn't conflict with the xml module.
Required for Feature #114
diff -r 86b87e378792 -r 806b80ebe58e chirp/generic_xml.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/chirp/generic_xml.py Tue Apr 10 10:00:18 2012 -0600 @@ -0,0 +1,152 @@ +# Copyright 2008 Dan Smith dsmith@danplanet.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +import os +import libxml2 + +from chirp import chirp_common, errors, xml_ll, platform, directory + +def validate_doc(doc): + basepath = platform.get_platform().executable_path() + path = os.path.abspath(os.path.join(basepath, "chirp.xsd")) + if not os.path.exists(path): + path = "/usr/share/chirp/chirp.xsd" + + try: + ctx = libxml2.schemaNewParserCtxt(path) + schema = ctx.schemaParse() + except libxml2.parserError, e: + print "Unable to load schema: %s" % e + print "Path: %s" % path + raise errors.RadioError("Unable to load schema") + + del ctx + + errs = [] + warnings = [] + + def err(msg, arg=None): + errs.append("ERROR: %s" % msg) + + def wrn(msg, arg=None): + print "WARNING: %s" % msg + warnings.append("WARNING: %s" % msg) + + validCtx = schema.schemaNewValidCtxt() + validCtx.setValidityErrorHandler(err, wrn) + err = validCtx.schemaValidateDoc(doc) + print os.linesep.join(warnings) + if err: + print "---DOC---\n%s\n------" % doc.serialize(format=1) + print os.linesep.join(errs) + raise errors.RadioError("Schema error") + +def default_banks(): + banks = [] + + for i in range(0, 26): + banks.append("Bank-%s" % (chr(ord("A") + i))) + + return banks + +@directory.register +class XMLRadio(chirp_common.FileBackedRadio, chirp_common.IcomDstarSupport): + VENDOR = "Generic" + MODEL = "XML" + FILE_EXTENSION = "chirp" + + def __init__(self, pipe): + chirp_common.FileBackedRadio.__init__(self, None) + self._filename = pipe + if self._filename and os.path.exists(self._filename): + self.doc = libxml2.parseFile(self._filename) + validate_doc(self.doc) + else: + self.doc = libxml2.newDoc("1.0") + radio = self.doc.newChild(None, "radio", None) + radio.newChild(None, "memories", None) + radio.newChild(None, "banks", None) + radio.newProp("version", "0.1.1") + + def get_features(self): + rf = chirp_common.RadioFeatures() + rf.has_bank = False + #rf.has_bank_index = True + rf.requires_call_lists = False + rf.has_implicit_calls = False + rf.memory_bounds = (0, 1000) + rf.valid_characters = chirp_common.CHARSET_ASCII + rf.valid_name_length = 999 + rf.valid_tmodes = ["", "Tone", "TSQL", "DTCS"] + return rf + + def load(self, filename=None): + if not self._filename and not filename: + raise errors.RadioError("Need a location to load from") + + if filename: + self._filename = filename + + self.doc = libxml2.parseFile(self._filename) + validate_doc(self.doc) + + def save(self, filename=None): + if not self._filename and not filename: + raise errors.RadioError("Need a location to save to") + + if filename: + self._filename = filename + + f = file(self._filename, "w") + f.write(self.doc.serialize(format=1)) + f.close() + + def get_memories(self, lo=0, hi=999): + mems = [] + for i in range(lo, hi): + try: + mems.append(xml_ll.get_memory(self.doc, i)) + except errors.InvalidMemoryLocation: + pass + + return mems + + def get_memory(self, number): + mem = xml_ll.get_memory(self.doc, number) + + return mem + + def set_memory(self, mem): + xml_ll.set_memory(self.doc, mem) + + def erase_memory(self, number): + xml_ll.del_memory(self.doc, number) + + @classmethod + def match_model(cls, filedata, filename): + return filename.lower().endswith("." + cls.FILE_EXTENSION) + +if __name__ == "__main__": + r = XMLRadio("testmem.chirp") + + print r.get_memory(3) + + m = chirp_common.Memory() + m.name = "TestMem2" + m.freq = 123.456 + m.number = 10 + + #r.set_memory(m) + #r.erase_memory(10) diff -r 86b87e378792 -r 806b80ebe58e chirp/xml.py --- a/chirp/xml.py Mon Apr 09 18:38:22 2012 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,152 +0,0 @@ -# Copyright 2008 Dan Smith dsmith@danplanet.com -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import os -import libxml2 - -from chirp import chirp_common, errors, xml_ll, platform, directory - -def validate_doc(doc): - basepath = platform.get_platform().executable_path() - path = os.path.abspath(os.path.join(basepath, "chirp.xsd")) - if not os.path.exists(path): - path = "/usr/share/chirp/chirp.xsd" - - try: - ctx = libxml2.schemaNewParserCtxt(path) - schema = ctx.schemaParse() - except libxml2.parserError, e: - print "Unable to load schema: %s" % e - print "Path: %s" % path - raise errors.RadioError("Unable to load schema") - - del ctx - - errs = [] - warnings = [] - - def err(msg, arg=None): - errs.append("ERROR: %s" % msg) - - def wrn(msg, arg=None): - print "WARNING: %s" % msg - warnings.append("WARNING: %s" % msg) - - validCtx = schema.schemaNewValidCtxt() - validCtx.setValidityErrorHandler(err, wrn) - err = validCtx.schemaValidateDoc(doc) - print os.linesep.join(warnings) - if err: - print "---DOC---\n%s\n------" % doc.serialize(format=1) - print os.linesep.join(errs) - raise errors.RadioError("Schema error") - -def default_banks(): - banks = [] - - for i in range(0, 26): - banks.append("Bank-%s" % (chr(ord("A") + i))) - - return banks - -@directory.register -class XMLRadio(chirp_common.FileBackedRadio, chirp_common.IcomDstarSupport): - VENDOR = "Generic" - MODEL = "XML" - FILE_EXTENSION = "chirp" - - def __init__(self, pipe): - chirp_common.FileBackedRadio.__init__(self, None) - self._filename = pipe - if self._filename and os.path.exists(self._filename): - self.doc = libxml2.parseFile(self._filename) - validate_doc(self.doc) - else: - self.doc = libxml2.newDoc("1.0") - radio = self.doc.newChild(None, "radio", None) - radio.newChild(None, "memories", None) - radio.newChild(None, "banks", None) - radio.newProp("version", "0.1.1") - - def get_features(self): - rf = chirp_common.RadioFeatures() - rf.has_bank = False - #rf.has_bank_index = True - rf.requires_call_lists = False - rf.has_implicit_calls = False - rf.memory_bounds = (0, 1000) - rf.valid_characters = chirp_common.CHARSET_ASCII - rf.valid_name_length = 999 - rf.valid_tmodes = ["", "Tone", "TSQL", "DTCS"] - return rf - - def load(self, filename=None): - if not self._filename and not filename: - raise errors.RadioError("Need a location to load from") - - if filename: - self._filename = filename - - self.doc = libxml2.parseFile(self._filename) - validate_doc(self.doc) - - def save(self, filename=None): - if not self._filename and not filename: - raise errors.RadioError("Need a location to save to") - - if filename: - self._filename = filename - - f = file(self._filename, "w") - f.write(self.doc.serialize(format=1)) - f.close() - - def get_memories(self, lo=0, hi=999): - mems = [] - for i in range(lo, hi): - try: - mems.append(xml_ll.get_memory(self.doc, i)) - except errors.InvalidMemoryLocation: - pass - - return mems - - def get_memory(self, number): - mem = xml_ll.get_memory(self.doc, number) - - return mem - - def set_memory(self, mem): - xml_ll.set_memory(self.doc, mem) - - def erase_memory(self, number): - xml_ll.del_memory(self.doc, number) - - @classmethod - def match_model(cls, filedata, filename): - return filename.lower().endswith("." + cls.FILE_EXTENSION) - -if __name__ == "__main__": - r = XMLRadio("testmem.chirp") - - print r.get_memory(3) - - m = chirp_common.Memory() - m.name = "TestMem2" - m.freq = 123.456 - m.number = 10 - - #r.set_memory(m) - #r.erase_memory(10) diff -r 86b87e378792 -r 806b80ebe58e chirpui/editorset.py --- a/chirpui/editorset.py Mon Apr 09 18:38:22 2012 -0700 +++ b/chirpui/editorset.py Tue Apr 10 10:00:18 2012 -0600 @@ -17,7 +17,7 @@ import gtk import gobject
-from chirp import chirp_common, directory, generic_csv, xml +from chirp import chirp_common, directory, generic_csv, generic_xml from chirpui import memedit, dstaredit, bankedit, common, importdialog from chirpui import inputdialog, reporting, settingsedit
@@ -279,7 +279,7 @@ if filen.lower().endswith(".csv"): dst_radio = generic_csv.CSVRadio(filen) elif filen.lower().endswith(".chirp"): - dst_radio = xml.XMLRadio(filen) + dst_radio = generic_xml.XMLRadio(filen) else: raise Exception(_("Unsupported file type")) except Exception, e: diff -r 86b87e378792 -r 806b80ebe58e chirpui/importdialog.py --- a/chirpui/importdialog.py Mon Apr 09 18:38:22 2012 -0700 +++ b/chirpui/importdialog.py Tue Apr 10 10:00:18 2012 -0600 @@ -17,7 +17,7 @@ import gobject import pango
-from chirp import errors, chirp_common, xml, import_logic +from chirp import errors, chirp_common, generic_xml, import_logic from chirpui import common
class WaitWindow(gtk.Window): @@ -225,7 +225,7 @@ print "One or more of the radios doesn't support banks" return
- if not isinstance(self.dst_radio, xml.XMLRadio) and \ + if not isinstance(self.dst_radio, generic_xml.XMLRadio) and \ len(dst_banks) != len(src_banks): print "Source and destination radios have a different number of banks" else: diff -r 86b87e378792 -r 806b80ebe58e chirpui/mainapp.py --- a/chirpui/mainapp.py Mon Apr 09 18:38:22 2012 -0700 +++ b/chirpui/mainapp.py Tue Apr 10 10:00:18 2012 -0600 @@ -34,7 +34,7 @@ except ImportError,e: common.log_exception() common.show_error("\nThe Pyserial module is not installed!") -from chirp import platform, xml, generic_csv, directory, util +from chirp import platform, generic_xml, generic_csv, directory, util from chirp import ic9x, kenwood_live, idrp, vx7, vx5 from chirp import CHIRP_VERSION, chirp_common, detect, errors from chirp import icf, ic9x_icf
# HG changeset patch # User Tom Hayward tom@tomh.us # Date 1334073630 21600 # Node ID 309d2038f31cf7e5137990c889ba4a6ca98ef321 # Parent 806b80ebe58e7d62f62ee9d6e9ab0fc929236e49 Add import from RadioReference.com. Feature #114
diff -r 806b80ebe58e -r 309d2038f31c chirp/directory.py --- a/chirp/directory.py Tue Apr 10 10:00:18 2012 -0600 +++ b/chirp/directory.py Tue Apr 10 10:00:30 2012 -0600 @@ -1,4 +1,5 @@ # Copyright 2010 Dan Smith dsmith@danplanet.com +# Copyright 2012 Tom Hayward tom@tomh.us # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +18,7 @@ import tempfile
from chirp import icf -from chirp import chirp_common, util, rfinder, errors +from chirp import chirp_common, util, rfinder, radioreference, errors
def radio_class_id(cls): ident = "%s_%s" % (cls.VENDOR, cls.MODEL) @@ -79,6 +80,12 @@ raise Exception("Unsupported model")
def get_radio_by_image(image_file): + if image_file.startswith("radioreference://"): + method, _, zipcode, username, password = image_file.split("/", 4) + rr = radioreference.RadioReferenceRadio(None) + rr.set_params(zipcode, username, password) + return rr + if image_file.startswith("rfinder://"): method, _, email, passwd, lat, lon = image_file.split("/") rf = rfinder.RFinderRadio(None) diff -r 806b80ebe58e -r 309d2038f31c chirp/radioreference.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/chirp/radioreference.py Tue Apr 10 10:00:30 2012 -0600 @@ -0,0 +1,150 @@ +# Copyright 2012 Tom Hayward tom@tomh.us +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +from chirp import chirp_common, CHIRP_VERSION, errors +try: + from suds.client import Client + HAVE_SUDS = True +except ImportError: + HAVE_SUDS = False + +MODES = { + "FM" : "FM", + "AM" : "AM", + "FMN" : "NFM", + "D-STAR": "DV", + "USB" : "USB", + "LSB" : "LSB", +} + +class RadioReferenceRadio(chirp_common.Radio): + VENDOR = "Radio Reference LLC" + MODEL = "RadioReference.com" + + URL = "http://api.radioreference.com/soap2/?wsdl" + APPKEY = "46785108" + + def __init__(self, *args, **kwargs): + chirp_common.Radio.__init__(self, *args, **kwargs) + + if not HAVE_SUDS: + raise errors.RadioError( + "Suds library required for RadioReference.com import.\n" + \ + "Try installing your distribution's python-suds package.") + + self._auth = {"appKey": self.APPKEY, "username": "", "password": ""} + self._client = Client(self.URL) + self._freqs = None + self._modes = None + + def set_params(self, zip, username, password): + self._zip = zip + self._auth["username"] = username + self._auth["password"] = password + + def do_fetch(self): + """Fetches frequencies for all subcategories in a county.""" + self._freqs = [] + + zipcode = self._client.service.getZipcodeInfo(self._zip, self._auth) + county = self._client.service.getCountyInfo(zipcode.ctid, self._auth) + for cat in county.cats: + print "Fetching category:", cat.cName + for subcat in cat.subcats: + print "\t", subcat.scName + result = self._client.service.getSubcatFreqs(subcat.scid, self._auth) + self._freqs += result + for agency in county.agencyList: + agency = self._client.service.getAgencyInfo(agency.aid, self._auth) + for cat in agency.cats: + print "Fetching category:", cat.cName + for subcat in cat.subcats: + print "\t", subcat.scName + result = self._client.service.getSubcatFreqs(subcat.scid, self._auth) + self._freqs += result + + def get_features(self): + if not self._freqs: + self.do_fetch() + + rf = chirp_common.RadioFeatures() + rf.memory_bounds = (0, len(self._freqs)-1) + return rf + + def get_raw_memory(self, number): + return repr(self._freqs[number]) + + def get_memory(self, number): + if not self._freqs: + self.do_fetch() + + freq = self._freqs[number] + + mem = chirp_common.Memory() + mem.number = number + + mem.name = freq.alpha + mem.freq = int(freq.out * 1000 * 1000) + if freq["in"] == 0.0: + mem.duplex = "" + else: + mem.duplex = "split" + mem.offset = int(freq["in"] * 1000 * 1000) + if freq.tone is not None: + if str(freq.tone) == "CSQ": # Carrier Squelch + mem.tmode = "" + else: + try: + tone, tmode = freq.tone.split(" ") + except: + tone, tmode = None, None + if tmode == "PL": + mem.tmode = "TSQL" + mem.rtone = mem.ctone = float(tone) + elif tmode == "DPL": + mem.tmode = "DTCS" + mem.dtcs = int(tone) + else: + print "Error: unsupported tone" + print freq + mem.mode = self._get_mode(freq.mode) + mem.comment = freq.descr.strip() + + return mem + + def _get_mode(self, modeid): + if not self._modes: + self._modes = {} + for mode in self._client.service.getMode("0", self._auth): + # sax.text.Text cannot be coerced directly to int + self._modes[int(str(mode.mode))] = str(mode.modeName) + return MODES[self._modes[int(str(modeid))]] + + +def main(): + """ + Usage: + cd ~/src/chirp.hg + python ./chirp/radioreference.py [ZIPCODE] [USERNAME] [PASSWORD] + """ + import sys + rrr = RadioReferenceRadio(None) + rrr.set_params(zip=sys.argv[1], username=sys.argv[2], password=sys.argv[3]) + rrr.do_fetch() + print rrr.get_raw_memory(0) + print rrr.get_memory(0) + +if __name__ == "__main__": + main() \ No newline at end of file diff -r 806b80ebe58e -r 309d2038f31c chirpui/mainapp.py --- a/chirpui/mainapp.py Tue Apr 10 10:00:18 2012 -0600 +++ b/chirpui/mainapp.py Tue Apr 10 10:00:30 2012 -0600 @@ -1,4 +1,5 @@ # Copyright 2008 Dan Smith dsmith@danplanet.com +# Copyright 2012 Tom Hayward tom@tomh.us # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -96,7 +97,7 @@ set_action_sensitive(i, eset is not None and not mmap_sens)
for i in ["export", "import", "close", "columns", "rbook", "rfinder", - "stock", "move_up", "move_dn", "exchange", + "stock", "move_up", "move_dn", "exchange", "radioreference", "cut", "copy", "paste", "delete", "viewdeveloper"]: set_action_sensitive(i, eset is not None)
@@ -872,6 +873,60 @@
self.window.set_cursor(None)
+ def do_radioreference_prompt(self): + fields = {"1Username" : (gtk.Entry(), lambda x: x), + "2Password" : (gtk.Entry(), lambda x: x), + "3Zipcode" : (gtk.Entry(), lambda x: x), + } + + d = inputdialog.FieldDialog(title="RadioReference.com Query", parent=self) + for k in sorted(fields.keys()): + d.add_field(k[1:], fields[k][0]) + fields[k][0].set_text(CONF.get(k[1:], "radioreference") or "") + fields[k][0].set_visibility(k != "2Password") + + while d.run() == gtk.RESPONSE_OK: + valid = True + for k in sorted(fields.keys()): + widget, validator = fields[k] + try: + if validator(widget.get_text()): + CONF.set(k[1:], widget.get_text(), "radioreference") + continue + except Exception: + pass + common.show_error("Invalid value for %s" % k[1:]) + valid = False + break + + if valid: + d.destroy() + return True + + d.destroy() + return False + + def do_radioreference(self): + self.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.WATCH)) + if not self.do_radioreference_prompt(): + self.window.set_cursor(None) + return + + username = CONF.get("Username", "radioreference") + passwd = CONF.get("Password", "radioreference") + zipcode = CONF.get("Zipcode", "radioreference") + + # Do this in case the import process is going to take a while + # to make sure we process events leading up to this + gtk.gdk.window_process_all_updates() + while gtk.events_pending(): + gtk.main_iteration(False) + + eset = self.get_current_editorset() + count = eset.do_import("radioreference://%s/%s/%s" % (zipcode, username, passwd)) + + self.window.set_cursor(None) + def do_export(self): types = [(_("CSV Files") + " (*.csv)", "csv"), (_("CHIRP Files") + " (*.chirp)", "chirp"), @@ -1079,6 +1134,8 @@ self.do_import() elif action == "rfinder": self.do_rfinder() + elif action == "radioreference": + self.do_radioreference() elif action == "export": self.do_export() elif action == "rbook": @@ -1153,6 +1210,7 @@ <menuitem action="upload"/> <menuitem action="rbook"/> <menuitem action="rfinder"/> + <menuitem action="radioreference"/> <menu action="stock" name="stock"/> <separator/> <menuitem action="autorpt"/> @@ -1197,6 +1255,7 @@ ('upload', None, _("Upload To Radio"), "<Alt>u", None, self.mh), ('import', None, _("Import"), "<Alt>i", None, self.mh), ('export', None, _("Export"), "<Alt>x", None, self.mh), + ('radioreference', None, _("Import from RadioReference.com"), None, None, self.mh), ('rfinder', None, _("Import from RFinder"), None, None, self.mh), ('export_chirp', None, _("CHIRP Native File"), None, None, self.mh), ('export_csv', None, _("CSV File"), None, None, self.mh),
Sweet.
When I run against 97124, I get:
I did see it fetching a bunch of categories and stuff in the console though.
It takes almost as long to fetch my zipcode as it does to do a simple RFinder query. Granted, it's doing a LOT more than RFinder, but it still causes my window to dim. Since it's fetching things, we should work on trying to make a throbbing progress bar of some sort. With RFinder, we're just waiting for a single http request to complete, so we don't have anything to drive a throbber naturally, but I think we can do better for this. It doesn't need to gate inclusion, but is something to think about, perhaps in one of your follow-on patches.
You will want to use chirp_common.parse_freq() here, which takes care not to propagate the inevitable error to be had by storing the frequencies in IEEE floating point format. Otherwise, you'll get some issues with 6.25kHz (at least) channels here.
I'll get going on trying to get suds into the windows runtime (ugh).
On Tue, Apr 10, 2012 at 16:38, Dan Smith dsmith@danplanet.com wrote:
Ah, so mem.name expects "" for an empty name. Fixed.
I didn't use chirp_common.parse_freq() because it requires a str, and I have a float. But that's fine, I'll just convert to str before calling chirp_common.parse_freq().
New patch incoming.
Tom KD7LXL
participants (2)
-
Dan Smith -
Tom Hayward