[chirp_devel] [PATCH] [csv] Optimize generic csv files load times. Fixes #8991
# HG changeset patch # User Kosta A. ve7kcy@gmail.com # Date 1618643404 25200 # Sat Apr 17 00:10:04 2021 -0700 # Node ID cd3e2444040876b4a19b41c6cfecedb79ff4a8fe # Parent 42c5ae551170c33b04f997e3a07a9e1857bdb504 [csv] Optimize generic csv files load times. Fixes #8991
diff --git a/chirp/chirp_common.py b/chirp/chirp_common.py --- a/chirp/chirp_common.py +++ b/chirp/chirp_common.py @@ -268,11 +268,11 @@ # or an empty list if none extra = []
- def __init__(self): + def __init__(self, number=0, empty=False, name=""): self.freq = 0 - self.number = 0 + self.number = number self.extd_number = "" - self.name = "" + self.name = name self.vfo = 0 self.rtone = 88.5 self.ctone = 88.5 @@ -290,7 +290,7 @@
self.comment = ""
- self.empty = False + self.empty = empty
self.immutable = []
diff --git a/chirp/drivers/generic_csv.py b/chirp/drivers/generic_csv.py --- a/chirp/drivers/generic_csv.py +++ b/chirp/drivers/generic_csv.py @@ -47,7 +47,7 @@
@directory.register -class CSVRadio(chirp_common.FileBackedRadio, chirp_common.IcomDstarSupport): +class CSVRadio(chirp_common.FileBackedRadio): """A driver for Generic CSV files""" VENDOR = "Generic" MODEL = "CSV" @@ -67,20 +67,15 @@ "Mode": (str, "mode"), "TStep": (float, "tuning_step"), "Skip": (str, "skip"), - "URCALL": (str, "dv_urcall"), - "RPT1CALL": (str, "dv_rpt1call"), - "RPT2CALL": (str, "dv_rpt2call"), "Comment": (str, "comment"), }
- def _blank(self): + def _blank(self, defaults=False): self.errors = [] - self.memories = [] - for i in range(0, 1000): - mem = chirp_common.Memory() - mem.number = i - mem.empty = True - self.memories.append(mem) + self.memories = [chirp_common.Memory(i, True) for i in range(0, 1000)] + if (defaults): + self.memories[0].empty = False + self.memories[0].freq = 146010000
def __init__(self, pipe): chirp_common.FileBackedRadio.__init__(self, None) @@ -92,14 +87,14 @@ if self._filename and os.path.exists(self._filename): self.load() else: - self._blank() + self._blank(True)
def get_features(self): rf = chirp_common.RadioFeatures() rf.has_bank = False rf.requires_call_lists = False rf.has_implicit_calls = False - rf.memory_bounds = (0, len(self.memories)) + rf.memory_bounds = (0, len(self.memories)-1) rf.has_infinite_number = True rf.has_nostep_tuning = True rf.has_comment = True diff --git a/chirp/drivers/ic2200.py b/chirp/drivers/ic2200.py --- a/chirp/drivers/ic2200.py +++ b/chirp/drivers/ic2200.py @@ -217,7 +217,6 @@ return mem
def get_memories(self, lo=0, hi=199): - return [m for m in self._memories if m.number >= lo and m.number <= hi]
def set_memory(self, mem): diff --git a/chirp/ui/bandplans.py b/chirp/ui/bandplans.py --- a/chirp/ui/bandplans.py +++ b/chirp/ui/bandplans.py @@ -47,7 +47,7 @@ # Check for duplicates. duplicates = [x for x in plan.BANDS if x == band] if len(duplicates) > 1: - LOG.warn("Bandplan %s has duplicates %s" % + LOG.info("Bandplan %s has duplicates %s" % (name, duplicates)) # Add repeater inputs. rpt_input = band.inverse() diff --git a/chirp/ui/editorset.py b/chirp/ui/editorset.py --- a/chirp/ui/editorset.py +++ b/chirp/ui/editorset.py @@ -386,19 +386,6 @@ "export: {error}").format(error=e), self)
- def prime(self): - # NOTE: this is only called to prime new CSV files, so assume - # only one memory editor for now - mem = chirp_common.Memory() - mem.freq = 146010000 - - def cb(*args): - gobject.idle_add(self.editors["memedit0"].prefill) - - job = common.RadioJob(cb, "set_memory", mem) - job.set_desc(_("Priming memory")) - self.rthread.submit(job) - def tab_selected(self, notebook, foo, pagenum): widget = notebook.get_nth_page(pagenum) for k, v in self.editors.items(): diff --git a/chirp/ui/mainapp.py b/chirp/ui/mainapp.py --- a/chirp/ui/mainapp.py +++ b/chirp/ui/mainapp.py @@ -271,7 +271,6 @@ def do_new(self): eset = editorset.EditorSet(_("Untitled") + ".csv", self) self._connect_editorset(eset) - eset.prime() eset.show()
tab = self.tabs.append_page(eset, eset.get_tab_label()) diff --git a/chirp/ui/memedit.py b/chirp/ui/memedit.py --- a/chirp/ui/memedit.py +++ b/chirp/ui/memedit.py @@ -1062,10 +1062,7 @@ if not mem.empty or self.show_empty: gobject.idle_add(self.set_memory, mem) else: - mem = chirp_common.Memory() - mem.number = number - mem.name = "ERROR" - mem.empty = True + mem = chirp_common.Memory(number, True, "Error") gobject.idle_add(self.set_memory, mem)
for i in range(lo, hi+1):
Resending patch for consideration.
On Sat, Apr 17, 2021 at 12:11 AM Kosta A. ve7kcy@gmail.com wrote:
# HG changeset patch # User Kosta A. ve7kcy@gmail.com # Date 1618643404 25200 # Sat Apr 17 00:10:04 2021 -0700 # Node ID cd3e2444040876b4a19b41c6cfecedb79ff4a8fe # Parent 42c5ae551170c33b04f997e3a07a9e1857bdb504 [csv] Optimize generic csv files load times. Fixes #8991
diff --git a/chirp/chirp_common.py b/chirp/chirp_common.py --- a/chirp/chirp_common.py +++ b/chirp/chirp_common.py @@ -268,11 +268,11 @@ # or an empty list if none extra = []
- def __init__(self):
- def __init__(self, number=0, empty=False, name=""): self.freq = 0
self.number = 0
self.number = number self.extd_number = ""
self.name = ""
self.name = name self.vfo = 0 self.rtone = 88.5 self.ctone = 88.5@@ -290,7 +290,7 @@
self.comment = ""
self.empty = False
self.empty = empty self.immutable = []diff --git a/chirp/drivers/generic_csv.py b/chirp/drivers/generic_csv.py --- a/chirp/drivers/generic_csv.py +++ b/chirp/drivers/generic_csv.py @@ -47,7 +47,7 @@
@directory.register -class CSVRadio(chirp_common.FileBackedRadio, chirp_common.IcomDstarSupport): +class CSVRadio(chirp_common.FileBackedRadio): """A driver for Generic CSV files""" VENDOR = "Generic" MODEL = "CSV" @@ -67,20 +67,15 @@ "Mode": (str, "mode"), "TStep": (float, "tuning_step"), "Skip": (str, "skip"),
"URCALL": (str, "dv_urcall"),"RPT1CALL": (str, "dv_rpt1call"),"RPT2CALL": (str, "dv_rpt2call"), "Comment": (str, "comment"), }def _blank(self):
- def _blank(self, defaults=False): self.errors = []
self.memories = []for i in range(0, 1000):mem = chirp_common.Memory()mem.number = imem.empty = Trueself.memories.append(mem)
self.memories = [chirp_common.Memory(i, True) for i in range(0,1000)]
if (defaults):self.memories[0].empty = Falseself.memories[0].freq = 146010000def __init__(self, pipe): chirp_common.FileBackedRadio.__init__(self, None)
@@ -92,14 +87,14 @@ if self._filename and os.path.exists(self._filename): self.load() else:
self._blank()
self._blank(True)def get_features(self): rf = chirp_common.RadioFeatures() rf.has_bank = False rf.requires_call_lists = False rf.has_implicit_calls = False
rf.memory_bounds = (0, len(self.memories))
rf.memory_bounds = (0, len(self.memories)-1) rf.has_infinite_number = True rf.has_nostep_tuning = True rf.has_comment = Truediff --git a/chirp/drivers/ic2200.py b/chirp/drivers/ic2200.py --- a/chirp/drivers/ic2200.py +++ b/chirp/drivers/ic2200.py @@ -217,7 +217,6 @@ return mem
def get_memories(self, lo=0, hi=199):
return [m for m in self._memories if m.number >= lo and m.number<= hi]
def set_memory(self, mem):diff --git a/chirp/ui/bandplans.py b/chirp/ui/bandplans.py --- a/chirp/ui/bandplans.py +++ b/chirp/ui/bandplans.py @@ -47,7 +47,7 @@ # Check for duplicates. duplicates = [x for x in plan.BANDS if x == band] if len(duplicates) > 1:
LOG.warn("Bandplan %s has duplicates %s" %
LOG.info("Bandplan %s has duplicates %s" % (name, duplicates)) # Add repeater inputs. rpt_input = band.inverse()diff --git a/chirp/ui/editorset.py b/chirp/ui/editorset.py --- a/chirp/ui/editorset.py +++ b/chirp/ui/editorset.py @@ -386,19 +386,6 @@ "export: {error}").format(error=e), self)
- def prime(self):
# NOTE: this is only called to prime new CSV files, so assume# only one memory editor for nowmem = chirp_common.Memory()mem.freq = 146010000def cb(*args):gobject.idle_add(self.editors["memedit0"].prefill)job = common.RadioJob(cb, "set_memory", mem)job.set_desc(_("Priming memory"))self.rthread.submit(job)- def tab_selected(self, notebook, foo, pagenum): widget = notebook.get_nth_page(pagenum) for k, v in self.editors.items():
diff --git a/chirp/ui/mainapp.py b/chirp/ui/mainapp.py --- a/chirp/ui/mainapp.py +++ b/chirp/ui/mainapp.py @@ -271,7 +271,6 @@ def do_new(self): eset = editorset.EditorSet(_("Untitled") + ".csv", self) self._connect_editorset(eset)
eset.prime() eset.show() tab = self.tabs.append_page(eset, eset.get_tab_label())diff --git a/chirp/ui/memedit.py b/chirp/ui/memedit.py --- a/chirp/ui/memedit.py +++ b/chirp/ui/memedit.py @@ -1062,10 +1062,7 @@ if not mem.empty or self.show_empty: gobject.idle_add(self.set_memory, mem) else:
mem = chirp_common.Memory()mem.number = numbermem.name = "ERROR"mem.empty = True
mem = chirp_common.Memory(number, True, "Error") gobject.idle_add(self.set_memory, mem) for i in range(lo, hi+1):
# HG changeset patch # User Kosta A. ve7kcy@gmail.com # Date 1618643404 25200 # Sat Apr 17 00:10:04 2021 -0700 # Node ID cd3e2444040876b4a19b41c6cfecedb79ff4a8fe # Parent 42c5ae551170c33b04f997e3a07a9e1857bdb504 [csv] Optimize generic csv files load times. Fixes #8991
This patch does this, but a lot more. I'd like some more words here about what it does for speedups, and splitting out the other parts of the change. If you can turn this around quickly, I'll commit to getting it in this weekend, so you don't have to wait again. If you can't, let me know and I'll do it for you.
- def __init__(self):
- def __init__(self, number=0, empty=False, name=""): self.freq = 0
self.number = 0
self.number = number self.extd_number = ""
self.name = ""
self.name = name self.vfo = 0 self.rtone = 88.5 self.ctone = 88.5@@ -290,7 +290,7 @@
self.comment = ""
self.empty = False
self.empty = empty
This is fine, but how about we just go with something like this to make it even more flexible?
def __init__(self, **attrs): self.freq = 0 self.number = 0 self.extd_number = "" # ... etc for k, v in attrs.items(): setattr(self, k, v)
Then you can initialize any of the fields in Memory when you create it, which will be nicer all around.
@directory.register -class CSVRadio(chirp_common.FileBackedRadio, chirp_common.IcomDstarSupport): +class CSVRadio(chirp_common.FileBackedRadio): """A driver for Generic CSV files""" VENDOR = "Generic" MODEL = "CSV" @@ -67,20 +67,15 @@ "Mode": (str, "mode"), "TStep": (float, "tuning_step"), "Skip": (str, "skip"),
"URCALL": (str, "dv_urcall"),"RPT1CALL": (str, "dv_rpt1call"),"RPT2CALL": (str, "dv_rpt2call"), "Comment": (str, "comment"), }
This is a pretty major change that has nothing to do with the speed improvements, removes functionality, and isn't even mentioned in the commit message. I don't really mind dropping the DstarSupport from this because I don't think it's useful for anything other than the first generation radios, and wasn't really that useful then anyway. But, let's please make it a separate commit so that it stands alone.
- def _blank(self):
- def _blank(self, defaults=False): self.errors = []
self.memories = []for i in range(0, 1000):mem = chirp_common.Memory()mem.number = imem.empty = Trueself.memories.append(mem)
self.memories = [chirp_common.Memory(i, True) for i in range(0, 1000)]if (defaults):self.memories[0].empty = Falseself.memories[0].freq = 146010000
I expect this listcomp is the bulk of the intended performance improvement here, right?
To be honest, I can't really tell a difference in the load times on my machine. Does this patch make a noticeable improvement for you (and others)?
rf.memory_bounds = (0, len(self.memories))
rf.memory_bounds = (0, len(self.memories)-1)
This is unrelated to performance I'm sure. Is this really a bug, or does it matter?
diff --git a/chirp/drivers/ic2200.py b/chirp/drivers/ic2200.py --- a/chirp/drivers/ic2200.py +++ b/chirp/drivers/ic2200.py @@ -217,7 +217,6 @@ return mem
def get_memories(self, lo=0, hi=199):
Big performance gain on this one? :)
Please remove this random whitespace damage.
def set_memory(self, mem):diff --git a/chirp/ui/bandplans.py b/chirp/ui/bandplans.py --- a/chirp/ui/bandplans.py +++ b/chirp/ui/bandplans.py @@ -47,7 +47,7 @@ # Check for duplicates. duplicates = [x for x in plan.BANDS if x == band] if len(duplicates) > 1:
LOG.warn("Bandplan %s has duplicates %s" %
LOG.info("Bandplan %s has duplicates %s" % (name, duplicates))
Also unrelated to performance, and not a good improvement, IMHO, as it just makes this more chatty. Was this intentional?
--- a/chirp/ui/memedit.py +++ b/chirp/ui/memedit.py @@ -1062,10 +1062,7 @@ if not mem.empty or self.show_empty: gobject.idle_add(self.set_memory, mem) else:
mem = chirp_common.Memory()mem.number = numbermem.name = "ERROR"mem.empty = True
mem = chirp_common.Memory(number, True, "Error")
This is okay, but presumably just a cleanup, right?
Thanks!
--Dan
Thanks for taking the time to review the change Dan. I have posted my comments inline below, I will attempt to turn around a commit quickly so as not to take up more of your time.
-kosta
On Fri, Jun 11, 2021 at 4:59 PM Dan Smith via chirp_devel < chirp_devel@intrepid.danplanet.com> wrote:
# HG changeset patch # User Kosta A. ve7kcy@gmail.com # Date 1618643404 25200 # Sat Apr 17 00:10:04 2021 -0700 # Node ID cd3e2444040876b4a19b41c6cfecedb79ff4a8fe # Parent 42c5ae551170c33b04f997e3a07a9e1857bdb504 [csv] Optimize generic csv files load times. Fixes #8991
This patch does this, but a lot more. I'd like some more words here about what it does for speedups, and splitting out the other parts of the change. If you can turn this around quickly, I'll commit to getting it in this weekend, so you don't have to wait again. If you can't, let me know and I'll do it for you.
- def __init__(self):
- def __init__(self, number=0, empty=False, name=""): self.freq = 0
self.number = 0
self.number = number self.extd_number = ""
self.name = ""
self.name = name self.vfo = 0 self.rtone = 88.5 self.ctone = 88.5@@ -290,7 +290,7 @@
self.comment = ""
self.empty = False
self.empty = emptyThis is fine, but how about we just go with something like this to make it even more flexible?
def __init__(self, **attrs): self.freq = 0 self.number = 0 self.extd_number = "" # ... etc for k, v in attrs.items(): setattr(self, k, v)
Then you can initialize any of the fields in Memory when you create it, which will be nicer all around.
This change was specifically made as an optimization as memories are
constructed in a rather tight loop; it was slightly faster to remove the double member assignment and replace it with a parametrized constructor. While a generic iterator would be flexible, I was not aiming for added flexibility here as much as reducing the overall number of instructions for the add perf.
@directory.register -class CSVRadio(chirp_common.FileBackedRadio,
chirp_common.IcomDstarSupport):
+class CSVRadio(chirp_common.FileBackedRadio): """A driver for Generic CSV files""" VENDOR = "Generic" MODEL = "CSV" @@ -67,20 +67,15 @@ "Mode": (str, "mode"), "TStep": (float, "tuning_step"), "Skip": (str, "skip"),
"URCALL": (str, "dv_urcall"),"RPT1CALL": (str, "dv_rpt1call"),"RPT2CALL": (str, "dv_rpt2call"), "Comment": (str, "comment"), }This is a pretty major change that has nothing to do with the speed improvements, removes functionality, and isn't even mentioned in the commit message. I don't really mind dropping the DstarSupport from this because I don't think it's useful for anything other than the first generation radios, and wasn't really that useful then anyway. But, let's please make it a separate commit so that it stands alone.
You are correct about this change, I can separate it out np.
- def _blank(self):
- def _blank(self, defaults=False): self.errors = []
self.memories = []for i in range(0, 1000):mem = chirp_common.Memory()mem.number = imem.empty = Trueself.memories.append(mem)
self.memories = [chirp_common.Memory(i, True) for i in range(0,1000)]
if (defaults):self.memories[0].empty = Falseself.memories[0].freq = 146010000I expect this listcomp is the bulk of the intended performance improvement here, right?
To be honest, I can't really tell a difference in the load times on my machine. Does this patch make a noticeable improvement for you (and others)?
The 'bulk' of the realized optimization stems from the removal of the
redundant call to prefill - which is best visualized without a profiler when 'show empty' is enabled as the list will typically flash twice during the subsequent removal and re-addition of 1k items.
The list comprehension + the explicitly parameterized memory constructor simply aid in the optimization but do require profiling to measure the impact. I will post my breakdown of perf. timings with the follow up commit as I do not have them in front of me at the moment.
rf.memory_bounds = (0, len(self.memories))
rf.memory_bounds = (0, len(self.memories)-1)This is unrelated to performance I'm sure. Is this really a bug, or does it matter?
This is a bug and shows up when you have empty memories enabled, in which
the memories start a 1 instead of 0.
diff --git a/chirp/drivers/ic2200.py b/chirp/drivers/ic2200.py --- a/chirp/drivers/ic2200.py +++ b/chirp/drivers/ic2200.py @@ -217,7 +217,6 @@ return mem
def get_memories(self, lo=0, hi=199):
Big performance gain on this one? :)
Please remove this random whitespace damage.
Ack.
def set_memory(self, mem):diff --git a/chirp/ui/bandplans.py b/chirp/ui/bandplans.py --- a/chirp/ui/bandplans.py +++ b/chirp/ui/bandplans.py @@ -47,7 +47,7 @@ # Check for duplicates. duplicates = [x for x in plan.BANDS if x == band] if len(duplicates) > 1:
LOG.warn("Bandplan %s has duplicates %s" %
LOG.info("Bandplan %s has duplicates %s" % (name, duplicates))Also unrelated to performance, and not a good improvement, IMHO, as it just makes this more chatty. Was this intentional?
Technically this was not intentional, however I always make this change
locally because it reduces console spew, in that warnings are echoed to stdout by default whereas infos are not. Probably not the best way to get rid of this spam, but it works for me; nonetheless I can revert.
--- a/chirp/ui/memedit.py +++ b/chirp/ui/memedit.py @@ -1062,10 +1062,7 @@ if not mem.empty or self.show_empty: gobject.idle_add(self.set_memory, mem) else:
mem = chirp_common.Memory()mem.number = numbermem.name = "ERROR"mem.empty = True
mem = chirp_common.Memory(number, True, "Error")This is okay, but presumably just a cleanup, right?
This was a combination of cleanup and subsequent taking advantage of the
optimization pattern mentioned previosly.
Thanks!
--Dan
chirp_devel mailing list chirp_devel@intrepid.danplanet.com http://intrepid.danplanet.com/mailman/listinfo/chirp_devel Developer docs: http://chirp.danplanet.com/projects/chirp/wiki/Developers
Technically this was not intentional, however I always make this change locally because it reduces console spew, in that warnings are echoed to stdout by default whereas infos are not. Probably not the best way to get rid of this spam, but it works for me; nonetheless I can revert.
I read the logic here wrong, I see why this is better, so I'm happy to apply that change, but on it's own.
--Dan
participants (3)
-
Dan Smith -
Kosta A. -
Kosta Arvanitis