# HG changeset patch # User Dan Smith dsmith@danplanet.com # Date 1533593320 25200 # Mon Aug 06 15:08:40 2018 -0700 # Node ID e8d26a9a98c56c6a8bed7967b6b2e089f9ac0b33 # Parent 4f6978a84b3e461b13c6e0f996d3bec058431b70 Add metadata blob trailer transparently to image files

This makes FileBackedRadio transparently save image files with an extra blob of metadata tacked onto the end of the file. The metadata region begins with a magic string of "chirp-img0001-" followed by a base64-encoded json blob. I figure we can call all images created up until now "version zero" format, since they really aren't formatted at all. Any major format changes after this can increment the counter in the magic (although it's unlikely we would need to).

The metadata currently saved includes the radio model details and the class used at the time the file was created. All existing images will still load, but when they are saved, will be augmented with metadata.

One major feature this brings is the ability to make radio aliases (and other weak subclasses) be "sticky" across save/load cycles. Right now if you download with an alias (such as the Arcshell AR-6 alias of the BF-888), the file will be detected and reported as a BF-888 from that point forward. With metadata in the image, we can properly select the alias on load to provide better UX.

This includes some low-effort tweaking of the directory and clone code to select the right radio class based on metadata if present, and to properly return a radio class for the alias, if one matches. We might want to re-structure the alias class arrangement after this to make them properly-registered and fully-functional subclasses so that the hacky DynamicRadioAlias subclassing done here isn't necessary.

Related to will-be-a-real-issue #0

diff -r 4f6978a84b3e -r e8d26a9a98c5 chirp/chirp_common.py --- a/chirp/chirp_common.py Sun Jul 01 17:52:10 2018 +0200 +++ b/chirp/chirp_common.py Mon Aug 06 15:08:40 2018 -0700 @@ -13,9 +13,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/.

+import base64 +import json +import logging import math from chirp import errors, memmap

+LOG = logging.getLogger(__name__) + SEPCHAR = ","

# 50 Tones @@ -1104,6 +1109,7 @@ class FileBackedRadio(Radio): """A file-backed radio stores its data in a file""" FILE_EXTENSION = "img" + MAGIC = 'chirp-img0001'

def __init__(self, *args, **kwargs): Radio.__init__(self, *args, **kwargs) @@ -1121,10 +1127,45 @@ """Process a newly-loaded or downloaded memory map""" pass

+ @classmethod + def _strip_metadata(cls, raw_data): + try: + idx = raw_data.index(cls.MAGIC) + except ValueError: + LOG.debug('Image data has no metadata blob') + return raw_data, {} + + # Find the beginning of the base64 blob + raw_metadata = raw_data[idx + len(cls.MAGIC) + 1:] + metadata = {} + try: + metadata = json.loads(base64.b64decode(raw_metadata)) + except ValueError as e: + LOG.error('Failed to parse decoded metadata blob: %s' % e) + except TypeError as e: + LOG.error('Failed to decode metadata blob: %s' % e) + + if metadata: + LOG.debug('Loaded metadata: %s' % metadata) + + return raw_data[:idx], metadata + + @classmethod + def _make_metadata(cls): + return base64.b64encode(json.dumps( + {'rclass': cls.__name__, + 'vendor': cls.VENDOR, + 'model': cls.MODEL, + 'variant': cls.VARIANT, + })) + def load_mmap(self, filename): """Load the radio's memory map from @filename""" mapfile = file(filename, "rb") - self._mmap = memmap.MemoryMap(mapfile.read()) + data = mapfile.read() + if self.MAGIC in data: + data, metadata = self._strip_metadata(data) + self._mmap = memmap.MemoryMap(data) mapfile.close() self.process_mmap()

@@ -1136,6 +1177,8 @@ try: mapfile = file(filename, "wb") mapfile.write(self._mmap.get_packed()) + mapfile.write(self.MAGIC + '-') + mapfile.write(self._make_metadata()) mapfile.close() except IOError: raise Exception("File Access Error") diff -r 4f6978a84b3e -r e8d26a9a98c5 chirp/directory.py --- a/chirp/directory.py Sun Jul 01 17:52:10 2018 +0200 +++ b/chirp/directory.py Mon Aug 06 15:08:40 2018 -0700 @@ -137,9 +137,26 @@ else: filedata = ""

+ data, metadata = chirp_common.FileBackedRadio._strip_metadata(filedata) + for rclass in DRV_TO_RADIO.values(): if not issubclass(rclass, chirp_common.FileBackedRadio): continue - if rclass.match_model(filedata, image_file): + + # If no metadata, we do the old thing + if not metadata and rclass.match_model(filedata, image_file): return rclass(image_file) + + # If metadata, then it has to match one of the aliases or the parent + for alias in rclass.ALIASES + [rclass]: + if (alias.VENDOR == metadata.get('vendor') and + alias.MODEL == metadata.get('model')): + + class DynamicRadioAlias(rclass): + VENDOR = metadata.get('vendor') + MODEL = metadata.get('model') + VARIANT = metadata.get('variant') + + return DynamicRadioAlias(image_file) + raise errors.ImageDetectFailed("Unknown file format") diff -r 4f6978a84b3e -r e8d26a9a98c5 chirp/ui/clone.py --- a/chirp/ui/clone.py Sun Jul 01 17:52:10 2018 +0200 +++ b/chirp/ui/clone.py Mon Aug 06 15:08:40 2018 -0700 @@ -191,9 +191,16 @@ for alias in rclass.ALIASES: if alias.MODEL == model: alias_match = rclass + alias_class = alias break if alias_match: - cs.radio_class = rclass + + class DynamicRadioAlias(rclass): + VENDOR = alias.VENDOR + MODEL = alias.MODEL + VARIANT = alias.VARIANT + + cs.radio_class = DynamicRadioAlias LOG.debug( 'Chose %s alias for %s because model %s selected' % ( alias_match, cs.radio_class, model)) diff -r 4f6978a84b3e -r e8d26a9a98c5 tests/unit/test_chirp_common.py --- a/tests/unit/test_chirp_common.py Sun Jul 01 17:52:10 2018 +0200 +++ b/tests/unit/test_chirp_common.py Mon Aug 06 15:08:40 2018 -0700 @@ -1,3 +1,10 @@ +import base64 +import json +import os +import tempfile + +import mock + from tests.unit import base from chirp import chirp_common from chirp import errors @@ -253,3 +260,84 @@ def test_fix_rounded_step_750(self): self.assertEqual(146118750, chirp_common.fix_rounded_step(146118000)) + + +class TestImageMetadata(base.BaseTest): + def test_make_metadata(self): + class TestRadio(chirp_common.FileBackedRadio): + VENDOR = 'Dan' + MODEL = 'Foomaster 9000' + VARIANT = 'R' + + raw_metadata = TestRadio._make_metadata() + metadata = json.loads(base64.b64decode(raw_metadata)) + expected = { + 'vendor': 'Dan', + 'model': 'Foomaster 9000', + 'variant': 'R', + 'rclass': 'TestRadio', + } + self.assertEqual(expected, metadata) + + def test_strip_metadata(self): + class TestRadio(chirp_common.FileBackedRadio): + VENDOR = 'Dan' + MODEL = 'Foomaster 9000' + VARIANT = 'R' + + raw_metadata = TestRadio._make_metadata() + raw_data = ('foooooooooooooooooooooo' + TestRadio.MAGIC + '-' + + TestRadio._make_metadata()) + data, metadata = chirp_common.FileBackedRadio._strip_metadata(raw_data) + self.assertEqual('foooooooooooooooooooooo', data) + expected = { + 'vendor': 'Dan', + 'model': 'Foomaster 9000', + 'variant': 'R', + 'rclass': 'TestRadio', + } + self.assertEqual(expected, metadata) + + def test_load_mmap_no_metadata(self): + f = tempfile.NamedTemporaryFile() + f.write('thisisrawdata') + f.flush() + + with mock.patch('chirp.memmap.MemoryMap') as mock_mmap: + chirp_common.FileBackedRadio(None).load_mmap(f.name) + mock_mmap.assert_called_once_with('thisisrawdata') + + def test_load_mmap_bad_metadata(self): + f = tempfile.NamedTemporaryFile() + f.write('thisisrawdata') + f.write(chirp_common.FileBackedRadio.MAGIC + '-' + 'bad') + f.flush() + + with mock.patch('chirp.memmap.MemoryMap') as mock_mmap: + chirp_common.FileBackedRadio(None).load_mmap(f.name) + mock_mmap.assert_called_once_with('thisisrawdata') + + def test_save_mmap_includes_metadata(self): + class TestRadio(chirp_common.FileBackedRadio): + VENDOR = 'Dan' + MODEL = 'Foomaster 9000' + VARIANT = 'R' + + with tempfile.NamedTemporaryFile() as f: + fn = f.name + r = TestRadio(None) + r._mmap = mock.Mock() + r._mmap.get_packed.return_value = 'thisisrawdata' + r.save_mmap(fn) + with file(fn) as f: + filedata = f.read() + os.remove(fn) + data, metadata = chirp_common.FileBackedRadio._strip_metadata(filedata) + self.assertEqual('thisisrawdata', data) + expected = { + 'vendor': 'Dan', + 'model': 'Foomaster 9000', + 'variant': 'R', + 'rclass': 'TestRadio', + } + self.assertEqual(expected, metadata) diff -r 4f6978a84b3e -r e8d26a9a98c5 tests/unit/test_directory.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/unit/test_directory.py Mon Aug 06 15:08:40 2018 -0700 @@ -0,0 +1,69 @@ +import base64 +import json +import tempfile + +from tests.unit import base +from chirp import chirp_common +from chirp import directory + + +class TestDirectory(base.BaseTest): + def setUp(self): + super(TestDirectory, self).setUp() + + directory.enable_reregistrations() + + class FakeAlias(chirp_common.Alias): + VENDOR = 'Taylor' + MODEL = 'Barmaster 2000' + VARIANT = 'A' + + @directory.register + class FakeRadio(chirp_common.FileBackedRadio): + VENDOR = 'Dan' + MODEL = 'Foomaster 9000' + VARIANT = 'R' + ALIASES = [FakeAlias] + + @classmethod + def match_model(cls, file_data, image_file): + return file_data == 'thisisrawdata' + + self.test_class = FakeRadio + + def _test_detect_finds_our_class(self, tempfn): + radio = directory.get_radio_by_image(tempfn) + self.assertTrue(isinstance(radio, self.test_class)) + return radio + + def test_detect_with_no_metadata(self): + with tempfile.NamedTemporaryFile() as f: + f.write('thisisrawdata') + f.flush() + self._test_detect_finds_our_class(f.name) + + def test_detect_with_metadata_base_class(self): + with tempfile.NamedTemporaryFile() as f: + f.write('thisisrawdata') + f.write(self.test_class.MAGIC + '-') + f.write(self.test_class._make_metadata()) + f.flush() + self._test_detect_finds_our_class(f.name) + + def test_detect_with_metadata_alias_class(self): + with tempfile.NamedTemporaryFile() as f: + f.write('thisisrawdata') + f.write(self.test_class.MAGIC + '-') + FakeAlias = self.test_class.ALIASES[0] + fake_metadata = base64.b64encode(json.dumps( + {'vendor': FakeAlias.VENDOR, + 'model': FakeAlias.MODEL, + 'variant': FakeAlias.VARIANT, + })) + f.write(fake_metadata) + f.flush() + radio = self._test_detect_finds_our_class(f.name) + self.assertEqual('Taylor', radio.VENDOR) + self.assertEqual('Barmaster 2000', radio.MODEL) + self.assertEqual('A', radio.VARIANT) +