[PATCH] Add support for lz4 compression in .lz4, .tar.lz4 and .tlz4 files

[Mike Kazantsev]

Hi,

I've been using lz4 for img files locally, as it provides marginally
better compression than lzo with still better performance than gzip,
and wanting to use bmap-tools recently, found that it's not supported
there yet.


Notes:

- Patch is for https://github.com/tizenorg/tools.bmap-tools master
  (3.3 release).

- Following files might also need changes, but not sure if I should be
  doing that:

  - docs/RELEASE_NOTES
  - debian/changelog
  - debian/control
  - packaging/bmap-tools.spec
  - packaging/bmap-tools.changes

- Basic approach for adding the feature was to "grep -ri lzo *" and
  copy-paste these blocks for lz4, as it has exactly same interface for
  cli tool (checked).

- tar doesn't have --lz4 option, but -Ilz4 works, as again, same cli as
  with all compression binaries.

- Also available at https://github.com/mk-fg/tools.bmap-tools/commit/0afae2c1ed

- Both nose tests and actual copy from .lz4 seem to succeed.


Would be cool to see such support upstream.

Please let me know if any changes might be needed, and/or if it's not
really worth supporting lz4 (no idea how un-/popular it might be).

Thanks!


---
 bmaptools/TransRead.py | 34 +++++++++++++++++++++++++++++-----
 docs/man1/bmaptool.1   |  3 +++
 tests/test_api_base.py |  4 +++-
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/bmaptools/TransRead.py b/bmaptools/TransRead.py
index 6d14260..e8ca8cc 100644
--- a/bmaptools/TransRead.py
+++ b/bmaptools/TransRead.py
@@ -14,9 +14,9 @@ This module allows opening and reading local and remote files and decompress
 them on-the-fly if needed. Remote files are read using urllib2 (except of
 "ssh://" URLs, which are handled differently). Supported file extentions are:
 'bz2', 'gz', 'xz', 'lzo' and a "tar" version of them: 'tar.bz2', 'tbz2', 'tbz',
-'tb2', 'tar.gz', 'tgz', 'tar.xz', 'txz', 'tar.lzo', 'tzo'. This module uses
-the following system programs for decompressing: pbzip2, bzip2, gzip, pigz, xz,
-lzop, and tar.
+'tb2', 'tar.gz', 'tgz', 'tar.xz', 'txz', 'tar.lzo', 'tzo', 'tar.lz4', 'tlz4'.
+This module uses the following system programs for decompressing: pbzip2, bzip2,
+gzip, pigz, xz, lzop, lz4 and tar.
 """
 
 import os
@@ -42,8 +42,8 @@ from bmaptools import BmapHelpers
 # pylint: disable=R0915
 
 # A list of supported compression types
-SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'xz', 'lzo', 'tar.gz', 'tar.bz2',
-                               'tar.xz', 'tar.lzo')
+SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'xz', 'lzo', 'lz4', 'tar.gz',
+                               'tar.bz2', 'tar.xz', 'tar.lzo')
 
 def _fake_seek_forward(file_obj, cur_pos, offset, whence=os.SEEK_SET):
     """
@@ -239,6 +239,12 @@ class TransRead(object):
                 return True
             return False
 
+        def is_lz4(name):
+            """Returns 'True' if file 'name' is compressed with 'lz4'."""
+            if name.endswith('.lz4') and not name.endswith('.tar.lz4'):
+                return True
+            return False
+
         def is_tar_gz(name):
             """
             Returns 'True' if file 'name' is a tar archive compressed with
@@ -279,6 +285,16 @@ class TransRead(object):
                 return True
             return False
 
+        def is_tar_lz4(name):
+            """
+            Returns 'True' if file 'name' is a tar archive compressed with
+            'lz4'.
+            """
+
+            if name.endswith('.tar.lz4') or name.endswith('.tlz4'):
+                return True
+            return False
+
         archiver = None
         if is_tar_gz(self.name) or is_gzip(self.name):
             self.compression_type = 'gzip'
@@ -320,6 +336,14 @@ class TransRead(object):
             else:
                 archiver = "tar"
                 args = "-x --lzo -O"
+        elif is_tar_lz4(self.name) or is_lz4(self.name):
+            self.compression_type = 'lz4'
+            decompressor = "lz4"
+            if is_lz4(self.name):
+                args = "-d -c"
+            else:
+                archiver = "tar"
+                args = "-x -Ilz4 -O"
         else:
             if not self.is_url:
                 self.size = os.fstat(self._f_objs[-1].fileno()).st_size
diff --git a/docs/man1/bmaptool.1 b/docs/man1/bmaptool.1
index 1d86bac..e585ac4 100644
--- a/docs/man1/bmaptool.1
+++ b/docs/man1/bmaptool.1
@@ -106,6 +106,9 @@ extensions are supported:
 .RS 4
 4. ".lzo", "tar.lzo", ".tzo" for files and tar archives compressed with "\fIlzo\fR" program
 .RE
+.RS 4
+4. ".lz4", "tar.lz4", ".tlz4" for files and tar archives compressed with "\fIlz4\fR" program
+.RE
 
 .PP
 IMAGE files with other extensions are assumed to be uncompressed. Note,
diff --git a/tests/test_api_base.py b/tests/test_api_base.py
index 934ee5f..6d748f7 100644
--- a/tests/test_api_base.py
+++ b/tests/test_api_base.py
@@ -84,12 +84,14 @@ def _generate_compressed_files(file_path, delete=True):
                    ("pigz",   None, ".p.gz",  "-c -k"),
                    ("xz",     None, ".xz",    "-c -k"),
                    ("lzop",   None, ".lzo",   "-c -k"),
+                   ("lz4",   None, ".lz4",   "-c -k"),
                    # The "-P -C /" trick is used to avoid silly warnings:
                    # "tar: Removing leading `/' from member names"
                    ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"),
                    ("gzip",  "tar", ".tar.gz",  "-c -z -O -P -C /"),
                    ("xz",    "tar", ".tar.xz",  "-c -J -O -P -C /"),
-                   ("lzop",  "tar", ".tar.lzo", "-c --lzo -O -P -C /")]
+                   ("lzop",  "tar", ".tar.lzo", "-c --lzo -O -P -C /"),
+                   ("lz4",  "tar", ".tar.lz4", "-c -Ilz4 -O -P -C /")]
 
     for decompressor, archiver, suffix, options in compressors:
         if not BmapHelpers.program_is_available(decompressor):
-- 
2.9.2


-- 
Mike Kazantsev // fraggod.net