[PATCH] Add support for lz4 compression in .lz4, .tar.lz4 and .tlz4 files
Artem Bityutskiy
dedekind1 at gmail.com
Thu Aug 25 07:08:08 PDT 2016
Hi Mike,
Thanks for the patch. This project moved to github and Alexander is
taking care of it now. We should do better job making this clear to the
people. Sorry for that!
This is the project:
https://github.com/01org/bmap-tools
The code-base at infradead.org is out of date. I'll push a commit there
with a readme file telling about this so that people wouldn't spend
their time with old code.
On Thu, 2016-08-25 at 18:06 +0500, Mike Kazantsev wrote:
> Hi,
>
> I've been using lz4 for img files locally, as it provides marginally
> better compression than lzo with still better performance than gzip,
> and wanting to use bmap-tools recently, found that it's not supported
> there yet.
>
>
> Notes:
>
> - Patch is for https://github.com/tizenorg/tools.bmap-tools master
> (3.3 release).
>
> - Following files might also need changes, but not sure if I should
> be
> doing that:
>
> - docs/RELEASE_NOTES
> - debian/changelog
> - debian/control
> - packaging/bmap-tools.spec
> - packaging/bmap-tools.changes
>
> - Basic approach for adding the feature was to "grep -ri lzo *" and
> copy-paste these blocks for lz4, as it has exactly same interface
> for
> cli tool (checked).
>
> - tar doesn't have --lz4 option, but -Ilz4 works, as again, same cli
> as
> with all compression binaries.
>
> - Also available at https://github.com/mk-fg/tools.bmap-tools/commit/
> 0afae2c1ed
>
> - Both nose tests and actual copy from .lz4 seem to succeed.
>
>
> Would be cool to see such support upstream.
>
> Please let me know if any changes might be needed, and/or if it's not
> really worth supporting lz4 (no idea how un-/popular it might be).
>
> Thanks!
>
>
> ---
> bmaptools/TransRead.py | 34 +++++++++++++++++++++++++++++-----
> docs/man1/bmaptool.1 | 3 +++
> tests/test_api_base.py | 4 +++-
> 3 files changed, 35 insertions(+), 6 deletions(-)
>
> diff --git a/bmaptools/TransRead.py b/bmaptools/TransRead.py
> index 6d14260..e8ca8cc 100644
> --- a/bmaptools/TransRead.py
> +++ b/bmaptools/TransRead.py
> @@ -14,9 +14,9 @@ This module allows opening and reading local and
> remote files and decompress
> them on-the-fly if needed. Remote files are read using urllib2
> (except of
> "ssh://" URLs, which are handled differently). Supported file
> extentions are:
> 'bz2', 'gz', 'xz', 'lzo' and a "tar" version of them: 'tar.bz2',
> 'tbz2', 'tbz',
> -'tb2', 'tar.gz', 'tgz', 'tar.xz', 'txz', 'tar.lzo', 'tzo'. This
> module uses
> -the following system programs for decompressing: pbzip2, bzip2,
> gzip, pigz, xz,
> -lzop, and tar.
> +'tb2', 'tar.gz', 'tgz', 'tar.xz', 'txz', 'tar.lzo', 'tzo',
> 'tar.lz4', 'tlz4'.
> +This module uses the following system programs for decompressing:
> pbzip2, bzip2,
> +gzip, pigz, xz, lzop, lz4 and tar.
> """
>
> import os
> @@ -42,8 +42,8 @@ from bmaptools import BmapHelpers
> # pylint: disable=R0915
>
> # A list of supported compression types
> -SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'xz', 'lzo', 'tar.gz',
> 'tar.bz2',
> - 'tar.xz', 'tar.lzo')
> +SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'xz', 'lzo', 'lz4',
> 'tar.gz',
> + 'tar.bz2', 'tar.xz', 'tar.lzo')
>
> def _fake_seek_forward(file_obj, cur_pos, offset,
> whence=os.SEEK_SET):
> """
> @@ -239,6 +239,12 @@ class TransRead(object):
> return True
> return False
>
> + def is_lz4(name):
> + """Returns 'True' if file 'name' is compressed with
> 'lz4'."""
> + if name.endswith('.lz4') and not
> name.endswith('.tar.lz4'):
> + return True
> + return False
> +
> def is_tar_gz(name):
> """
> Returns 'True' if file 'name' is a tar archive
> compressed with
> @@ -279,6 +285,16 @@ class TransRead(object):
> return True
> return False
>
> + def is_tar_lz4(name):
> + """
> + Returns 'True' if file 'name' is a tar archive
> compressed with
> + 'lz4'.
> + """
> +
> + if name.endswith('.tar.lz4') or name.endswith('.tlz4'):
> + return True
> + return False
> +
> archiver = None
> if is_tar_gz(self.name) or is_gzip(self.name):
> self.compression_type = 'gzip'
> @@ -320,6 +336,14 @@ class TransRead(object):
> else:
> archiver = "tar"
> args = "-x --lzo -O"
> + elif is_tar_lz4(self.name) or is_lz4(self.name):
> + self.compression_type = 'lz4'
> + decompressor = "lz4"
> + if is_lz4(self.name):
> + args = "-d -c"
> + else:
> + archiver = "tar"
> + args = "-x -Ilz4 -O"
> else:
> if not self.is_url:
> self.size = os.fstat(self._f_objs[-
> 1].fileno()).st_size
> diff --git a/docs/man1/bmaptool.1 b/docs/man1/bmaptool.1
> index 1d86bac..e585ac4 100644
> --- a/docs/man1/bmaptool.1
> +++ b/docs/man1/bmaptool.1
> @@ -106,6 +106,9 @@ extensions are supported:
> .RS 4
> 4. ".lzo", "tar.lzo", ".tzo" for files and tar archives compressed
> with "\fIlzo\fR" program
> .RE
> +.RS 4
> +4. ".lz4", "tar.lz4", ".tlz4" for files and tar archives compressed
> with "\fIlz4\fR" program
> +.RE
>
> .PP
> IMAGE files with other extensions are assumed to be uncompressed.
> Note,
> diff --git a/tests/test_api_base.py b/tests/test_api_base.py
> index 934ee5f..6d748f7 100644
> --- a/tests/test_api_base.py
> +++ b/tests/test_api_base.py
> @@ -84,12 +84,14 @@ def _generate_compressed_files(file_path,
> delete=True):
> ("pigz", None, ".p.gz", "-c -k"),
> ("xz", None, ".xz", "-c -k"),
> ("lzop", None, ".lzo", "-c -k"),
> + ("lz4", None, ".lz4", "-c -k"),
> # The "-P -C /" trick is used to avoid silly
> warnings:
> # "tar: Removing leading `/' from member names"
> ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"),
> ("gzip", "tar", ".tar.gz", "-c -z -O -P -C /"),
> ("xz", "tar", ".tar.xz", "-c -J -O -P -C /"),
> - ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C
> /")]
> + ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C
> /"),
> + ("lz4", "tar", ".tar.lz4", "-c -Ilz4 -O -P -C
> /")]
>
> for decompressor, archiver, suffix, options in compressors:
> if not BmapHelpers.program_is_available(decompressor):
> --
> 2.9.2
>
>
More information about the Bmap-tools
mailing list