pg_basebackup: Allow client-side LZ4 (de)compression.

LZ4 compression can now be performed on the client using
pg_basebackup -Ft --compress client-lz4, and LZ4 decompression of
a backup compressed on the server can be performed on the client
using pg_basebackup -Fp --compress server-lz4.

Dipesh Pandit, reviewed and tested by Jeevan Ladhe and Tushar Ahuja,
with a few corrections - and some documentation - by me.

Discussion: http://postgr.es/m/CAN1g5_FeDmiA9D8wdG2W6Lkq5CpubxOAqTmd2et9hsinTJtsMQ@mail.gmail.com

2 files changed, 117 insertions, 1 deletions

diff --git a/src/bin/pg_verifybackup/t/009_extract.pl b/src/bin/pg_verifybackup/t/009_extract.pl
index 51b77e4bfe6..9f9a7cc6a5f 100644
--- a/src/bin/pg_verifybackup/t/009_extract.pl
+++ b/src/bin/pg_verifybackup/t/009_extract.pl
@@ -11,7 +11,7 @@ use Config;
 use File::Path qw(rmtree);
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
-use Test::More tests => 4;
+use Test::More tests => 6;
 
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
@@ -27,6 +27,11 @@ my @test_configuration = (
 		'compression_method' => 'gzip',
 		'backup_flags' => ['--compress', 'server-gzip:5'],
 		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
+	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags' => ['--compress', 'server-lz4:5'],
+		'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
 	}
 );
 
diff --git a/src/bin/pg_verifybackup/t/010_client_untar.pl b/src/bin/pg_verifybackup/t/010_client_untar.pl
new file mode 100644
index 00000000000..34c9b90669d
--- /dev/null
+++ b/src/bin/pg_verifybackup/t/010_client_untar.pl
@@ -0,0 +1,111 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# This test case aims to verify that client-side backup compression work
+# properly, and it also aims to verify that pg_verifybackup can verify a base
+# backup that didn't start out in plain format.
+
+use strict;
+use warnings;
+use Config;
+use File::Path qw(rmtree);
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 9;
+
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+my $backup_path = $primary->backup_dir . '/client-backup';
+my $extract_path = $primary->backup_dir . '/extracted-backup';
+
+my @test_configuration = (
+	{
+		'compression_method' => 'none',
+		'backup_flags' => [],
+		'backup_archive' => 'base.tar',
+		'enabled' => 1
+	},
+	{
+		'compression_method' => 'gzip',
+		'backup_flags' => ['--compress', 'client-gzip:5'],
+		'backup_archive' => 'base.tar.gz',
+		'decompress_program' => $ENV{'GZIP_PROGRAM'},
+		'decompress_flags' => [ '-d' ],
+		'enabled' => check_pg_config("#define HAVE_LIBZ 1")
+	},
+	{
+		'compression_method' => 'lz4',
+		'backup_flags' => ['--compress', 'client-lz4:5'],
+		'backup_archive' => 'base.tar.lz4',
+		'decompress_program' => $ENV{'LZ4'},
+		'decompress_flags' => [ '-d' ],
+		'output_file' => 'base.tar',
+		'enabled' => check_pg_config("#define HAVE_LIBLZ4 1")
+	}
+);
+
+for my $tc (@test_configuration)
+{
+	my $method = $tc->{'compression_method'};
+
+	SKIP: {
+		skip "$method compression not supported by this build", 3
+			if ! $tc->{'enabled'};
+		skip "no decompressor available for $method", 3
+			if exists $tc->{'decompress_program'} &&
+			!defined $tc->{'decompress_program'};
+
+		# Take a client-side backup.
+		my @backup      = (
+			'pg_basebackup', '-D', $backup_path,
+			'-Xfetch', '--no-sync', '-cfast', '-Ft');
+		push @backup, @{$tc->{'backup_flags'}};
+		$primary->command_ok(\@backup,
+							 "client side backup, compression $method");
+
+
+		# Verify that the we got the files we expected.
+		my $backup_files = join(',',
+			sort grep { $_ ne '.' && $_ ne '..' } slurp_dir($backup_path));
+		my $expected_backup_files = join(',',
+			sort ('backup_manifest', $tc->{'backup_archive'}));
+		is($backup_files,$expected_backup_files,
+			"found expected backup files, compression $method");
+
+		# Decompress.
+		if (exists $tc->{'decompress_program'})
+		{
+			my @decompress = ($tc->{'decompress_program'});
+			push @decompress, @{$tc->{'decompress_flags'}}
+				if $tc->{'decompress_flags'};
+			push @decompress, $backup_path . '/' . $tc->{'backup_archive'};
+			push @decompress, $backup_path . '/' . $tc->{'output_file'}
+				if $tc->{'output_file'};
+			system_or_bail(@decompress);
+		}
+
+		SKIP: {
+			my $tar = $ENV{TAR};
+			# don't check for a working tar here, to accomodate various odd
+			# cases such as AIX. If tar doesn't work the init_from_backup below
+			# will fail.
+			skip "no tar program available", 1
+				if (!defined $tar || $tar eq '');
+
+			# Untar.
+			mkdir($extract_path);
+			system_or_bail($tar, 'xf', $backup_path . '/base.tar',
+				'-C', $extract_path);
+
+			# Verify.
+			$primary->command_ok([ 'pg_verifybackup', '-n',
+				'-m', "$backup_path/backup_manifest", '-e', $extract_path ],
+				"verify backup, compression $method");
+		}
+
+		# Cleanup.
+		rmtree($extract_path);
+		rmtree($backup_path);
+	}
+}