From 8a9597e32e98707d5bcdc1a41d8c45e3174d4047 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 20:56:20 +0200 Subject: DiscIO: Allow converting from formats other than ISO and GCZ The constant DESIRED_BUFFER_SIZE was determined by multiplying the old hardcoded value 32 with the default GCZ block size 16 KiB. Not sure if it actually is the best value, but it seems fine. --- Source/Core/DiscIO/CompressedBlob.cpp | 94 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 47 deletions(-) (limited to 'Source/Core/DiscIO/CompressedBlob.cpp') diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5248170683..33790ad3b0 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -17,6 +17,7 @@ #include #include +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/FileUtil.h" @@ -153,24 +154,20 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type, int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, + int block_size, CompressCB callback, void* arg) { bool scrubbing = false; - File::IOFile infile(infile_path, "rb"); - if (IsGCZBlob(infile)) - { - PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile_path.c_str()); - return false; - } - + std::unique_ptr infile = CreateDisc(infile_path); if (!infile) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(infile->IsSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -182,11 +179,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi } DiscScrubber disc_scrubber; - std::unique_ptr volume; if (sub_type == 1) { - volume = CreateDisc(infile_path); - if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size)) + if (!disc_scrubber.SetupScrub(infile.get())) { PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile_path.c_str()); @@ -206,7 +201,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile.GetSize(); + header.data_size = infile->GetSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -220,10 +215,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi outfile.Seek(sizeof(CompressedBlobHeader), SEEK_CUR); // seek past the offset and hash tables (we will write them at the end) outfile.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR); - // seek to the start of the input file to make sure we get everything - infile.Seek(0, SEEK_SET); // Now we are ready to write compressed data! + u64 inpos = 0; u64 position = 0; int num_compressed = 0; int num_stored = 0; @@ -234,7 +228,6 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi { if (i % progress_monitor == 0) { - const u64 inpos = infile.Tell(); int ratio = 0; if (inpos != 0) ratio = (int)(100 * position / inpos); @@ -252,13 +245,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi offsets[i] = position; - size_t read_bytes; - if (scrubbing) - read_bytes = disc_scrubber.GetNextBlock(infile, in_buf.data()); - else - infile.ReadArray(in_buf.data(), header.block_size, &read_bytes); - if (read_bytes < header.block_size) - std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0); + const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? + 0 : + std::min(block_size, header.data_size - inpos); + + success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + if (!success) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + break; + } + + std::fill(in_buf.begin() + bytes_to_read, in_buf.begin() + header.block_size, 0); int retval = deflateReset(&z); z.next_in = in_buf.data(); @@ -305,6 +303,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi break; } + inpos += block_size; position += write_size; hashes[i] = Common::HashAdler32(write_buf, write_size); @@ -337,27 +336,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi return success; } -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback, void* arg) { - std::unique_ptr reader; - { - File::IOFile infile(infile_path, "rb"); - if (!IsGCZBlob(infile)) - { - PanicAlertT("File not compressed"); - return false; - } - - reader = CompressedBlobReader::Create(std::move(infile), infile_path); - } - + std::unique_ptr reader = CreateBlobReader(infile_path); if (!reader) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(reader->IsDataSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -368,11 +358,20 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out return false; } - const CompressedBlobHeader& header = reader->GetHeader(); - static const size_t BUFFER_BLOCKS = 32; - size_t buffer_size = header.block_size * BUFFER_BLOCKS; + constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; + u64 buffer_size = reader->GetBlockSize(); + if (buffer_size == 0) + { + buffer_size = DESIRED_BUFFER_SIZE; + } + else + { + while (buffer_size < DESIRED_BUFFER_SIZE) + buffer_size *= 2; + } + std::vector buffer(buffer_size); - u32 num_buffers = (header.num_blocks + BUFFER_BLOCKS - 1) / BUFFER_BLOCKS; + const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; int progress_monitor = std::max(1, num_buffers / 100); bool success = true; @@ -389,8 +388,13 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out } } const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, header.data_size - inpos); - reader->Read(inpos, sz, buffer.data()); + const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); + if (!reader->Read(inpos, sz, buffer.data())) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + success = false; + break; + } if (!outfile.WriteBytes(buffer.data(), sz)) { PanicAlertT("Failed to write the output file \"%s\".\n" @@ -407,10 +411,6 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out outfile.Close(); File::Delete(outfile_path); } - else - { - outfile.Resize(header.data_size); - } return success; } -- cgit v1.2.3 From 42f6913bccdafcb8a32e5d82bd54947062cfd0a9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 21:47:28 +0200 Subject: Move DiscIO::ConvertToPlain to FileBlob.cpp There is no longer anything GCZ specific about it. --- Source/Core/DiscIO/CompressedBlob.cpp | 79 ----------------------------------- 1 file changed, 79 deletions(-) (limited to 'Source/Core/DiscIO/CompressedBlob.cpp') diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 33790ad3b0..5e6723679a 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -336,85 +336,6 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat return success; } -bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) -{ - std::unique_ptr reader = CreateBlobReader(infile_path); - if (!reader) - { - PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); - return false; - } - - ASSERT(reader->IsDataSizeAccurate()); - - File::IOFile outfile(outfile_path, "wb"); - if (!outfile) - { - PanicAlertT("Failed to open the output file \"%s\".\n" - "Check that you have permissions to write the target folder and that the media can " - "be written.", - outfile_path.c_str()); - return false; - } - - constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; - u64 buffer_size = reader->GetBlockSize(); - if (buffer_size == 0) - { - buffer_size = DESIRED_BUFFER_SIZE; - } - else - { - while (buffer_size < DESIRED_BUFFER_SIZE) - buffer_size *= 2; - } - - std::vector buffer(buffer_size); - const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; - int progress_monitor = std::max(1, num_buffers / 100); - bool success = true; - - for (u64 i = 0; i < num_buffers; i++) - { - if (i % progress_monitor == 0) - { - const bool was_cancelled = - !callback(Common::GetStringT("Unpacking"), (float)i / (float)num_buffers, arg); - if (was_cancelled) - { - success = false; - break; - } - } - const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); - if (!reader->Read(inpos, sz, buffer.data())) - { - PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); - success = false; - break; - } - if (!outfile.WriteBytes(buffer.data(), sz)) - { - PanicAlertT("Failed to write the output file \"%s\".\n" - "Check that you have enough space available on the target drive.", - outfile_path.c_str()); - success = false; - break; - } - } - - if (!success) - { - // Remove the incomplete output file. - outfile.Close(); - File::Delete(outfile_path); - } - - return success; -} - bool IsGCZBlob(File::IOFile& file) { const u64 position = file.Tell(); -- cgit v1.2.3 From 6ffcbcee70c043a0d158d87458f99d20eddbb939 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 10 Apr 2020 17:40:07 +0200 Subject: DiscIO: Move scrubbing code out of ConvertToGCZ This way, scrubbing can also be performed when converting to other formats. --- Source/Core/DiscIO/CompressedBlob.cpp | 37 +++++++---------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) (limited to 'Source/Core/DiscIO/CompressedBlob.cpp') diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5e6723679a..cda3cad6d3 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -154,19 +154,11 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, - int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, u32 sub_type, int block_size, + CompressCB callback, void* arg) { - bool scrubbing = false; - - std::unique_ptr infile = CreateDisc(infile_path); - if (!infile) - { - PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); - return false; - } - - ASSERT(infile->IsSizeAccurate()); + ASSERT(infile->IsDataSizeAccurate()); File::IOFile outfile(outfile_path, "wb"); if (!outfile) @@ -178,19 +170,6 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat return false; } - DiscScrubber disc_scrubber; - if (sub_type == 1) - { - if (!disc_scrubber.SetupScrub(infile.get())) - { - PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", - infile_path.c_str()); - return false; - } - - scrubbing = true; - } - z_stream z = {}; if (deflateInit(&z, 9) != Z_OK) return false; @@ -201,7 +180,7 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile->GetSize(); + header.data_size = infile->GetDataSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -245,11 +224,9 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat offsets[i] = position; - const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? - 0 : - std::min(block_size, header.data_size - inpos); + const u64 bytes_to_read = std::min(block_size, header.data_size - inpos); - success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + success = infile->Read(inpos, bytes_to_read, in_buf.data()); if (!success) { PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); -- cgit v1.2.3