noahbackus/godot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Support 64-bit sizes in Compression

Browse Source
This commit is contained in:
Aaron Franke
2025-05-08 14:13:30 -07:00
parent e45cc68092
commit 5777a88b76
13 changed files with 58 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/io/compression.cpp
+15 -9
View File
@@ -47,12 +47,13 @@ static ZSTD_DCtx *current_zstd_d_ctx = nullptr;
static bool current_zstd_long_distance_matching;
static int current_zstd_window_log_size;
int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int64_t Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break;
case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
if (p_src_size < 16) {
uint8_t src[16];
memset(&src[p_src_size], 0, 16 - p_src_size);
@@ -65,6 +66,7 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
} break;
case MODE_DEFLATE:
case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm;
@@ -95,22 +97,23 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size);
}
int max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
int ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
const int64_t max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
const size_t ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
ZSTD_freeCCtx(cctx);
return ret;
return (int64_t)ret;
} break;
}
ERR_FAIL_V(-1);
}
int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
int64_t Compression::get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break;
case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int ss = p_src_size + p_src_size * 6 / 100;
if (ss < 66) {
ss = 66;
@@ -120,6 +123,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
} break;
case MODE_DEFLATE:
case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm;
@@ -142,7 +146,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
ERR_FAIL_V(-1);
}
int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int64_t Compression::decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
#ifdef BROTLI_ENABLED
@@ -155,6 +159,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
#endif
} break;
case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int ret_size = 0;
if (p_dst_max_size < 16) {
@@ -169,6 +174,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
} break;
case MODE_DEFLATE:
case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm;
@@ -207,8 +213,8 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
current_zstd_window_log_size = zstd_window_log_size;
}
int ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
return ret;
size_t ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
return (int64_t)ret;
} break;
}
@@ -220,7 +226,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies.
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
*/
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
uint8_t *dst = nullptr;
int out_mark = 0;
core/io/compression.h
+4 -4
View File
@@ -52,8 +52,8 @@ public:
MODE_BROTLI
};
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
static int64_t compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int64_t get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int64_t decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode);
};
core/io/file_access_compressed.cpp
+7 -6
View File
@@ -68,7 +68,7 @@ Error FileAccessCompressed::open_after_magic(Ref<FileAccess> p_base) {
read_block_count = bc;
read_block_size = read_blocks.size() == 1 ? read_total : block_size;
int ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode);
const int64_t ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode);
read_block = 0;
read_pos = 0;
@@ -137,10 +137,11 @@ void FileAccessCompressed::_close() {
Vector<uint8_t> cblock;
cblock.resize(Compression::get_max_compressed_buffer_size(bl, cmode));
int s = Compression::compress(cblock.ptrw(), bp, bl, cmode);
const int64_t compressed_size = Compression::compress(cblock.ptrw(), bp, bl, cmode);
ERR_FAIL_COND_MSG(compressed_size < 0, "FileAccessCompressed: Error compressing data.");
f->store_buffer(cblock.ptr(), s);
block_sizes.push_back(s);
f->store_buffer(cblock.ptr(), (uint64_t)compressed_size);
block_sizes.push_back(compressed_size);
}
f->seek(16); //ok write block sizes
@@ -200,7 +201,7 @@ void FileAccessCompressed::seek(uint64_t p_position) {
read_block = block_idx;
f->seek(read_blocks[read_block].offset);
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
}
@@ -288,7 +289,7 @@ uint64_t FileAccessCompressed::get_buffer(uint8_t *p_dst, uint64_t p_length) con
// Read the next block of compressed data.
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
ERR_FAIL_COND_V_MSG(ret == -1, -1, "Compressed file is corrupt.");
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
read_pos = 0;
core/io/remote_filesystem_client.cpp
+2 -2
View File
@@ -215,7 +215,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
Vector<uint8_t> file_cache_buffer;
if (file_cache.size()) {
StringBuilder sbuild;
for (int i = 0; i < file_cache.size(); i++) {
for (int64_t i = 0; i < file_cache.size(); i++) {
sbuild.append(file_cache[i].path);
sbuild.append("::");
sbuild.append(itos(file_cache[i].server_modified_time));
@@ -224,7 +224,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
String s = sbuild.as_string();
CharString cs = s.utf8();
file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD));
int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
const int64_t res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
file_cache_buffer.resize(res_len);
tcp_client->put_32(cs.length()); // Size of buffer uncompressed