Added Virtual File System support
This commit is contained in:
snarmph
parent
363c4f19cb
commit
01f4ad7f62
18 changed files with 4582 additions and 64 deletions
4
arena.c
4
arena.c
|
|
@ -57,7 +57,7 @@ arena_t arenaScratch(arena_t *arena, usize size) {
|
|||
}
|
||||
|
||||
void *arenaAlloc(const arena_alloc_desc_t *desc) {
|
||||
if (!desc || !desc->arena) {
|
||||
if (!desc || !desc->arena || desc->arena->type == ARENA_TYPE_NONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ void *arenaAlloc(const arena_alloc_desc_t *desc) {
|
|||
if (desc->flags & ALLOC_SOFT_FAIL) {
|
||||
return NULL;
|
||||
}
|
||||
printf("finished space in arena, tried to allocate %zu bytes out of %zu\n", total, arenaRemaining(arena));
|
||||
fatal("finished space in arena, tried to allocate %_$$$dB out of %_$$$dB\n", total, arenaRemaining(arena));
|
||||
abort();
|
||||
}
|
||||
|
||||
|
|
|
|||
1
arena.h
1
arena.h
|
|
@ -9,6 +9,7 @@
|
|||
#endif
|
||||
|
||||
typedef enum {
|
||||
ARENA_TYPE_NONE, // only here so that a 0 initialised arena is valid
|
||||
ARENA_VIRTUAL,
|
||||
ARENA_MALLOC,
|
||||
ARENA_STATIC,
|
||||
|
|
|
|||
24
bits.h
24
bits.h
|
|
@ -3,6 +3,7 @@
|
|||
#include "collatypes.h"
|
||||
|
||||
uint32 bitsCtz(uint32 v);
|
||||
uint32 bitsNextPow2(uint32 v);
|
||||
|
||||
// == INLINE IMPLEMENTATION ==============================================================
|
||||
|
||||
|
|
@ -13,6 +14,9 @@ uint32 bitsCtz(uint32 v);
|
|||
#elif COLLA_GCC || COLLA_CLANG || COLLA_EMC
|
||||
#define BITS_WIN 0
|
||||
#define BITS_LIN 1
|
||||
#elif COLLA_TCC
|
||||
#define BITS_WIN 0
|
||||
#define BITS_LIN 0
|
||||
#else
|
||||
#error "bits header not supported on this compiler"
|
||||
#endif
|
||||
|
|
@ -24,11 +28,27 @@ inline uint32 bitsCtz(uint32 v) {
|
|||
return v ? __builtin_ctz(v) : 0;
|
||||
#elif BITS_WIN
|
||||
uint32 trailing = 0;
|
||||
return _BitScanForward((unsigned long *)&trailing, v) ? trailing : 0;
|
||||
return _BitScanForward((unsigned long *)&trailing, v) ? trailing : 32;
|
||||
#else
|
||||
return 0;
|
||||
if (v == 0) return 0;
|
||||
for (uint32 i = 0; i < 32; ++i) {
|
||||
if (v & (1 << i)) return i;
|
||||
}
|
||||
return 32;
|
||||
#endif
|
||||
}
|
||||
|
||||
inline uint32 bitsNextPow2(uint32 v) {
|
||||
v--;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
v++;
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
#undef BITS_WIN
|
||||
#undef BITS_LIN
|
||||
|
|
|
|||
|
|
@ -100,6 +100,9 @@
|
|||
|
||||
#if COLLA_WIN
|
||||
|
||||
#undef NOMINMAX
|
||||
#undef WIN32_LEAN_AND_MEAN
|
||||
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#define NOMINMAX
|
||||
|
||||
|
|
|
|||
69
file.c
69
file.c
|
|
@ -5,6 +5,10 @@
|
|||
#include "tracelog.h"
|
||||
#include "format.h"
|
||||
|
||||
#define FILE_MAKE_SCRATCH() \
|
||||
uint8 tmpbuf[KB(1)]; \
|
||||
arena_t scratch = arenaMake(ARENA_STATIC, sizeof(tmpbuf), tmpbuf)
|
||||
|
||||
#if COLLA_WIN
|
||||
|
||||
#include <windows.h>
|
||||
|
|
@ -36,17 +40,18 @@ static DWORD file__mode_to_creation(filemode_e mode) {
|
|||
return OPEN_ALWAYS;
|
||||
}
|
||||
|
||||
bool fileExists(const char *name) {
|
||||
return GetFileAttributesA(name) != INVALID_FILE_ATTRIBUTES;
|
||||
bool fileExists(strview_t path) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
str_t name = str(&scratch, path);
|
||||
return GetFileAttributesA(name.buf) != INVALID_FILE_ATTRIBUTES;
|
||||
}
|
||||
|
||||
TCHAR *fileGetFullPath(arena_t *arena, strview_t filename) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
|
||||
TCHAR long_path_prefix[] = TEXT("\\\\?\\");
|
||||
const usize prefix_len = arrlen(long_path_prefix) - 1;
|
||||
|
||||
uint8 tempbuf[4096];
|
||||
arena_t scratch = arenaMake(ARENA_STATIC, sizeof(tempbuf), tempbuf);
|
||||
|
||||
TCHAR *rel_path = strvToTChar(&scratch, filename);
|
||||
DWORD pathlen = GetFullPathName(rel_path, 0, NULL, NULL);
|
||||
|
||||
|
|
@ -58,12 +63,15 @@ TCHAR *fileGetFullPath(arena_t *arena, strview_t filename) {
|
|||
return full_path;
|
||||
}
|
||||
|
||||
bool fileDelete(arena_t scratch, strview_t filename) {
|
||||
bool fileDelete(strview_t filename) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
wchar_t *wfname = strvToWChar(&scratch, filename, NULL);
|
||||
return DeleteFileW(wfname);
|
||||
}
|
||||
|
||||
file_t fileOpen(arena_t scratch, strview_t name, filemode_e mode) {
|
||||
file_t fileOpen(strview_t name, filemode_e mode) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
|
||||
TCHAR *full_path = fileGetFullPath(&scratch, name);
|
||||
|
||||
HANDLE handle = CreateFile(
|
||||
|
|
@ -105,6 +113,15 @@ usize fileWrite(file_t ctx, const void *buf, usize len) {
|
|||
return (usize)written;
|
||||
}
|
||||
|
||||
bool fileSeek(file_t ctx, usize pos) {
|
||||
if (!fileIsValid(ctx)) return false;
|
||||
LARGE_INTEGER offset = {
|
||||
.QuadPart = pos,
|
||||
};
|
||||
DWORD result = SetFilePointer((HANDLE)ctx.handle, offset.LowPart, &offset.HighPart, FILE_BEGIN);
|
||||
return result != INVALID_SET_FILE_POINTER;
|
||||
}
|
||||
|
||||
bool fileSeekEnd(file_t ctx) {
|
||||
if (!fileIsValid(ctx)) return false;
|
||||
DWORD result = SetFilePointer((HANDLE)ctx.handle, 0, NULL, FILE_END);
|
||||
|
|
@ -154,19 +171,25 @@ static const char *file__mode_to_stdio(filemode_e mode) {
|
|||
return "ab+";
|
||||
}
|
||||
|
||||
bool fileExists(const char *name) {
|
||||
FILE *fp = fopen(name, "rb");
|
||||
bool fileExists(strview_t path) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
str_t name = str(&scratch, path);
|
||||
|
||||
FILE *fp = fopen(name.buf, "rb");
|
||||
bool exists = fp != NULL;
|
||||
fclose(fp);
|
||||
|
||||
return exists;
|
||||
}
|
||||
|
||||
bool fileDelete(arena_t scratch, strview_t filename) {
|
||||
bool fileDelete(strview_t filename) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
str_t name = str(&scratch, filename);
|
||||
return remove(name.buf) == 0;
|
||||
}
|
||||
|
||||
file_t fileOpen(arena_t scratch, strview_t name, filemode_e mode) {
|
||||
file_t fileOpen(strview_t name, filemode_e mode) {
|
||||
FILE_MAKE_SCRATCH();
|
||||
str_t filename = str(&scratch, name);
|
||||
return (file_t) {
|
||||
.handle = (uintptr_t)fopen(filename.buf, file__mode_to_stdio(mode))
|
||||
|
|
@ -196,6 +219,12 @@ usize fileWrite(file_t ctx, const void *buf, usize len) {
|
|||
return fwrite(buf, 1, len, (FILE *)ctx.handle);
|
||||
}
|
||||
|
||||
bool fileSeek(file_t ctx, usize pos) {
|
||||
assert(pos < INT32_MAX);
|
||||
if (!fileIsValid(ctx)) return false;
|
||||
return fseek((FILE *)ctx.handle, (long)pos, SEEK_SET) == 0;
|
||||
}
|
||||
|
||||
bool fileSeekEnd(file_t ctx) {
|
||||
if (!fileIsValid(ctx)) return false;
|
||||
return fseek((FILE *)ctx.handle, 0, SEEK_END) == 0;
|
||||
|
|
@ -287,7 +316,7 @@ bool filePrintfv(arena_t scratch, file_t ctx, const char *fmt, va_list args) {
|
|||
}
|
||||
|
||||
buffer_t fileReadWhole(arena_t *arena, strview_t name) {
|
||||
file_t fp = fileOpen(*arena, name, FILE_READ);
|
||||
file_t fp = fileOpen(name, FILE_READ);
|
||||
if (!fileIsValid(fp)) {
|
||||
err("could not open file: %v", name);
|
||||
return (buffer_t){0};
|
||||
|
|
@ -315,7 +344,7 @@ buffer_t fileReadWholeFP(arena_t *arena, file_t ctx) {
|
|||
}
|
||||
|
||||
str_t fileReadWholeStr(arena_t *arena, strview_t name) {
|
||||
file_t fp = fileOpen(*arena, name, FILE_READ);
|
||||
file_t fp = fileOpen(name, FILE_READ);
|
||||
if (!fileIsValid(fp)) {
|
||||
warn("could not open file (%v)", name);
|
||||
}
|
||||
|
|
@ -342,8 +371,8 @@ str_t fileReadWholeStrFP(arena_t *arena, file_t ctx) {
|
|||
return out;
|
||||
}
|
||||
|
||||
bool fileWriteWhole(arena_t scratch, strview_t name, const void *buf, usize len) {
|
||||
file_t fp = fileOpen(scratch, name, FILE_WRITE);
|
||||
bool fileWriteWhole(strview_t name, const void *buf, usize len) {
|
||||
file_t fp = fileOpen(name, FILE_WRITE);
|
||||
if (!fileIsValid(fp)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -352,16 +381,18 @@ bool fileWriteWhole(arena_t scratch, strview_t name, const void *buf, usize len)
|
|||
return written == len;
|
||||
}
|
||||
|
||||
uint64 fileGetTime(arena_t scratch, strview_t name) {
|
||||
file_t fp = fileOpen(scratch, name, FILE_READ);
|
||||
uint64 fileGetTime(strview_t name) {
|
||||
file_t fp = fileOpen(name, FILE_READ);
|
||||
uint64 result = fileGetTimeFP(fp);
|
||||
fileClose(fp);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool fileHasChanged(arena_t scratch, strview_t name, uint64 last_timestamp) {
|
||||
uint64 timestamp = fileGetTime(scratch, name);
|
||||
bool fileHasChanged(strview_t name, uint64 last_timestamp) {
|
||||
uint64 timestamp = fileGetTime(name);
|
||||
return timestamp > last_timestamp;
|
||||
}
|
||||
|
||||
#include "warnings/colla_warn_end.h"
|
||||
|
||||
#undef FILE_MAKE_SCRATCH
|
||||
13
file.h
13
file.h
|
|
@ -16,14 +16,14 @@ typedef struct {
|
|||
uintptr_t handle;
|
||||
} file_t;
|
||||
|
||||
bool fileExists(const char *name);
|
||||
bool fileExists(strview_t path);
|
||||
TCHAR *fileGetFullPath(arena_t *arena, strview_t filename);
|
||||
strview_t fileGetFilename(strview_t path);
|
||||
strview_t fileGetExtension(strview_t path);
|
||||
void fileSplitPath(strview_t path, strview_t *dir, strview_t *name, strview_t *ext);
|
||||
bool fileDelete(arena_t scratch, strview_t filename);
|
||||
bool fileDelete(strview_t filename);
|
||||
|
||||
file_t fileOpen(arena_t scratch, strview_t name, filemode_e mode);
|
||||
file_t fileOpen(strview_t name, filemode_e mode);
|
||||
void fileClose(file_t ctx);
|
||||
|
||||
bool fileIsValid(file_t ctx);
|
||||
|
|
@ -36,6 +36,7 @@ bool filePrintfv(arena_t scratch, file_t ctx, const char *fmt, va_list args);
|
|||
usize fileRead(file_t ctx, void *buf, usize len);
|
||||
usize fileWrite(file_t ctx, const void *buf, usize len);
|
||||
|
||||
bool fileSeek(file_t ctx, usize pos);
|
||||
bool fileSeekEnd(file_t ctx);
|
||||
void fileRewind(file_t ctx);
|
||||
|
||||
|
|
@ -48,8 +49,8 @@ buffer_t fileReadWholeFP(arena_t *arena, file_t ctx);
|
|||
str_t fileReadWholeStr(arena_t *arena, strview_t name);
|
||||
str_t fileReadWholeStrFP(arena_t *arena, file_t ctx);
|
||||
|
||||
bool fileWriteWhole(arena_t scratch, strview_t name, const void *buf, usize len);
|
||||
bool fileWriteWhole(strview_t name, const void *buf, usize len);
|
||||
|
||||
uint64 fileGetTime(arena_t scratch, strview_t name);
|
||||
uint64 fileGetTime(strview_t name);
|
||||
uint64 fileGetTimeFP(file_t ctx);
|
||||
bool fileHasChanged(arena_t scratch, strview_t name, uint64 last_timestamp);
|
||||
bool fileHasChanged(strview_t name, uint64 last_timestamp);
|
||||
|
|
@ -473,7 +473,7 @@ static hl_htable_result_e hl_htable_add(arena_t *arena, hl_hashtable_t *table, s
|
|||
}
|
||||
|
||||
if ((float)table->used >= table->count * 0.6f) {
|
||||
warn("more than 60%% of the arena is being used: %d/%d", table->used, table->count);
|
||||
warn("more than 60%% of the hashmap is being used: %d/%d", table->used, table->count);
|
||||
}
|
||||
|
||||
uint64 hash = hl_htable_hash(key.buf, key.len);
|
||||
|
|
@ -502,7 +502,7 @@ static hl_htable_result_e hl_htable_add(arena_t *arena, hl_hashtable_t *table, s
|
|||
}
|
||||
|
||||
static hl_node_t *hl_htable_get(hl_hashtable_t *table, strview_t key) {
|
||||
if (!table || table->count == 0) {
|
||||
if (!table || table->used == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
51
hot_reload.c
51
hot_reload.c
|
|
@ -24,7 +24,7 @@ typedef struct {
|
|||
hr_f hr_close;
|
||||
} hr_internal_t;
|
||||
|
||||
static bool hr__os_reload(hr_internal_t *hr);
|
||||
static bool hr__os_reload(hr_internal_t *hr, str_t libpath);
|
||||
static void hr__os_free(hr_internal_t *hr);
|
||||
|
||||
static bool hr__file_copy(arena_t scratch, strview_t src, strview_t dst) {
|
||||
|
|
@ -33,7 +33,7 @@ static bool hr__file_copy(arena_t scratch, strview_t src, strview_t dst) {
|
|||
err("fileReadWhole(%v) returned an empty buffer", src);
|
||||
return false;
|
||||
}
|
||||
if (!fileWriteWhole(scratch, dst, srcbuf.data, srcbuf.len)) {
|
||||
if (!fileWriteWhole(dst, srcbuf.data, srcbuf.len)) {
|
||||
err("fileWriteWhole failed");
|
||||
return false;
|
||||
}
|
||||
|
|
@ -48,12 +48,12 @@ static bool hr__reload(hr_t *ctx) {
|
|||
hr_internal_t *hr = ctx->p;
|
||||
arena_t scratch = hr->arena;
|
||||
|
||||
if (!fileExists(hr->path.buf)) {
|
||||
if (!fileExists(strv(hr->path))) {
|
||||
err("dll file %v does not exist anymore!", hr->path);
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64 now = fileGetTime(scratch, strv(hr->path));
|
||||
uint64 now = fileGetTime(strv(hr->path));
|
||||
if (now <= hr->last_timestamp) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -63,16 +63,24 @@ static bool hr__reload(hr_t *ctx) {
|
|||
// can't copy the dll directly, make a temporary one based on the version
|
||||
strview_t dir, name, ext;
|
||||
fileSplitPath(strv(hr->path), &dir, &name, &ext);
|
||||
str_t dll = strFmt(&scratch, "%v/%v-%d%v", dir, name, ctx->version, ext);
|
||||
str_t libpath = strFmt(&scratch, "%v/%v-%d%v", dir, name, ctx->version, ext);
|
||||
|
||||
if (!hr__file_copy(scratch, strv(hr->path), strv(dll))) {
|
||||
err("failed to copy %v to %v", hr->path, dll);
|
||||
if (!hr__file_copy(scratch, strv(hr->path), strv(libpath))) {
|
||||
err("failed to copy %v to %v", hr->path, libpath);
|
||||
return false;
|
||||
}
|
||||
|
||||
info("loading library: %v", dll);
|
||||
info("loading library: %v", libpath);
|
||||
|
||||
return hr__os_reload(hr);
|
||||
bool success = hr__os_reload(hr, libpath);
|
||||
if (success) {
|
||||
info("Reloaded, version: %d", ctx->version);
|
||||
ctx->last_working_version = ctx->version;
|
||||
hr->last_timestamp = now;
|
||||
hr->hr_init(ctx);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
bool hrOpen(hr_t *ctx, strview_t path) {
|
||||
|
|
@ -80,19 +88,17 @@ bool hrOpen(hr_t *ctx, strview_t path) {
|
|||
cr_init(ctx);
|
||||
return true;
|
||||
#endif
|
||||
arena_t arena = arenaMake(ARENA_VIRTUAL, MB(1));
|
||||
|
||||
str_t path_copy = str(&arena, path);
|
||||
|
||||
if (!fileExists(path_copy.buf)) {
|
||||
if (!fileExists(path)) {
|
||||
err("dll file: %v does not exist", path);
|
||||
arenaCleanup(&arena);
|
||||
return false;
|
||||
}
|
||||
|
||||
arena_t arena = arenaMake(ARENA_VIRTUAL, MB(1));
|
||||
|
||||
hr_internal_t *hr = alloc(&arena, hr_internal_t);
|
||||
hr->arena = arena;
|
||||
hr->path = path_copy;
|
||||
hr->path = str(&arena, path);
|
||||
|
||||
ctx->p = hr;
|
||||
ctx->last_working_version = 0;
|
||||
|
|
@ -124,7 +130,7 @@ void hrClose(hr_t *ctx, bool clean_temp_files) {
|
|||
|
||||
for (int i = 0; i < ctx->last_working_version; ++i) {
|
||||
str_t fname = strFmt(&scratch, "%v/%v-%d%v", dir, name, i + 1, ext);
|
||||
if (!fileDelete(scratch, strv(fname))) {
|
||||
if (!fileDelete(strv(fname))) {
|
||||
err("couldn't delete %v", fname);
|
||||
}
|
||||
}
|
||||
|
|
@ -158,14 +164,14 @@ bool hrReload(hr_t *ctx) {
|
|||
|
||||
#if COLLA_WIN
|
||||
|
||||
static bool hr__os_reload(hr_internal_t *hr) {
|
||||
static bool hr__os_reload(hr_internal_t *hr, str_t libpath) {
|
||||
if (hr->handle) {
|
||||
FreeLibrary(hr->handle);
|
||||
}
|
||||
|
||||
hr->handle = LoadLibraryA(dll.buf);
|
||||
hr->handle = LoadLibraryA(libpath.buf);
|
||||
if (!hr->handle) {
|
||||
err("couldn't load %v: %u", dll, GetLastError());
|
||||
err("couldn't load %v: %u", libpath, GetLastError());
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -191,11 +197,6 @@ static bool hr__os_reload(hr_internal_t *hr) {
|
|||
goto error;
|
||||
}
|
||||
|
||||
info("Reloaded, version: %d", ctx->version);
|
||||
hr->last_timestamp = now;
|
||||
ctx->last_working_version = ctx->version;
|
||||
|
||||
hr->hr_init(ctx);
|
||||
|
||||
return true;
|
||||
|
||||
|
|
@ -214,7 +215,7 @@ static void hr__os_free(hr_internal_t *hr) {
|
|||
|
||||
#elif COLLA_LIN
|
||||
|
||||
static bool hr__os_reload(hr_internal_t *hr) {
|
||||
static bool hr__os_reload(hr_internal_t *hr, str_t libpath) {
|
||||
fatal("todo: linux hot reload not implemented yet");
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
2
html.h
2
html.h
|
|
@ -41,7 +41,7 @@ typedef struct {
|
|||
static void html__tag(html_context_t *ctx, const char *tag, html_tag_t *args) {
|
||||
ostrPrintf(&ctx->stream, "<%s", tag);
|
||||
|
||||
#define X(name, ...) if (args->name) { ostrPrintf(&ctx->stream, #name "=\"%s\" ", args->name); }
|
||||
#define X(name, ...) if (args->name) { ostrPrintf(&ctx->stream, " " #name "=\"%s\"", args->name); }
|
||||
html__args()
|
||||
#undef X
|
||||
|
||||
|
|
|
|||
2
ini.c
2
ini.c
|
|
@ -9,7 +9,7 @@
|
|||
static void ini__parse(arena_t *arena, ini_t *ini, const iniopts_t *options);
|
||||
|
||||
ini_t iniParse(arena_t *arena, strview_t filename, const iniopts_t *options) {
|
||||
file_t fp = fileOpen(*arena, filename, FILE_READ);
|
||||
file_t fp = fileOpen(filename, FILE_READ);
|
||||
ini_t out = iniParseFile(arena, fp, options);
|
||||
fileClose(fp);
|
||||
return out;
|
||||
|
|
|
|||
884
lz4/lz4.h
Normal file
884
lz4/lz4.h
Normal file
|
|
@ -0,0 +1,884 @@
|
|||
/*
|
||||
* LZ4 - Fast LZ compression algorithm
|
||||
* Header File
|
||||
* Copyright (C) 2011-2023, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://www.lz4.org
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
*/
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef LZ4_H_2983827168210
|
||||
#define LZ4_H_2983827168210
|
||||
|
||||
/* --- Dependency --- */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/**
|
||||
Introduction
|
||||
|
||||
LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
|
||||
scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
|
||||
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
|
||||
|
||||
The LZ4 compression library provides in-memory compression and decompression functions.
|
||||
It gives full buffer control to user.
|
||||
Compression can be done in:
|
||||
- a single step (described as Simple Functions)
|
||||
- a single step, reusing a context (described in Advanced Functions)
|
||||
- unbounded multiple steps (described as Streaming compression)
|
||||
|
||||
lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
|
||||
Decompressing such a compressed block requires additional metadata.
|
||||
Exact metadata depends on exact decompression function.
|
||||
For the typical case of LZ4_decompress_safe(),
|
||||
metadata includes block's compressed size, and maximum bound of decompressed size.
|
||||
Each application is free to encode and pass such metadata in whichever way it wants.
|
||||
|
||||
lz4.h only handle blocks, it can not generate Frames.
|
||||
|
||||
Blocks are different from Frames (doc/lz4_Frame_format.md).
|
||||
Frames bundle both blocks and metadata in a specified manner.
|
||||
Embedding metadata is required for compressed data to be self-contained and portable.
|
||||
Frame format is delivered through a companion API, declared in lz4frame.h.
|
||||
The `lz4` CLI can only manage frames.
|
||||
*/
|
||||
|
||||
/*^***************************************************************
|
||||
* Export parameters
|
||||
*****************************************************************/
|
||||
/*
|
||||
* LZ4_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
* LZ4LIB_VISIBILITY :
|
||||
* Control library symbols visibility.
|
||||
*/
|
||||
#ifndef LZ4LIB_VISIBILITY
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
# else
|
||||
# define LZ4LIB_VISIBILITY
|
||||
# endif
|
||||
#endif
|
||||
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
|
||||
# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
|
||||
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
|
||||
# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define LZ4LIB_API LZ4LIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
/*! LZ4_FREESTANDING :
|
||||
* When this macro is set to 1, it enables "freestanding mode" that is
|
||||
* suitable for typical freestanding environment which doesn't support
|
||||
* standard C library.
|
||||
*
|
||||
* - LZ4_FREESTANDING is a compile-time switch.
|
||||
* - It requires the following macros to be defined:
|
||||
* LZ4_memcpy, LZ4_memmove, LZ4_memset.
|
||||
* - It only enables LZ4/HC functions which don't use heap.
|
||||
* All LZ4F_* functions are not supported.
|
||||
* - See tests/freestanding.c to check its basic setup.
|
||||
*/
|
||||
#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
|
||||
# define LZ4_HEAPMODE 0
|
||||
# define LZ4HC_HEAPMODE 0
|
||||
# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
|
||||
# if !defined(LZ4_memcpy)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
|
||||
# endif
|
||||
# if !defined(LZ4_memset)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
|
||||
# endif
|
||||
# if !defined(LZ4_memmove)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
|
||||
# endif
|
||||
#elif ! defined(LZ4_FREESTANDING)
|
||||
# define LZ4_FREESTANDING 0
|
||||
#endif
|
||||
|
||||
|
||||
/*------ Version ------*/
|
||||
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
|
||||
#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */
|
||||
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
|
||||
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
|
||||
|
||||
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
|
||||
#define LZ4_QUOTE(str) #str
|
||||
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
|
||||
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */
|
||||
|
||||
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */
|
||||
LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Tuning memory usage
|
||||
**************************************/
|
||||
/*!
|
||||
* LZ4_MEMORY_USAGE :
|
||||
* Can be selected at compile time, by setting LZ4_MEMORY_USAGE.
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB)
|
||||
* Increasing memory usage improves compression ratio, generally at the cost of speed.
|
||||
* Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
|
||||
* Default value is 14, for 16KB, which nicely fits into most L1 caches.
|
||||
*/
|
||||
#ifndef LZ4_MEMORY_USAGE
|
||||
# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
|
||||
#endif
|
||||
|
||||
/* These are absolute limits, they should not be changed by users */
|
||||
#define LZ4_MEMORY_USAGE_MIN 10
|
||||
#define LZ4_MEMORY_USAGE_DEFAULT 14
|
||||
#define LZ4_MEMORY_USAGE_MAX 20
|
||||
|
||||
#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
|
||||
# error "LZ4_MEMORY_USAGE is too small !"
|
||||
#endif
|
||||
|
||||
#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
|
||||
# error "LZ4_MEMORY_USAGE is too large !"
|
||||
#endif
|
||||
|
||||
/*-************************************
|
||||
* Simple Functions
|
||||
**************************************/
|
||||
/*! LZ4_compress_default() :
|
||||
* Compresses 'srcSize' bytes from buffer 'src'
|
||||
* into already allocated 'dst' buffer of size 'dstCapacity'.
|
||||
* Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
|
||||
* It also runs faster, so it's a recommended setting.
|
||||
* If the function cannot compress 'src' into a more limited 'dst' budget,
|
||||
* compression stops *immediately*, and the function result is zero.
|
||||
* In which case, 'dst' content is undefined (invalid).
|
||||
* srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
|
||||
* dstCapacity : size of buffer 'dst' (which must be already allocated)
|
||||
* @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
|
||||
* or 0 if compression fails
|
||||
* Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
|
||||
|
||||
/*! LZ4_decompress_safe() :
|
||||
* @compressedSize : is the exact complete size of the compressed block.
|
||||
* @dstCapacity : is the size of destination buffer (which must be already allocated),
|
||||
* presumed an upper bound of decompressed size.
|
||||
* @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
|
||||
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
|
||||
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
* Note 1 : This function is protected against malicious data packets :
|
||||
* it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
|
||||
* even if the compressed block is maliciously modified to order the decoder to do these actions.
|
||||
* In such case, the decoder stops immediately, and considers the compressed block malformed.
|
||||
* Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
|
||||
* The implementation is free to send / store / derive this information in whichever way is most beneficial.
|
||||
* If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Advanced Functions
|
||||
**************************************/
|
||||
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
|
||||
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
|
||||
|
||||
/*! LZ4_compressBound() :
|
||||
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
|
||||
This function is primarily useful for memory allocation purposes (destination buffer size).
|
||||
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
|
||||
Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
|
||||
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
|
||||
return : maximum output size in a "worst case" scenario
|
||||
or 0, if input size is incorrect (too large or negative)
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compressBound(int inputSize);
|
||||
|
||||
/*! LZ4_compress_fast() :
|
||||
Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
|
||||
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
|
||||
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
|
||||
An acceleration value of "1" is the same as regular LZ4_compress_default()
|
||||
Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c).
|
||||
Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
|
||||
/*! LZ4_compress_fast_extState() :
|
||||
* Same as LZ4_compress_fast(), using an externally allocated memory space for its state.
|
||||
* Use LZ4_sizeofState() to know how much memory must be allocated,
|
||||
* and allocate it on 8-bytes boundaries (using `malloc()` typically).
|
||||
* Then, provide this buffer as `void* state` to compression function.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_sizeofState(void);
|
||||
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
/*! LZ4_compress_destSize() :
|
||||
* Reverse the logic : compresses as much data as possible from 'src' buffer
|
||||
* into already allocated buffer 'dst', of size >= 'dstCapacity'.
|
||||
* This function either compresses the entire 'src' content into 'dst' if it's large enough,
|
||||
* or fill 'dst' buffer completely with as much data as possible from 'src'.
|
||||
* note: acceleration parameter is fixed to "default".
|
||||
*
|
||||
* *srcSizePtr : in+out parameter. Initially contains size of input.
|
||||
* Will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
|
||||
* New value is necessarily <= input value.
|
||||
* @return : Nb bytes written into 'dst' (necessarily <= dstCapacity)
|
||||
* or 0 if compression fails.
|
||||
*
|
||||
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
|
||||
* the produced compressed content could, in specific circumstances,
|
||||
* require to be decompressed into a destination buffer larger
|
||||
* by at least 1 byte than the content to decompress.
|
||||
* If an application uses `LZ4_compress_destSize()`,
|
||||
* it's highly recommended to update liblz4 to v1.9.2 or better.
|
||||
* If this can't be done or ensured,
|
||||
* the receiving decompression function should provide
|
||||
* a dstCapacity which is > decompressedSize, by at least 1 byte.
|
||||
* See https://github.com/lz4/lz4/issues/859 for details
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize);
|
||||
|
||||
/*! LZ4_decompress_safe_partial() :
|
||||
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
|
||||
* into destination buffer 'dst' of size 'dstCapacity'.
|
||||
* Up to 'targetOutputSize' bytes will be decoded.
|
||||
* The function stops decoding on reaching this objective.
|
||||
* This can be useful to boost performance
|
||||
* whenever only the beginning of a block is required.
|
||||
*
|
||||
* @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize)
|
||||
* If source stream is detected malformed, function returns a negative result.
|
||||
*
|
||||
* Note 1 : @return can be < targetOutputSize, if compressed block contains less data.
|
||||
*
|
||||
* Note 2 : targetOutputSize must be <= dstCapacity
|
||||
*
|
||||
* Note 3 : this function effectively stops decoding on reaching targetOutputSize,
|
||||
* so dstCapacity is kind of redundant.
|
||||
* This is because in older versions of this function,
|
||||
* decoding operation would still write complete sequences.
|
||||
* Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize,
|
||||
* it could write more bytes, though only up to dstCapacity.
|
||||
* Some "margin" used to be required for this operation to work properly.
|
||||
* Thankfully, this is no longer necessary.
|
||||
* The function nonetheless keeps the same signature, in an effort to preserve API compatibility.
|
||||
*
|
||||
* Note 4 : If srcSize is the exact size of the block,
|
||||
* then targetOutputSize can be any value,
|
||||
* including larger than the block's decompressed size.
|
||||
* The function will, at most, generate block's decompressed size.
|
||||
*
|
||||
* Note 5 : If srcSize is _larger_ than block's compressed size,
|
||||
* then targetOutputSize **MUST** be <= block's decompressed size.
|
||||
* Otherwise, *silent corruption will occur*.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
|
||||
|
||||
|
||||
/*-*********************************************
|
||||
* Streaming Compression Functions
|
||||
***********************************************/
|
||||
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
|
||||
|
||||
/*!
|
||||
Note about RC_INVOKED
|
||||
|
||||
- RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
|
||||
https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros
|
||||
|
||||
- Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars)
|
||||
and reports warning "RC4011: identifier truncated".
|
||||
|
||||
- To eliminate the warning, we surround long preprocessor symbol with
|
||||
"#if !defined(RC_INVOKED) ... #endif" block that means
|
||||
"skip this block when rc.exe is trying to read it".
|
||||
*/
|
||||
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
|
||||
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
|
||||
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
|
||||
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
|
||||
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
|
||||
#endif
|
||||
|
||||
/*! LZ4_resetStream_fast() : v1.9.0+
|
||||
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
|
||||
* (e.g., LZ4_compress_fast_continue()).
|
||||
*
|
||||
* An LZ4_stream_t must be initialized once before usage.
|
||||
* This is automatically done when created by LZ4_createStream().
|
||||
* However, should the LZ4_stream_t be simply declared on stack (for example),
|
||||
* it's necessary to initialize it first, using LZ4_initStream().
|
||||
*
|
||||
* After init, start any new stream with LZ4_resetStream_fast().
|
||||
* A same LZ4_stream_t can be re-used multiple times consecutively
|
||||
* and compress multiple streams,
|
||||
* provided that it starts each new stream with LZ4_resetStream_fast().
|
||||
*
|
||||
* LZ4_resetStream_fast() is much faster than LZ4_initStream(),
|
||||
* but is not compatible with memory regions containing garbage data.
|
||||
*
|
||||
* Note: it's only useful to call LZ4_resetStream_fast()
|
||||
* in the context of streaming compression.
|
||||
* The *extState* functions perform their own resets.
|
||||
* Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.
|
||||
*/
|
||||
LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
|
||||
|
||||
/*! LZ4_loadDict() :
|
||||
* Use this function to reference a static dictionary into LZ4_stream_t.
|
||||
* The dictionary must remain available during compression.
|
||||
* LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
|
||||
* The same dictionary will have to be loaded on decompression side for successful decoding.
|
||||
* Dictionary are useful for better compression of small data (KB range).
|
||||
* While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic.
|
||||
* When in doubt, employ the Zstandard's Dictionary Builder.
|
||||
* Loading a size of 0 is allowed, and is the same as reset.
|
||||
* @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded)
|
||||
*/
|
||||
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
|
||||
|
||||
/*! LZ4_loadDictSlow() : v1.10.0+
|
||||
* Same as LZ4_loadDict(),
|
||||
* but uses a bit more cpu to reference the dictionary content more thoroughly.
|
||||
* This is expected to slightly improve compression ratio.
|
||||
* The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions.
|
||||
* @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded)
|
||||
*/
|
||||
LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
|
||||
|
||||
/*! LZ4_attach_dictionary() : stable since v1.10.0
|
||||
*
|
||||
* This allows efficient re-use of a static dictionary multiple times.
|
||||
*
|
||||
* Rather than re-loading the dictionary buffer into a working context before
|
||||
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
|
||||
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
|
||||
* in which the working stream references @dictionaryStream in-place.
|
||||
*
|
||||
* Several assumptions are made about the state of @dictionaryStream.
|
||||
* Currently, only states which have been prepared by LZ4_loadDict() or
|
||||
* LZ4_loadDictSlow() should be expected to work.
|
||||
*
|
||||
* Alternatively, the provided @dictionaryStream may be NULL,
|
||||
* in which case any existing dictionary stream is unset.
|
||||
*
|
||||
* If a dictionary is provided, it replaces any pre-existing stream history.
|
||||
* The dictionary contents are the only history that can be referenced and
|
||||
* logically immediately precede the data compressed in the first subsequent
|
||||
* compression call.
|
||||
*
|
||||
* The dictionary will only remain attached to the working stream through the
|
||||
* first compression call, at the end of which it is cleared.
|
||||
* @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged
|
||||
* through the completion of the compression session.
|
||||
*
|
||||
* Note: there is no equivalent LZ4_attach_*() method on the decompression side
|
||||
* because there is no initialization cost, hence no need to share the cost across multiple sessions.
|
||||
* To decompress LZ4 blocks using dictionary, attached or not,
|
||||
* just employ the regular LZ4_setStreamDecode() for streaming,
|
||||
* or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression.
|
||||
*/
|
||||
LZ4LIB_API void
|
||||
LZ4_attach_dictionary(LZ4_stream_t* workingStream,
|
||||
const LZ4_stream_t* dictionaryStream);
|
||||
|
||||
/*! LZ4_compress_fast_continue() :
|
||||
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
|
||||
* 'dst' buffer must be already allocated.
|
||||
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
|
||||
*
|
||||
* @return : size of compressed block
|
||||
* or 0 if there is an error (typically, cannot fit into 'dst').
|
||||
*
|
||||
* Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
|
||||
* Each block has precise boundaries.
|
||||
* Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.
|
||||
* It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
|
||||
*
|
||||
* Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !
|
||||
*
|
||||
* Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
|
||||
* Make sure that buffers are separated, by at least one byte.
|
||||
* This construction ensures that each block only depends on previous block.
|
||||
*
|
||||
* Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
|
||||
*
|
||||
* Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
/*! LZ4_saveDict() :
|
||||
* If last 64KB data cannot be guaranteed to remain available at its current memory location,
|
||||
* save it into a safer place (char* safeBuffer).
|
||||
* This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
|
||||
* but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
|
||||
* @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
|
||||
|
||||
|
||||
/*-**********************************************
|
||||
* Streaming Decompression Functions
|
||||
* Bufferless synchronous API
|
||||
************************************************/
|
||||
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */
|
||||
|
||||
/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
|
||||
* creation / destruction of streaming decompression tracking context.
|
||||
* A tracking context can be re-used multiple times.
|
||||
*/
|
||||
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
|
||||
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
|
||||
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
|
||||
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
|
||||
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
|
||||
#endif
|
||||
|
||||
/*! LZ4_setStreamDecode() :
|
||||
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
|
||||
* Use this function to start decompression of a new stream of blocks.
|
||||
* A dictionary can optionally be set. Use NULL or size 0 for a reset order.
|
||||
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
|
||||
* @return : 1 if OK, 0 if error
|
||||
*/
|
||||
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
|
||||
|
||||
/*! LZ4_decoderRingBufferSize() : v1.8.2+
|
||||
* Note : in a ring buffer scenario (optional),
|
||||
* blocks are presumed decompressed next to each other
|
||||
* up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
|
||||
* at which stage it resumes from beginning of ring buffer.
|
||||
* When setting such a ring buffer for streaming decompression,
|
||||
* provides the minimum size of this ring buffer
|
||||
* to be compatible with any source respecting maxBlockSize condition.
|
||||
* @return : minimum ring buffer size,
|
||||
* or 0 if there is an error (invalid maxBlockSize).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
|
||||
#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */
|
||||
|
||||
/*! LZ4_decompress_safe_continue() :
|
||||
* This decoding function allows decompression of consecutive blocks in "streaming" mode.
|
||||
* The difference with the usual independent blocks is that
|
||||
* new blocks are allowed to find references into former blocks.
|
||||
* A block is an unsplittable entity, and must be presented entirely to the decompression function.
|
||||
* LZ4_decompress_safe_continue() only accepts one block at a time.
|
||||
* It's modeled after `LZ4_decompress_safe()` and behaves similarly.
|
||||
*
|
||||
* @LZ4_streamDecode : decompression state, tracking the position in memory of past data
|
||||
* @compressedSize : exact complete size of one compressed block.
|
||||
* @dstCapacity : size of destination buffer (which must be already allocated),
|
||||
* must be an upper bound of decompressed size.
|
||||
* @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
|
||||
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
|
||||
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
*
|
||||
* The last 64KB of previously decoded data *must* remain available and unmodified
|
||||
* at the memory position where they were previously decoded.
|
||||
* If less than 64KB of data has been decoded, all the data must be present.
|
||||
*
|
||||
* Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
|
||||
* - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
|
||||
* maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
|
||||
* In which case, encoding and decoding buffers do not need to be synchronized.
|
||||
* Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
|
||||
* - Synchronized mode :
|
||||
* Decompression buffer size is _exactly_ the same as compression buffer size,
|
||||
* and follows exactly same update rule (block boundaries at same positions),
|
||||
* and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
|
||||
* _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||
* - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
|
||||
* In which case, encoding and decoding buffers do not need to be synchronized,
|
||||
* and encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||
*
|
||||
* Whenever these conditions are not possible,
|
||||
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
|
||||
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
|
||||
const char* src, char* dst,
|
||||
int srcSize, int dstCapacity);
|
||||
|
||||
|
||||
/*! LZ4_decompress_safe_usingDict() :
|
||||
* Works the same as
|
||||
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue()
|
||||
* However, it's stateless: it doesn't need any LZ4_streamDecode_t state.
|
||||
* Dictionary is presumed stable : it must remain accessible and unmodified during decompression.
|
||||
* Performance tip : Decompression speed can be substantially increased
|
||||
* when dst == dictStart + dictSize.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_usingDict(const char* src, char* dst,
|
||||
int srcSize, int dstCapacity,
|
||||
const char* dictStart, int dictSize);
|
||||
|
||||
/*! LZ4_decompress_safe_partial_usingDict() :
|
||||
* Behaves the same as LZ4_decompress_safe_partial()
|
||||
* with the added ability to specify a memory segment for past data.
|
||||
* Performance tip : Decompression speed can be substantially increased
|
||||
* when dst == dictStart + dictSize.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
|
||||
int compressedSize,
|
||||
int targetOutputSize, int maxOutputSize,
|
||||
const char* dictStart, int dictSize);
|
||||
|
||||
#endif /* LZ4_H_2983827168210 */
|
||||
|
||||
|
||||
/*^*************************************
|
||||
* !!!!!! STATIC LINKING ONLY !!!!!!
|
||||
***************************************/
|
||||
|
||||
/*-****************************************************************************
|
||||
* Experimental section
|
||||
*
|
||||
* Symbols declared in this section must be considered unstable. Their
|
||||
* signatures or semantics may change, or they may be removed altogether in the
|
||||
* future. They are therefore only safe to depend on when the caller is
|
||||
* statically linked against the library.
|
||||
*
|
||||
* To protect against unsafe usage, not only are the declarations guarded,
|
||||
* the definitions are hidden by default
|
||||
* when building LZ4 as a shared/dynamic library.
|
||||
*
|
||||
* In order to access these declarations,
|
||||
* define LZ4_STATIC_LINKING_ONLY in your application
|
||||
* before including LZ4's headers.
|
||||
*
|
||||
* In order to make their implementations accessible dynamically, you must
|
||||
* define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
|
||||
******************************************************************************/
|
||||
|
||||
#ifdef LZ4_STATIC_LINKING_ONLY
|
||||
|
||||
#ifndef LZ4_STATIC_3504398509
|
||||
#define LZ4_STATIC_3504398509
|
||||
|
||||
#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
|
||||
# define LZ4LIB_STATIC_API LZ4LIB_API
|
||||
#else
|
||||
# define LZ4LIB_STATIC_API
|
||||
#endif
|
||||
|
||||
|
||||
/*! LZ4_compress_fast_extState_fastReset() :
|
||||
* A variant of LZ4_compress_fast_extState().
|
||||
*
|
||||
* Using this variant avoids an expensive initialization step.
|
||||
* It is only safe to call if the state buffer is known to be correctly initialized already
|
||||
* (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").
|
||||
* From a high level, the difference is that
|
||||
* this function initializes the provided state with a call to something like LZ4_resetStream_fast()
|
||||
* while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
|
||||
*/
|
||||
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
/*! LZ4_compress_destSize_extState() : introduced in v1.10.0
|
||||
* Same as LZ4_compress_destSize(), but using an externally allocated state.
|
||||
* Also: exposes @acceleration
|
||||
*/
|
||||
int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration);
|
||||
|
||||
/*! In-place compression and decompression
|
||||
*
|
||||
* It's possible to have input and output sharing the same buffer,
|
||||
* for highly constrained memory environments.
|
||||
* In both cases, it requires input to lay at the end of the buffer,
|
||||
* and decompression to start at beginning of the buffer.
|
||||
* Buffer size must feature some margin, hence be larger than final size.
|
||||
*
|
||||
* |<------------------------buffer--------------------------------->|
|
||||
* |<-----------compressed data--------->|
|
||||
* |<-----------decompressed size------------------>|
|
||||
* |<----margin---->|
|
||||
*
|
||||
* This technique is more useful for decompression,
|
||||
* since decompressed size is typically larger,
|
||||
* and margin is short.
|
||||
*
|
||||
* In-place decompression will work inside any buffer
|
||||
* which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
|
||||
* This presumes that decompressedSize > compressedSize.
|
||||
* Otherwise, it means compression actually expanded data,
|
||||
* and it would be more efficient to store such data with a flag indicating it's not compressed.
|
||||
* This can happen when data is not compressible (already compressed, or encrypted).
|
||||
*
|
||||
* For in-place compression, margin is larger, as it must be able to cope with both
|
||||
* history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
|
||||
* and data expansion, which can happen when input is not compressible.
|
||||
* As a consequence, buffer size requirements are much higher,
|
||||
* and memory savings offered by in-place compression are more limited.
|
||||
*
|
||||
* There are ways to limit this cost for compression :
|
||||
* - Reduce history size, by modifying LZ4_DISTANCE_MAX.
|
||||
* Note that it is a compile-time constant, so all compressions will apply this limit.
|
||||
* Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
|
||||
* so it's a reasonable trick when inputs are known to be small.
|
||||
* - Require the compressor to deliver a "maximum compressed size".
|
||||
* This is the `dstCapacity` parameter in `LZ4_compress*()`.
|
||||
* When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
|
||||
* in which case, the return code will be 0 (zero).
|
||||
* The caller must be ready for these cases to happen,
|
||||
* and typically design a backup scheme to send data uncompressed.
|
||||
* The combination of both techniques can significantly reduce
|
||||
* the amount of margin required for in-place compression.
|
||||
*
|
||||
* In-place compression can work in any buffer
|
||||
* which size is >= (maxCompressedSize)
|
||||
* with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
|
||||
* LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
|
||||
* so it's possible to reduce memory requirements by playing with them.
|
||||
*/
|
||||
|
||||
#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)
|
||||
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
|
||||
|
||||
#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
|
||||
# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
|
||||
#endif
|
||||
|
||||
#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
|
||||
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
|
||||
|
||||
#endif /* LZ4_STATIC_3504398509 */
|
||||
#endif /* LZ4_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
|
||||
#ifndef LZ4_H_98237428734687
|
||||
#define LZ4_H_98237428734687
|
||||
|
||||
/*-************************************************************
|
||||
* Private Definitions
|
||||
**************************************************************
|
||||
* Do not use these definitions directly.
|
||||
* They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
|
||||
* Accessing members will expose user code to API and/or ABI break in future versions of the library.
|
||||
**************************************************************/
|
||||
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
|
||||
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
|
||||
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
|
||||
|
||||
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# include <stdint.h>
|
||||
typedef int8_t LZ4_i8;
|
||||
typedef uint8_t LZ4_byte;
|
||||
typedef uint16_t LZ4_u16;
|
||||
typedef uint32_t LZ4_u32;
|
||||
#else
|
||||
typedef signed char LZ4_i8;
|
||||
typedef unsigned char LZ4_byte;
|
||||
typedef unsigned short LZ4_u16;
|
||||
typedef unsigned int LZ4_u32;
|
||||
#endif
|
||||
|
||||
/*! LZ4_stream_t :
|
||||
* Never ever use below internal definitions directly !
|
||||
* These definitions are not API/ABI safe, and may change in future versions.
|
||||
* If you need static allocation, declare or allocate an LZ4_stream_t object.
|
||||
**/
|
||||
|
||||
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
|
||||
struct LZ4_stream_t_internal {
|
||||
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
|
||||
const LZ4_byte* dictionary;
|
||||
const LZ4_stream_t_internal* dictCtx;
|
||||
LZ4_u32 currentOffset;
|
||||
LZ4_u32 tableType;
|
||||
LZ4_u32 dictSize;
|
||||
/* Implicit padding to ensure structure is aligned */
|
||||
};
|
||||
|
||||
#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */
|
||||
union LZ4_stream_u {
|
||||
char minStateSize[LZ4_STREAM_MINSIZE];
|
||||
LZ4_stream_t_internal internal_donotuse;
|
||||
}; /* previously typedef'd to LZ4_stream_t */
|
||||
|
||||
|
||||
/*! LZ4_initStream() : v1.9.0+
|
||||
* An LZ4_stream_t structure must be initialized at least once.
|
||||
* This is automatically done when invoking LZ4_createStream(),
|
||||
* but it's not when the structure is simply declared on stack (for example).
|
||||
*
|
||||
* Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.
|
||||
* It can also initialize any arbitrary buffer of sufficient size,
|
||||
* and will @return a pointer of proper type upon initialization.
|
||||
*
|
||||
* Note : initialization fails if size and alignment conditions are not respected.
|
||||
* In which case, the function will @return NULL.
|
||||
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
|
||||
* Note3: Before v1.9.0, use LZ4_resetStream() instead
|
||||
**/
|
||||
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size);
|
||||
|
||||
|
||||
/*! LZ4_streamDecode_t :
|
||||
* Never ever use below internal definitions directly !
|
||||
* These definitions are not API/ABI safe, and may change in future versions.
|
||||
* If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
|
||||
**/
|
||||
typedef struct {
|
||||
const LZ4_byte* externalDict;
|
||||
const LZ4_byte* prefixEnd;
|
||||
size_t extDictSize;
|
||||
size_t prefixSize;
|
||||
} LZ4_streamDecode_t_internal;
|
||||
|
||||
#define LZ4_STREAMDECODE_MINSIZE 32
|
||||
union LZ4_streamDecode_u {
|
||||
char minStateSize[LZ4_STREAMDECODE_MINSIZE];
|
||||
LZ4_streamDecode_t_internal internal_donotuse;
|
||||
} ; /* previously typedef'd to LZ4_streamDecode_t */
|
||||
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Obsolete Functions
|
||||
**************************************/
|
||||
|
||||
/*! Deprecation warnings
|
||||
*
|
||||
* Deprecated functions make the compiler generate a warning when invoked.
|
||||
* This is meant to invite users to update their source code.
|
||||
* Should deprecation warnings be a problem, it is generally possible to disable them,
|
||||
* typically with -Wno-deprecated-declarations for gcc
|
||||
* or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
*
|
||||
* Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||
* before including the header file.
|
||||
*/
|
||||
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
|
||||
#else
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
|
||||
# elif defined(_MSC_VER)
|
||||
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
|
||||
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45))
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||
# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31)
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
|
||||
# else
|
||||
# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler")
|
||||
# define LZ4_DEPRECATED(message) /* disabled */
|
||||
# endif
|
||||
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
|
||||
|
||||
/*! Obsolete compression functions (since v1.7.3) */
|
||||
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
/*! Obsolete decompression functions (since v1.8.0) */
|
||||
LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
|
||||
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
|
||||
|
||||
/* Obsolete streaming functions (since v1.7.0)
|
||||
* degraded functionality; do not use!
|
||||
*
|
||||
* In order to perform streaming compression, these functions depended on data
|
||||
* that is no longer tracked in the state. They have been preserved as well as
|
||||
* possible: using them will still produce a correct output. However, they don't
|
||||
* actually retain any history between compression calls. The compression ratio
|
||||
* achieved will therefore be no better than compressing each chunk
|
||||
* independently.
|
||||
*/
|
||||
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
|
||||
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);
|
||||
LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
|
||||
LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
|
||||
|
||||
/*! Obsolete streaming decoding functions (since v1.7.0) */
|
||||
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
|
||||
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
|
||||
|
||||
/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) :
|
||||
* These functions used to be faster than LZ4_decompress_safe(),
|
||||
* but this is no longer the case. They are now slower.
|
||||
* This is because LZ4_decompress_fast() doesn't know the input size,
|
||||
* and therefore must progress more cautiously into the input buffer to not read beyond the end of block.
|
||||
* On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
|
||||
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
|
||||
*
|
||||
* The last remaining LZ4_decompress_fast() specificity is that
|
||||
* it can decompress a block without knowing its compressed size.
|
||||
* Such functionality can be achieved in a more secure manner
|
||||
* by employing LZ4_decompress_safe_partial().
|
||||
*
|
||||
* Parameters:
|
||||
* originalSize : is the uncompressed size to regenerate.
|
||||
* `dst` must be already allocated, its size must be >= 'originalSize' bytes.
|
||||
* @return : number of bytes read from source buffer (== compressed size).
|
||||
* The function expects to finish at block's end exactly.
|
||||
* If the source stream is detected malformed, the function stops decoding and returns a negative result.
|
||||
* note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
|
||||
* However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
|
||||
* Also, since match offsets are not validated, match reads from 'src' may underflow too.
|
||||
* These issues never happen if input (compressed) data is correct.
|
||||
* But they may happen if input data is invalid (error or intentional tampering).
|
||||
* As a consequence, use these functions in trusted environments with trusted data **only**.
|
||||
*/
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead")
|
||||
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. "
|
||||
"Note that the contract will change (requires block's compressed size, instead of decompressed size)")
|
||||
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead")
|
||||
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
|
||||
|
||||
/*! LZ4_resetStream() :
|
||||
* An LZ4_stream_t structure must be initialized at least once.
|
||||
* This is done with LZ4_initStream(), or LZ4_resetStream().
|
||||
* Consider switching to LZ4_initStream(),
|
||||
* invoking LZ4_resetStream() will trigger deprecation warnings in the future.
|
||||
*/
|
||||
LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
|
||||
|
||||
|
||||
#endif /* LZ4_H_98237428734687 */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
|
@ -75,6 +75,7 @@ str_t markdownStr(arena_t *arena, strview_t markdown_str, md_options_t *options)
|
|||
static void markdown__parse_config(arena_t *arena, instream_t *in, ini_t *out) {
|
||||
strview_t first_line = strvTrim(istrGetLine(in));
|
||||
if (!strvEquals(first_line, strv("---"))) {
|
||||
istrRewind(in);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -503,7 +503,7 @@ buffer_t obstrAsBuf(obytestream_t *ctx) {
|
|||
}
|
||||
|
||||
void obstrWrite(obytestream_t *ctx, const void *buf, usize buflen) {
|
||||
uint8 *dst = alloc(ctx->arena, uint8, buflen);
|
||||
uint8 *dst = alloc(ctx->arena, uint8, buflen, ALLOC_NOZERO);
|
||||
memcpy(dst, buf, buflen);
|
||||
}
|
||||
|
||||
|
|
|
|||
50
tools/docs.c
50
tools/docs.c
|
|
@ -17,6 +17,11 @@
|
|||
|
||||
const char *raw_css;
|
||||
|
||||
struct {
|
||||
bool gen;
|
||||
strview_t gendir;
|
||||
} options = {0};
|
||||
|
||||
typedef struct page_t {
|
||||
str_t title;
|
||||
str_t url;
|
||||
|
|
@ -144,14 +149,18 @@ page_t *get_pages(arena_t *arena, strview_t path, strview_t default_page) {
|
|||
divBeg(.class="pages");
|
||||
for_each(item, head) {
|
||||
str_t class = strFmt(&scratch, "page-item%s", item == page ? " page-current" : "");
|
||||
str_t href = strFmt(&scratch, "/%v", item->url);
|
||||
str_t onclick = strFmt(&scratch, "window.location = \"/%v\"", item->url);
|
||||
str_t href = STR_EMPTY;
|
||||
if (options.gen) {
|
||||
href = strFmt(&scratch, "%v.html", item->url);
|
||||
}
|
||||
else {
|
||||
href = strFmt(&scratch, "%v", item->url);
|
||||
}
|
||||
|
||||
a(
|
||||
item->title,
|
||||
.href = href.buf,
|
||||
.class = class.buf,
|
||||
.onclick = onclick.buf
|
||||
);
|
||||
}
|
||||
divEnd();
|
||||
|
|
@ -203,15 +212,48 @@ str_t server_quit(arena_t scratch, server_t *server, server_req_t *req, void *us
|
|||
return STR_EMPTY;
|
||||
}
|
||||
|
||||
int main() {
|
||||
int main(int argc, char **argv) {
|
||||
arena_t arena = arenaMake(ARENA_VIRTUAL, MB(1));
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
strview_t arg = strv(argv[i]);
|
||||
if (strvEquals(arg, strv("-h"))) {
|
||||
info("usage: %s [-h, -gen <outdir>]", argv[0]);
|
||||
return 0;
|
||||
}
|
||||
else if (strvEquals(arg, strv("-gen"))) {
|
||||
options.gen = true;
|
||||
if ((i + 1) < argc) {
|
||||
options.gendir = strv(argv[++i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
page_t *pages = get_pages(&arena, strv("."), strv("readme"));
|
||||
if (!pages) {
|
||||
err("could not get pages");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (options.gen) {
|
||||
if (strvBack(options.gendir) == '/' || strvBack(options.gendir) == '\\') {
|
||||
options.gendir.len -= 1;
|
||||
}
|
||||
|
||||
for_each(page, pages) {
|
||||
arena_t scratch = arena;
|
||||
str_t fname = strFmt(&scratch, "%v/%v.html", options.gendir, page->url);
|
||||
if (!fileWriteWhole(strv(fname), page->data.buf, page->data.len)) {
|
||||
err("couldn't save page %v", fname);
|
||||
}
|
||||
else {
|
||||
info("saved %v", fname);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
server_t *s = serverSetup(&arena, 8080, true);
|
||||
serverRouteDefault(&arena, s, server_default, pages);
|
||||
serverRoute(&arena, s, strv("/quit"), server_quit, NULL);
|
||||
|
|
|
|||
635
vfs.c
Normal file
635
vfs.c
Normal file
|
|
@ -0,0 +1,635 @@
|
|||
#include "vfs.h"
|
||||
|
||||
#include "lz4/lz4.c"
|
||||
|
||||
#include "file.h"
|
||||
#include "strstream.h"
|
||||
#include "arena.h"
|
||||
#include "dir.h"
|
||||
#include "bits.h"
|
||||
|
||||
/*
|
||||
vfs format:
|
||||
=====================
|
||||
header:
|
||||
-----------
|
||||
magic: VFS
|
||||
u32 header_size
|
||||
u32 file_count
|
||||
u8 flags
|
||||
for each file:
|
||||
u64 hash // todo remove
|
||||
u8 namelen
|
||||
char *name
|
||||
u64 offset
|
||||
u64 size
|
||||
u64 compressed_size
|
||||
-----------
|
||||
* binary data *
|
||||
*/
|
||||
|
||||
typedef struct vfsfile_t vfsfile_t;
|
||||
|
||||
struct vfsfile_t {
|
||||
vfsfile_t *next;
|
||||
str_t path;
|
||||
uint64 size;
|
||||
uint64 comp_size;
|
||||
};
|
||||
|
||||
typedef struct vfshmap_t vfshmap_t;
|
||||
typedef struct vfshnode_t vfshnode_t;
|
||||
|
||||
typedef struct {
|
||||
uint64 offset;
|
||||
uint64 size;
|
||||
uint64 compsize;
|
||||
} vfsdata_t;
|
||||
|
||||
struct vfshnode_t {
|
||||
vfshnode_t *next;
|
||||
uint64 hash;
|
||||
str_t key;
|
||||
uint32 index;
|
||||
};
|
||||
|
||||
struct vfshmap_t {
|
||||
vfshnode_t **buckets;
|
||||
vfsdata_t *values;
|
||||
|
||||
uint32 size;
|
||||
uint32 count;
|
||||
uint32 collisions;
|
||||
uint32 max_values;
|
||||
};
|
||||
|
||||
struct virtualfs_t {
|
||||
file_t fp;
|
||||
buffer_t buffer;
|
||||
vfshmap_t hmap;
|
||||
uint64 base_offset;
|
||||
vfsflags_e flags;
|
||||
};
|
||||
|
||||
vfsflags_e vfs_validate_flags(vfsflags_e flags);
|
||||
bool vfs_read(arena_t *arena, virtualfs_t *vfs, vfsdata_t *data, buffer_t *out, bool null_terminate);
|
||||
vfsfile_t *vfs_add_dir(arena_t *arena, strview_t path, vfsfile_t *tail, uint32 *count, uint64 *bytesize);
|
||||
vfshmap_t vfs_hmap_init(arena_t *arena, int pow2, uint32 max_values);
|
||||
void vfs_hmap_add(arena_t *arena, vfshmap_t *hmap, strview_t key, vfsdata_t value);
|
||||
vfsdata_t *vfs_hmap_get(vfshmap_t *hmap, strview_t key);
|
||||
uint64 sdbm_hash(const void *bytes, usize count);
|
||||
uint64 djb2_hash(const void *bytes, usize count);
|
||||
|
||||
bool vfsVirtualiseDir(arena_t scratch, strview_t dirpath, strview_t outfile, vfsflags_e flags) {
|
||||
bool success = false;
|
||||
|
||||
flags = vfs_validate_flags(flags);
|
||||
|
||||
if (strvBack(dirpath) != '/') {
|
||||
str_t newpath = strFmt(&scratch, "%v/", dirpath);
|
||||
dirpath = strv(newpath);
|
||||
}
|
||||
|
||||
uint32 count = 0;
|
||||
uint64 bytesize = 0;
|
||||
vfsfile_t file_head = {0};
|
||||
vfs_add_dir(&scratch, dirpath, &file_head, &count, &bytesize);
|
||||
vfsfile_t *files = file_head.next;
|
||||
|
||||
arena_t comp_arena = {0};
|
||||
|
||||
if (flags & VFS_FLAGS_COMPRESSED) {
|
||||
arena_t comp_arena = arenaMake(ARENA_VIRTUAL, GB(1));
|
||||
|
||||
for_each (file, files) {
|
||||
arena_t tmp = scratch;
|
||||
buffer_t buf = fileReadWhole(&tmp, strv(file->path));
|
||||
usize maxlen = LZ4_compressBound(buf.len);
|
||||
|
||||
uint8 *compressed = alloc(&comp_arena, uint8, maxlen);
|
||||
|
||||
int actual_len = LZ4_compress_default(buf.data, compressed, buf.len, maxlen);
|
||||
assert(actual_len > 0 && actual_len <= maxlen);
|
||||
usize pop = maxlen - (usize)actual_len;
|
||||
|
||||
// pop extra bytes that were allocated but not used
|
||||
arenaPop(&comp_arena, pop);
|
||||
|
||||
file->comp_size = actual_len;
|
||||
}
|
||||
}
|
||||
|
||||
obytestream_t header = obstrInit(&scratch);
|
||||
|
||||
obstrAppendU32(&header, count);
|
||||
obstrAppendU8(&header, flags);
|
||||
|
||||
uint64 offset = 0;
|
||||
for_each (file, files) {
|
||||
assert(file->path.len < 256);
|
||||
uint64 hash = djb2_hash(file->path.buf, file->path.len);
|
||||
|
||||
obstrAppendU64(&header, hash);
|
||||
obstrAppendU8(&header, file->path.len);
|
||||
obstrPuts(&header, strv(file->path));
|
||||
obstrAppendU64(&header, offset);
|
||||
obstrAppendU64(&header, file->size);
|
||||
obstrAppendU64(&header, file->comp_size);
|
||||
|
||||
offset += file->comp_size;
|
||||
}
|
||||
|
||||
buffer_t headerbuf = obstrAsBuf(&header);
|
||||
|
||||
buffer_t binbuf = {0};
|
||||
|
||||
file_t fp = fileOpen(outfile, FILE_WRITE);
|
||||
|
||||
if (!fileIsValid(fp)) {
|
||||
err("could not open file %v", outfile);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
uint32 header_size = headerbuf.len + 3 + sizeof(uint32); // + strlen("VFS") + sizeof(header_size)
|
||||
|
||||
filePuts(fp, strv("VFS"));
|
||||
fileWrite(fp, &header_size, sizeof(header_size));
|
||||
fileWrite(fp, headerbuf.data, headerbuf.len);
|
||||
|
||||
if (flags & VFS_FLAGS_COMPRESSED) {
|
||||
buffer_t compressed = {
|
||||
.data = comp_arena.start,
|
||||
.len = arenaTell(&comp_arena)
|
||||
};
|
||||
fileWrite(fp, compressed.data, compressed.len);
|
||||
}
|
||||
else {
|
||||
for_each (file, files) {
|
||||
arena_t tmp = scratch;
|
||||
buffer_t bin = fileReadWhole(&tmp, strv(file->path));
|
||||
if (flags & VFS_FLAGS_NULL_TERMINATE_FILES) {
|
||||
alloc(&tmp, uint8);
|
||||
bin.len += 1;
|
||||
}
|
||||
fileWrite(fp, bin.data, bin.len);
|
||||
}
|
||||
}
|
||||
|
||||
fileClose(fp);
|
||||
|
||||
success = true;
|
||||
failed:
|
||||
arenaCleanup(&comp_arena);
|
||||
arenaCleanup(&scratch);
|
||||
return success;
|
||||
}
|
||||
|
||||
virtualfs_t *vfsReadFromFile(arena_t *arena, strview_t vfs_file, vfsflags_e flags) {
|
||||
usize pos_before = arenaTell(arena);
|
||||
|
||||
virtualfs_t *vfs = alloc(arena, virtualfs_t);
|
||||
|
||||
file_t fp = fileOpen(vfs_file, FILE_READ);
|
||||
if (!fileIsValid(fp)) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
// read header
|
||||
struct {
|
||||
char magic[3];
|
||||
uint32 size;
|
||||
uint32 file_count;
|
||||
uint8 flags;
|
||||
} header = {0};
|
||||
|
||||
fileRead(fp, &header.magic, sizeof(header.magic));
|
||||
fileRead(fp, &header.size, sizeof(header.size));
|
||||
fileRead(fp, &header.file_count, sizeof(header.file_count));
|
||||
fileRead(fp, &header.flags, sizeof(header.flags));
|
||||
|
||||
if (memcmp(header.magic, "VFS", 3) != 0) {
|
||||
err("VirtualFS: magic characters are wrong: %.3s (0x%x%x%x)", header.magic, header.magic[0], header.magic[1], header.magic[2]);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
uint32 default_pow2 = 1 << 10;
|
||||
uint32 pow2 = bitsNextPow2(header.file_count);
|
||||
pow2 = bitsCtz(max(pow2, default_pow2));
|
||||
|
||||
vfs->hmap = vfs_hmap_init(arena, pow2, header.file_count);
|
||||
|
||||
for (uint32 i = 0; i < header.file_count; ++i) {
|
||||
struct {
|
||||
uint64 hash;
|
||||
char name[256];
|
||||
uint64 offset;
|
||||
uint64 size;
|
||||
uint64 comp;
|
||||
} file = {0};
|
||||
|
||||
uint8 namelen = 0;
|
||||
|
||||
fileRead(fp, &file.hash, sizeof(file.hash));
|
||||
fileRead(fp, &namelen, sizeof(namelen));
|
||||
fileRead(fp, &file.name, namelen);
|
||||
fileRead(fp, &file.offset, sizeof(file.offset));
|
||||
fileRead(fp, &file.size, sizeof(file.size));
|
||||
fileRead(fp, &file.comp, sizeof(file.comp));
|
||||
|
||||
vfsdata_t data = {
|
||||
.offset = file.offset,
|
||||
.size = file.size,
|
||||
.compsize = file.comp,
|
||||
};
|
||||
|
||||
strview_t path = strvInitLen(file.name, namelen);
|
||||
|
||||
vfs_hmap_add(arena, &vfs->hmap, path, data);
|
||||
}
|
||||
|
||||
vfs->flags = vfs_validate_flags(header.flags | flags);
|
||||
vfs->base_offset = header.size;
|
||||
|
||||
if (vfs->flags & VFS_FLAGS_DONT_STREAM) {
|
||||
// get remaining size of the file
|
||||
usize pos = fileTell(fp);
|
||||
fileSeekEnd(fp);
|
||||
usize endpos = fileTell(fp);
|
||||
fileSeek(fp, pos);
|
||||
usize binsize = endpos - pos;
|
||||
// read binary data and save it to buffer for later
|
||||
buffer_t buf = {
|
||||
.data = alloc(arena, uint8, binsize),
|
||||
.len = binsize,
|
||||
};
|
||||
usize read = fileRead(fp, buf.data, buf.len);
|
||||
if (read != buf.len) {
|
||||
err("couldn't read all of the binary data, expected %zu bytes but got %zu", buf.len, read);
|
||||
goto failed;
|
||||
}
|
||||
fileClose(fp);
|
||||
|
||||
vfs->buffer = buf;
|
||||
}
|
||||
else {
|
||||
vfs->fp = fp;
|
||||
}
|
||||
|
||||
return vfs;
|
||||
failed:
|
||||
fileClose(fp);
|
||||
arenaRewind(arena, pos_before);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
buffer_t vfsRead(arena_t *arena, virtualfs_t *vfs, strview_t path) {
|
||||
buffer_t out = {0};
|
||||
usize pos_before = arenaTell(arena);
|
||||
|
||||
if (!vfs) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
vfsdata_t *data = vfs_hmap_get(&vfs->hmap, path);
|
||||
if (!data) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
if (!vfs_read(arena, vfs, data, &out, false)) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
return out;
|
||||
failed:
|
||||
arenaRewind(arena, pos_before);
|
||||
return (buffer_t){0};
|
||||
}
|
||||
|
||||
str_t vfsReadStr(arena_t *arena, virtualfs_t *vfs, strview_t path) {
|
||||
buffer_t buf = {0};
|
||||
usize pos_before = arenaTell(arena);
|
||||
|
||||
if (!vfs) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
vfsdata_t *data = vfs_hmap_get(&vfs->hmap, path);
|
||||
if (!data) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
if (!vfs_read(arena, vfs, data, &buf, true)) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
return (str_t){
|
||||
.buf = buf.data,
|
||||
.len = buf.len,
|
||||
};
|
||||
failed:
|
||||
arenaRewind(arena, pos_before);
|
||||
return STR_EMPTY;
|
||||
}
|
||||
|
||||
// == VFS FILE API ===================================
|
||||
|
||||
virtualfs_t *g_vfs = NULL;
|
||||
|
||||
void vfsSetGlobalVirtualFS(virtualfs_t *vfs) {
|
||||
g_vfs = vfs;
|
||||
}
|
||||
|
||||
bool vfsFileExists(strview_t path) {
|
||||
if (!g_vfs) return false;
|
||||
return vfs_hmap_get(&g_vfs->hmap, path) != NULL;
|
||||
}
|
||||
|
||||
vfs_file_t vfsFileOpen(strview_t name, int mode) {
|
||||
if (!g_vfs) goto failed;
|
||||
if (mode != FILE_READ) {
|
||||
err("VirtualFS: trying to open file (%v) for write, VirtualFS is read only!", name);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
vfsdata_t *data = vfs_hmap_get(&g_vfs->hmap, name);
|
||||
|
||||
return (vfs_file_t){
|
||||
.handle = (uintptr_t)data,
|
||||
};
|
||||
|
||||
failed:
|
||||
return (vfs_file_t){0};
|
||||
}
|
||||
|
||||
void vfsFileClose(vfs_file_t ctx) {
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
bool vfsFileIsValid(vfs_file_t ctx) {
|
||||
return g_vfs && ctx.handle != 0;
|
||||
}
|
||||
|
||||
usize vfsFileSize(vfs_file_t ctx) {
|
||||
if (!vfsFileIsValid(ctx)) return 0;
|
||||
vfsdata_t *data = (vfsdata_t *)ctx.handle;
|
||||
return data->size;
|
||||
}
|
||||
|
||||
buffer_t vfsFileReadWhole(arena_t *arena, strview_t name) {
|
||||
return vfsRead(arena, g_vfs, name);
|
||||
}
|
||||
|
||||
buffer_t vfsFileReadWholeFP(arena_t *arena, vfs_file_t ctx) {
|
||||
if (!vfsFileIsValid(ctx)) return (buffer_t){0};
|
||||
vfsdata_t *data = (vfsdata_t *)ctx.handle;
|
||||
buffer_t out = {0};
|
||||
usize pos_before = arenaTell(arena);
|
||||
if (!vfs_read(arena, g_vfs, data, &out, false)) {
|
||||
arenaRewind(arena, pos_before);
|
||||
return (buffer_t){0};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
str_t vfsFileReadWholeStr(arena_t *arena, strview_t name) {
|
||||
return vfsReadStr(arena, g_vfs, name);
|
||||
}
|
||||
|
||||
str_t vfsFileReadWholeStrFP(arena_t *arena, vfs_file_t ctx) {
|
||||
if (!vfsFileIsValid(ctx)) return STR_EMPTY;
|
||||
vfsdata_t *data = (vfsdata_t *)ctx.handle;
|
||||
buffer_t buf = {0};
|
||||
usize pos_before = arenaTell(arena);
|
||||
if (!vfs_read(arena, g_vfs, data, &buf, true)) {
|
||||
arenaRewind(arena, pos_before);
|
||||
return STR_EMPTY;
|
||||
}
|
||||
return (str_t){
|
||||
.buf = buf.data,
|
||||
.len = buf.len,
|
||||
};
|
||||
}
|
||||
|
||||
// == PRIVATE FUNCTIONS ==============================
|
||||
|
||||
vfsflags_e vfs_validate_flags(vfsflags_e flags) {
|
||||
if (flags & VFS_FLAGS_COMPRESSED && flags & VFS_FLAGS_NULL_TERMINATE_FILES) {
|
||||
warn("VirtualFS: both COMPRESSEd and NULL_TERMINATE_FILES flags are set to ON, but they are mutually exclusive. turning NULL_TERMINATE_FILES off");
|
||||
flags &= ~VFS_FLAGS_NULL_TERMINATE_FILES;
|
||||
}
|
||||
|
||||
return flags;
|
||||
}
|
||||
|
||||
bool vfs_read(arena_t *arena, virtualfs_t *vfs, vfsdata_t *data, buffer_t *out, bool null_terminate) {
|
||||
if (!vfs || !data || !out) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool is_allocated = true;
|
||||
out->len = data->size;
|
||||
|
||||
if (vfs->flags & VFS_FLAGS_COMPRESSED) {
|
||||
out->data = alloc(arena, uint8, out->len);
|
||||
|
||||
uint8 *compressed = NULL;
|
||||
|
||||
if (vfs->flags & VFS_FLAGS_DONT_STREAM) {
|
||||
assert((data->offset + data->compsize) < vfs->buffer.len);
|
||||
compressed = vfs->buffer.data + data->offset;
|
||||
}
|
||||
else {
|
||||
uint64 offset = vfs->base_offset + data->offset;
|
||||
fileSeek(vfs->fp, offset);
|
||||
|
||||
arena_t scratch = *arena;
|
||||
uint8 *compressed = alloc(&scratch, uint8, data->compsize);
|
||||
usize read = fileRead(vfs->fp, compressed, data->compsize);
|
||||
if (read != data->compsize) {
|
||||
err("VirtualFS: read %zu bytes, but should have read %zu", read, data->compsize);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
int decompsize = LZ4_decompress_safe(compressed, out->data, data->compsize, out->len);
|
||||
if (decompsize < 0) {
|
||||
err("VirtualFS: couldn't decompress buffer: %d", decompsize);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (vfs->flags & VFS_FLAGS_DONT_STREAM) {
|
||||
assert((data->offset + data->size) < vfs->buffer.len);
|
||||
out->data = vfs->buffer.data + data->offset;
|
||||
is_allocated = false;
|
||||
}
|
||||
else {
|
||||
out->data = alloc(arena, uint8, data->size);
|
||||
|
||||
uint64 offset = vfs->base_offset + data->offset;
|
||||
fileSeek(vfs->fp, offset);
|
||||
usize read = fileRead(vfs->fp, out->data, out->len);
|
||||
if (read != out->len) {
|
||||
err("VirtualFS: read %zu bytes, but should have read %zu", read, out->len);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (null_terminate && !(vfs->flags & VFS_FLAGS_NULL_TERMINATE_FILES)) {
|
||||
if (is_allocated) {
|
||||
alloc(arena, char);
|
||||
}
|
||||
else {
|
||||
uint8 *buf = alloc(arena, uint8, out->len + 1);
|
||||
memcpy(buf, out->data, out->len);
|
||||
out->data = buf;
|
||||
}
|
||||
out->len += 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
vfsfile_t *vfs_add_dir(arena_t *arena, strview_t path, vfsfile_t *tail, uint32 *count, uint64 *bytesize) {
|
||||
uint8 tmpbuf[KB(1)];
|
||||
|
||||
dir_t *dir = dirOpen(arena, path);
|
||||
dir_entry_t *entry = NULL;
|
||||
|
||||
if (strvEquals(path, strv("./"))) {
|
||||
path = STRV_EMPTY;
|
||||
}
|
||||
|
||||
vfsfile_t *head = tail;
|
||||
vfsfile_t *cur = tail;
|
||||
|
||||
while ((entry = dirNext(arena, dir))) {
|
||||
arena_t scratch = arenaMake(ARENA_STATIC, sizeof(tmpbuf), tmpbuf);
|
||||
|
||||
vfsfile_t *newfile = NULL;
|
||||
|
||||
if (entry->type == DIRTYPE_DIR) {
|
||||
if (strvEquals(strv(entry->name), strv(".")) ||
|
||||
strvEquals(strv(entry->name), strv(".."))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
str_t fullpath = strFmt(&scratch, "%v%v/", path, entry->name);
|
||||
newfile = vfs_add_dir(arena, strv(fullpath), cur, count, bytesize);
|
||||
}
|
||||
else {
|
||||
newfile = alloc(arena, vfsfile_t);
|
||||
newfile->path = strFmt(arena, "%v%v", path, entry->name);
|
||||
newfile->size = entry->filesize;
|
||||
newfile->comp_size = newfile->size;
|
||||
|
||||
if (cur) cur->next = newfile;
|
||||
(*count)++;
|
||||
(*bytesize) += newfile->size;
|
||||
}
|
||||
|
||||
if (!head) head = newfile;
|
||||
cur = newfile;
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
|
||||
// == HASH MAP =======================================
|
||||
|
||||
vfshmap_t vfs_hmap_init(arena_t *arena, int pow2, uint32 max_values) {
|
||||
uint size = 1 << pow2;
|
||||
return (vfshmap_t) {
|
||||
.size = size,
|
||||
.max_values = max_values,
|
||||
.buckets = alloc(arena, vfshnode_t*, size),
|
||||
.values = alloc(arena, vfsdata_t, max_values),
|
||||
};
|
||||
}
|
||||
|
||||
void vfs_hmap_add(arena_t *arena, vfshmap_t *hmap, strview_t key, vfsdata_t value) {
|
||||
if (!hmap) return;
|
||||
|
||||
if ((float)hmap->count >= (float)hmap->size * 0.6f) {
|
||||
warn("more than 60%% of the hashmap is being used: %d/%d", hmap->count, hmap->size);
|
||||
}
|
||||
|
||||
uint64 hash = djb2_hash(key.buf, key.len);
|
||||
usize index = hash & (hmap->size - 1);
|
||||
vfshnode_t *bucket = hmap->buckets[index];
|
||||
if (bucket) hmap->collisions++;
|
||||
while (bucket) {
|
||||
// already exists
|
||||
if (bucket->hash == hash && strvEquals(strv(bucket->key), key)) {
|
||||
hmap->values[bucket->index] = value;
|
||||
return;
|
||||
}
|
||||
bucket = bucket->next;
|
||||
}
|
||||
|
||||
assert(hmap->count < hmap->max_values);
|
||||
|
||||
bucket = alloc(arena, vfshnode_t);
|
||||
|
||||
bucket->hash = hash;
|
||||
bucket->key = str(arena, key);
|
||||
bucket->index = hmap->count;
|
||||
bucket->next = hmap->buckets[index];
|
||||
|
||||
hmap->values[hmap->count++] = value;
|
||||
hmap->buckets[index] = bucket;
|
||||
}
|
||||
|
||||
vfsdata_t *vfs_hmap_get(vfshmap_t *hmap, strview_t key) {
|
||||
if (!hmap || hmap->count == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint64 hash = djb2_hash(key.buf, key.len);
|
||||
usize index = hash & (hmap->size - 1);
|
||||
vfshnode_t *bucket = hmap->buckets[index];
|
||||
while (bucket) {
|
||||
if (bucket->hash == hash && strvEquals(strv(bucket->key), key)) {
|
||||
return &hmap->values[bucket->index];
|
||||
}
|
||||
bucket = bucket->next;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint64 sdbm_hash(const void *bytes, usize count) {
|
||||
const uint8 *data = bytes;
|
||||
uint64 hash = 0;
|
||||
|
||||
for (usize i = 0; i < count; ++i) {
|
||||
hash = data[i] + (hash << 6) + (hash << 16) - hash;
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
uint64 djb2_hash(const void *bytes, usize count) {
|
||||
const uint8 *data = bytes;
|
||||
uint64 hash = 5381;
|
||||
int c;
|
||||
|
||||
for (usize i = 0; i < count; ++i) {
|
||||
hash = ((hash << 5) + hash) + data[i];
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
uint64 java_hash(const void *bytes, usize count) {
|
||||
const uint8 *data = bytes;
|
||||
uint64 hash = 1125899906842597L;
|
||||
|
||||
for (usize i = 0; i < count; ++i) {
|
||||
hash = ((hash << 5) - hash) + data[i];
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
36
vfs.h
Normal file
36
vfs.h
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
#pragma once
|
||||
|
||||
#include "str.h"
|
||||
#include "arena.h"
|
||||
|
||||
typedef enum {
|
||||
VFS_FLAGS_NONE = 0,
|
||||
VFS_FLAGS_COMPRESSED = 1 << 0,
|
||||
VFS_FLAGS_NULL_TERMINATE_FILES = 1 << 1, // only valid when compress is false
|
||||
VFS_FLAGS_DONT_STREAM = 1 << 2,
|
||||
} vfsflags_e;
|
||||
|
||||
typedef struct virtualfs_t virtualfs_t;
|
||||
|
||||
bool vfsVirtualiseDir(arena_t scratch, strview_t dirpath, strview_t outfile, vfsflags_e flags);
|
||||
virtualfs_t *vfsReadFromFile(arena_t *arena, strview_t vfs_file, vfsflags_e flags);
|
||||
buffer_t vfsRead(arena_t *arena, virtualfs_t *vfs, strview_t path);
|
||||
str_t vfsReadStr(arena_t *arena, virtualfs_t *vfs, strview_t path);
|
||||
|
||||
// vfs replacement for the file api
|
||||
|
||||
typedef struct {
|
||||
uintptr_t handle;
|
||||
} vfs_file_t;
|
||||
|
||||
void vfsSetGlobalVirtualFS(virtualfs_t *vfs);
|
||||
|
||||
bool vfsFileExists(strview_t path);
|
||||
vfs_file_t vfsFileOpen(strview_t name, int mode);
|
||||
void vfsFileClose(vfs_file_t ctx);
|
||||
bool vfsFileIsValid(vfs_file_t ctx);
|
||||
usize vfsFileSize(vfs_file_t ctx);
|
||||
buffer_t vfsFileReadWhole(arena_t *arena, strview_t name);
|
||||
buffer_t vfsFileReadWholeFP(arena_t *arena, vfs_file_t ctx);
|
||||
str_t vfsFileReadWholeStr(arena_t *arena, strview_t name);
|
||||
str_t vfsFileReadWholeStrFP(arena_t *arena, vfs_file_t ctx);
|
||||
34
vfs_file.h
Normal file
34
vfs_file.h
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#pragma once
|
||||
|
||||
#include "vfs.h"
|
||||
#include "tracelog.h"
|
||||
|
||||
#define fileExists vfsFileExists
|
||||
#define fileOpen vfsFileOpen
|
||||
#define fileClose vfsFileClose
|
||||
#define fileIsValid vfsFileIsValid
|
||||
#define fileSize vfsFileSize
|
||||
#define fileReadWhole vfsFileReadWhole
|
||||
#define fileReadWholeFP vfsFileReadWholeFP
|
||||
#define fileReadWholeStr vfsFileReadWholeStr
|
||||
#define fileReadWholeStrFP vfsFileReadWholeStrFP
|
||||
|
||||
#define VFS_FAIL_READONLY(fn) err("VirtualFS: trying to call function " #fn "which requires writing to files. VirtualFS is read only!")
|
||||
#define VFS_FAIL_STATELESS(fn) err("VirtualFS: trying to call function " #fn "which requires state. VirtualFS only works on stateless file's commands!")
|
||||
|
||||
#define fileDelete(...) VFS_FAIL_READONLY(fileDelete)
|
||||
#define filePutc(...) VFS_FAIL_READONLY(filePutc)
|
||||
#define filePuts(...) VFS_FAIL_READONLY(filePuts)
|
||||
#define filePrintf(...) VFS_FAIL_READONLY(filePrintf)
|
||||
#define filePrintfv(...) VFS_FAIL_READONLY(filePrintfv)
|
||||
#define fileWrite(...) VFS_FAIL_READONLY(fileWrite)
|
||||
#define fileWriteWhole(...) VFS_FAIL_READONLY(fileWriteWhole)
|
||||
#define fileGetTime(...) VFS_FAIL_READONLY(fileGetTime)
|
||||
#define fileGetTimeFP(...) VFS_FAIL_READONLY(fileGetTimeFP)
|
||||
#define fileHasChanged(...) VFS_FAIL_READONLY(fileHasChanged)
|
||||
|
||||
#define fileTell(...) VFS_FAIL_STATELESS(fileTell)
|
||||
#define fileRead(...) VFS_FAIL_STATELESS(fileRead)
|
||||
#define fileSeek(...) VFS_FAIL_STATELESS(fileSeek)
|
||||
#define fileSeekEnd(...) VFS_FAIL_STATELESS(fileSeekEnd)
|
||||
#define fileRewind(...) VFS_FAIL_STATELESS(fileRewind)
|
||||
Loading…
Add table
Add a link
Reference in a new issue