From: Tulio A M Mendes Date: Mon, 16 Feb 2026 18:03:36 +0000 (-0300) Subject: feat: USTAR+LZ4 compressed initrd X-Git-Url: https://projects.tadryanom.me/docs/static/git-favicon.png?a=commitdiff_plain;h=c0e8f3d5be74802c0156e2b2e052ad46847ecaa7;p=AdrOS.git feat: USTAR+LZ4 compressed initrd Add LZ4 block compression to the initrd pipeline: - src/kernel/lz4.c + include/lz4.h: standalone LZ4 block decompressor (~80 lines, no external dependencies) - src/drivers/initrd.c: auto-detect LZ4B magic at boot, decompress into heap buffer, then parse the contained USTAR tar as before - tools/mkinitrd.c: built-in LZ4 block compressor (greedy hash-table), builds tar in memory then wraps in LZ4B envelope (magic + orig_size + comp_size + compressed data) Format: LZ4B header (12 bytes) + raw LZ4 block. Falls back to uncompressed tar if compression fails. Results on current initrd (12 files including doom.elf): TAR: 562 KB -> LZ4B: 326 KB (58% ratio) Backward compatible: kernel still accepts plain USTAR tar (no LZ4B magic = parse directly). 83/83 smoke tests pass (10s), cppcheck clean. --- diff --git a/include/lz4.h b/include/lz4.h new file mode 100644 index 0000000..f7231d5 --- /dev/null +++ b/include/lz4.h @@ -0,0 +1,33 @@ +#ifndef LZ4_H +#define LZ4_H + +#include +#include + +/* + * LZ4 block decompressor for AdrOS. + * + * Decompresses a single LZ4 block (raw format, no frame header). + * + * src - pointer to compressed data + * src_size - size of compressed data in bytes + * dst - pointer to output buffer (must be >= dst_cap bytes) + * dst_cap - capacity of the output buffer + * + * Returns the number of bytes written to dst, or negative on error. + */ +int lz4_decompress_block(const void* src, size_t src_size, + void* dst, size_t dst_cap); + +/* + * InitRD LZ4 wrapper header (prepended to compressed tar): + * [0..3] magic "LZ4B" + * [4..7] orig_sz uint32_t LE — uncompressed size + * [8..11] comp_sz uint32_t LE — compressed block size + * [12..] LZ4 compressed block data + */ +#define LZ4B_MAGIC "LZ4B" +#define LZ4B_MAGIC_U32 0x42345A4CU /* "LZ4B" as little-endian uint32 */ +#define LZ4B_HDR_SIZE 12 + +#endif /* LZ4_H */ diff --git a/src/drivers/initrd.c b/src/drivers/initrd.c index f12ec54..232eaa7 100644 --- a/src/drivers/initrd.c +++ b/src/drivers/initrd.c @@ -3,6 +3,7 @@ #include "heap.h" #include "console.h" #include "errno.h" +#include "lz4.h" #define TAR_BLOCK 512 @@ -223,13 +224,42 @@ static void initrd_finalize_nodes(void) { } fs_node_t* initrd_init(uint32_t location) { + const uint8_t* raw = (const uint8_t*)(uintptr_t)location; + uint8_t* decomp_buf = NULL; + + /* Detect LZ4B compressed initrd */ + if (raw[0] == 'L' && raw[1] == 'Z' && raw[2] == '4' && raw[3] == 'B') { + uint32_t orig_sz = (uint32_t)raw[4] | ((uint32_t)raw[5] << 8) | + ((uint32_t)raw[6] << 16) | ((uint32_t)raw[7] << 24); + uint32_t comp_sz = (uint32_t)raw[8] | ((uint32_t)raw[9] << 8) | + ((uint32_t)raw[10] << 16) | ((uint32_t)raw[11] << 24); + + decomp_buf = (uint8_t*)kmalloc(orig_sz); + if (!decomp_buf) { + kprintf("[INITRD] OOM decompressing LZ4 (%u bytes)\n", orig_sz); + return 0; + } + + int ret = lz4_decompress_block(raw + LZ4B_HDR_SIZE, comp_sz, + decomp_buf, orig_sz); + if (ret < 0 || (uint32_t)ret != orig_sz) { + kprintf("[INITRD] LZ4 decompress failed (ret=%d, expected=%u)\n", + ret, orig_sz); + kfree(decomp_buf); + return 0; + } + + kprintf("[INITRD] LZ4: %u -> %u bytes\n", comp_sz, orig_sz); + location = (uint32_t)(uintptr_t)decomp_buf; + } + initrd_location_base = location; // Initialize root entry_count = 0; int root = entry_alloc(); - if (root < 0) return 0; + if (root < 0) { kfree(decomp_buf); return 0; } strcpy(entries[root].name, ""); entries[root].flags = FS_DIRECTORY; entries[root].data_offset = 0; diff --git a/src/kernel/lz4.c b/src/kernel/lz4.c new file mode 100644 index 0000000..ee96dde --- /dev/null +++ b/src/kernel/lz4.c @@ -0,0 +1,79 @@ +#include "lz4.h" + +/* + * LZ4 block decompressor — minimal, standalone, no dependencies beyond memcpy. + * + * Reference: https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md + * + * Each sequence: + * token byte — high nibble = literal length, low nibble = match length - 4 + * [extra literal length bytes if high nibble == 15] + * literal data + * (if not last sequence): + * match offset — 2 bytes little-endian + * [extra match length bytes if low nibble == 15] + */ + +int lz4_decompress_block(const void *src, size_t src_size, + void *dst, size_t dst_cap) +{ + const uint8_t *ip = (const uint8_t *)src; + const uint8_t *ip_end = ip + src_size; + uint8_t *op = (uint8_t *)dst; + uint8_t *op_end = op + dst_cap; + + for (;;) { + if (ip >= ip_end) return -1; /* truncated input */ + + /* --- token --- */ + uint8_t token = *ip++; + size_t lit_len = (size_t)(token >> 4); + size_t match_len = (size_t)(token & 0x0F); + + /* extended literal length */ + if (lit_len == 15) { + uint8_t extra; + do { + if (ip >= ip_end) return -1; + extra = *ip++; + lit_len += extra; + } while (extra == 255); + } + + /* copy literals */ + if (ip + lit_len > ip_end) return -1; + if (op + lit_len > op_end) return -1; + for (size_t i = 0; i < lit_len; i++) op[i] = ip[i]; + ip += lit_len; + op += lit_len; + + /* last sequence has no match part — ends right after literals */ + if (ip >= ip_end) break; + + /* --- match offset (16-bit LE) --- */ + if (ip + 2 > ip_end) return -1; + size_t offset = (size_t)ip[0] | ((size_t)ip[1] << 8); + ip += 2; + if (offset == 0) return -1; /* offset 0 is invalid */ + + /* extended match length */ + if (match_len == 15) { + uint8_t extra; + do { + if (ip >= ip_end) return -1; + extra = *ip++; + match_len += extra; + } while (extra == 255); + } + match_len += 4; /* minimum match length is 4 */ + + /* copy match (byte-by-byte for overlapping copies) */ + const uint8_t *match_src = op - offset; + if (match_src < (const uint8_t *)dst) return -1; /* underflow */ + if (op + match_len > op_end) return -1; + for (size_t i = 0; i < match_len; i++) op[i] = match_src[i]; + op += match_len; + } + + return (int)(op - (uint8_t *)dst); +} diff --git a/tools/mkinitrd.c b/tools/mkinitrd.c index 108fdd2..5c40131 100644 --- a/tools/mkinitrd.c +++ b/tools/mkinitrd.c @@ -5,6 +5,131 @@ #define TAR_BLOCK 512 +/* ---- LZ4 block compressor (standalone, no external dependency) ---- */ + +#define LZ4_HASH_BITS 16 +#define LZ4_HASH_SIZE (1 << LZ4_HASH_BITS) +#define LZ4_MIN_MATCH 4 +#define LZ4_LAST_LITERALS 5 /* last 5 bytes are always literals */ + +static uint32_t lz4_hash4(const uint8_t *p) { + uint32_t v; + memcpy(&v, p, 4); + return (v * 2654435761U) >> (32 - LZ4_HASH_BITS); +} + +/* + * Compress src[0..src_size) into dst[0..dst_cap). + * Returns compressed size, or 0 on failure (output too large). + */ +static size_t lz4_compress_block(const uint8_t *src, size_t src_size, + uint8_t *dst, size_t dst_cap) +{ + if (src_size == 0) return 0; + if (src_size > 0x7E000000) return 0; /* too large */ + + uint32_t *htab = calloc(LZ4_HASH_SIZE, sizeof(uint32_t)); + if (!htab) return 0; + + const uint8_t *ip = src; + const uint8_t *ip_end = src + src_size; + const uint8_t *ip_limit = ip_end - LZ4_LAST_LITERALS; + const uint8_t *anchor = ip; /* start of pending literals */ + uint8_t *op = dst; + uint8_t *op_end = dst + dst_cap; + + ip++; /* first byte can't match */ + + while (ip < ip_limit) { + /* find a match */ + uint32_t h = lz4_hash4(ip); + const uint8_t *ref = src + htab[h]; + htab[h] = (uint32_t)(ip - src); + + if (ref < src || ip - ref > 65535 || + memcmp(ip, ref, 4) != 0) { + ip++; + continue; + } + + /* extend match forward */ + size_t match_len = LZ4_MIN_MATCH; + while (ip + match_len < ip_end && ip[match_len] == ref[match_len]) + match_len++; + + /* emit sequence */ + size_t lit_len = (size_t)(ip - anchor); + size_t token_pos_needed = 1 + (lit_len >= 15 ? 1 + lit_len / 255 : 0) + + lit_len + 2 + + (match_len - 4 >= 15 ? 1 + (match_len - 4 - 15) / 255 : 0); + if (op + token_pos_needed > op_end) { free(htab); return 0; } + + /* token byte */ + size_t ml_code = match_len - LZ4_MIN_MATCH; + uint8_t token = (uint8_t)((lit_len >= 15 ? 15 : lit_len) << 4); + token |= (uint8_t)(ml_code >= 15 ? 15 : ml_code); + *op++ = token; + + /* extended literal length */ + if (lit_len >= 15) { + size_t rem = lit_len - 15; + while (rem >= 255) { *op++ = 255; rem -= 255; } + *op++ = (uint8_t)rem; + } + + /* literal data */ + memcpy(op, anchor, lit_len); + op += lit_len; + + /* match offset (16-bit LE) */ + uint16_t off = (uint16_t)(ip - ref); + *op++ = (uint8_t)(off & 0xFF); + *op++ = (uint8_t)(off >> 8); + + /* extended match length */ + if (ml_code >= 15) { + size_t rem = ml_code - 15; + while (rem >= 255) { *op++ = 255; rem -= 255; } + *op++ = (uint8_t)rem; + } + + ip += match_len; + anchor = ip; + } + + /* emit remaining literals */ + { + size_t lit_len = (size_t)(ip_end - anchor); + size_t needed = 1 + (lit_len >= 15 ? 1 + lit_len / 255 : 0) + lit_len; + if (op + needed > op_end) { free(htab); return 0; } + + uint8_t token = (uint8_t)((lit_len >= 15 ? 15 : lit_len) << 4); + *op++ = token; + if (lit_len >= 15) { + size_t rem = lit_len - 15; + while (rem >= 255) { *op++ = 255; rem -= 255; } + *op++ = (uint8_t)rem; + } + memcpy(op, anchor, lit_len); + op += lit_len; + } + + free(htab); + return (size_t)(op - dst); +} + +/* LZ4B header: magic(4) + orig_size(4) + comp_size(4) */ +#define LZ4B_HDR_SIZE 12 + +static void write_le32(uint8_t *p, uint32_t v) { + p[0] = (uint8_t)(v); + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); +} + +/* ---- end LZ4 ---- */ + typedef struct { char name[100]; char mode[8]; @@ -51,43 +176,6 @@ static uint32_t tar_checksum(const tar_header_t* h) { return sum; } -static void tar_write_header(FILE* out, const char* name, uint32_t size, char typeflag) { - tar_header_t h; - memset(&h, 0, sizeof(h)); - - // Minimal USTAR header; we keep paths in the name field. - strncpy(h.name, name, sizeof(h.name) - 1); - tar_write_octal(h.mode, sizeof(h.mode), 0644); - tar_write_octal(h.uid, sizeof(h.uid), 0); - tar_write_octal(h.gid, sizeof(h.gid), 0); - tar_write_octal(h.size, sizeof(h.size), size); - tar_write_octal(h.mtime, sizeof(h.mtime), 0); - - memset(h.chksum, ' ', sizeof(h.chksum)); - h.typeflag = typeflag; - memcpy(h.magic, "ustar", 5); - memcpy(h.version, "00", 2); - - uint32_t sum = tar_checksum(&h); - // chksum is 6 digits, NUL, space - tar_write_octal(h.chksum, 7, sum); - h.chksum[6] = '\0'; - h.chksum[7] = ' '; - - fwrite(&h, 1, sizeof(h), out); -} - -static void write_zeros(FILE* out, size_t n) { - static uint8_t z[TAR_BLOCK]; - memset(z, 0, sizeof(z)); - while (n) { - size_t chunk = n; - if (chunk > sizeof(z)) chunk = sizeof(z); - fwrite(z, 1, chunk, out); - n -= chunk; - } -} - static int split_src_dest(const char* arg, char* src_out, size_t src_sz, char* dest_out, size_t dest_sz) { const char* colon = strchr(arg, ':'); if (!colon) return 0; @@ -113,13 +201,13 @@ int main(int argc, char* argv[]) { const char* out_name = argv[1]; int nfiles = argc - 2; - FILE* out = fopen(out_name, "wb"); - if (!out) { - perror("fopen output"); - return 1; - } + /* Build the tar archive in memory so we can compress it */ + size_t tar_cap = 4 * 1024 * 1024; /* 4MB initial */ + uint8_t* tar_buf = malloc(tar_cap); + if (!tar_buf) { perror("malloc"); return 1; } + size_t tar_len = 0; - printf("Creating InitRD (TAR USTAR) with %d files...\n", nfiles); + printf("Creating InitRD (USTAR+LZ4) with %d files...\n", nfiles); for (int i = 0; i < nfiles; i++) { char src[256]; @@ -143,49 +231,101 @@ int main(int argc, char* argv[]) { FILE* in = fopen(src, "rb"); if (!in) { perror("fopen input"); - fclose(out); + free(tar_buf); return 1; } fseek(in, 0, SEEK_END); long len = ftell(in); fseek(in, 0, SEEK_SET); - if (len < 0) { - fclose(in); - fclose(out); - return 1; + if (len < 0) { fclose(in); free(tar_buf); return 1; } + + uint32_t pad = (uint32_t)((TAR_BLOCK - ((uint32_t)len % TAR_BLOCK)) % TAR_BLOCK); + size_t needed = TAR_BLOCK + (size_t)len + pad; + + while (tar_len + needed > tar_cap) { + tar_cap *= 2; + tar_buf = realloc(tar_buf, tar_cap); + if (!tar_buf) { perror("realloc"); fclose(in); return 1; } } - tar_write_header(out, dest, (uint32_t)len, '0'); - - uint8_t buf[4096]; - long remaining = len; - while (remaining > 0) { - size_t chunk = (size_t)remaining; - if (chunk > sizeof(buf)) chunk = sizeof(buf); - size_t rd = fread(buf, 1, chunk, in); - if (rd != chunk) { - fclose(in); - fclose(out); - return 1; - } - fwrite(buf, 1, rd, out); - remaining -= (long)rd; + /* Write header into buffer */ + { + tar_header_t h; + memset(&h, 0, sizeof(h)); + strncpy(h.name, dest, sizeof(h.name) - 1); + tar_write_octal(h.mode, sizeof(h.mode), 0644); + tar_write_octal(h.uid, sizeof(h.uid), 0); + tar_write_octal(h.gid, sizeof(h.gid), 0); + tar_write_octal(h.size, sizeof(h.size), (uint32_t)len); + tar_write_octal(h.mtime, sizeof(h.mtime), 0); + memset(h.chksum, ' ', sizeof(h.chksum)); + h.typeflag = '0'; + memcpy(h.magic, "ustar", 5); + memcpy(h.version, "00", 2); + uint32_t sum = tar_checksum(&h); + tar_write_octal(h.chksum, 7, sum); + h.chksum[6] = '\0'; + h.chksum[7] = ' '; + memcpy(tar_buf + tar_len, &h, sizeof(h)); + tar_len += sizeof(h); } + /* Read file data */ + size_t rd = fread(tar_buf + tar_len, 1, (size_t)len, in); + if ((long)rd != len) { fclose(in); free(tar_buf); return 1; } + tar_len += (size_t)len; fclose(in); - // pad file to 512 - uint32_t pad = (uint32_t)((TAR_BLOCK - ((uint32_t)len % TAR_BLOCK)) % TAR_BLOCK); - if (pad) write_zeros(out, pad); + /* Pad to 512 */ + if (pad) { memset(tar_buf + tar_len, 0, pad); tar_len += pad; } + } + + /* Two zero blocks end-of-archive */ + while (tar_len + TAR_BLOCK * 2 > tar_cap) { + tar_cap *= 2; + tar_buf = realloc(tar_buf, tar_cap); + if (!tar_buf) { perror("realloc"); return 1; } } + memset(tar_buf + tar_len, 0, TAR_BLOCK * 2); + tar_len += TAR_BLOCK * 2; + + printf("TAR size: %zu bytes\n", tar_len); + + /* Compress with LZ4 */ + size_t comp_cap = tar_len + tar_len / 255 + 16; /* worst case */ + uint8_t* comp_buf = malloc(comp_cap); + if (!comp_buf) { perror("malloc comp"); free(tar_buf); return 1; } - // Two zero blocks end-of-archive - write_zeros(out, TAR_BLOCK * 2); + size_t comp_sz = lz4_compress_block(tar_buf, tar_len, comp_buf, comp_cap); + if (comp_sz == 0) { + printf("LZ4 compression failed, writing uncompressed tar.\n"); + FILE* out = fopen(out_name, "wb"); + if (!out) { perror("fopen"); free(tar_buf); free(comp_buf); return 1; } + fwrite(tar_buf, 1, tar_len, out); + fclose(out); + printf("Done. InitRD size: %zu bytes (uncompressed).\n", tar_len); + } else { + printf("LZ4: %zu -> %zu bytes (%.1f%%)\n", + tar_len, comp_sz, 100.0 * (double)comp_sz / (double)tar_len); - long end = ftell(out); - fclose(out); + FILE* out = fopen(out_name, "wb"); + if (!out) { perror("fopen"); free(tar_buf); free(comp_buf); return 1; } + + /* Write LZ4B header */ + uint8_t hdr[LZ4B_HDR_SIZE]; + memcpy(hdr, "LZ4B", 4); + write_le32(hdr + 4, (uint32_t)tar_len); + write_le32(hdr + 8, (uint32_t)comp_sz); + fwrite(hdr, 1, LZ4B_HDR_SIZE, out); + fwrite(comp_buf, 1, comp_sz, out); + fclose(out); + + printf("Done. InitRD size: %zu bytes (LZ4B header + compressed).\n", + LZ4B_HDR_SIZE + comp_sz); + } - printf("Done. InitRD size: %ld bytes.\n", end); + free(tar_buf); + free(comp_buf); return 0; }