From e1f0d044098f1996475bc426a3680e56c76e2997 Mon Sep 17 00:00:00 2001 From: kamkow1 Date: Sat, 28 Jun 2025 14:05:11 +0200 Subject: [PATCH] blog: Asset packing with zip.c --- .watcherignore | 3 + blog/blog-asset-packing-with-zip.c.md | 267 ++++++++++++++++++++++++++ build.c | 55 +++--- 3 files changed, 298 insertions(+), 27 deletions(-) create mode 100644 blog/blog-asset-packing-with-zip.c.md diff --git a/.watcherignore b/.watcherignore index 7558cc3..8b201fa 100644 --- a/.watcherignore +++ b/.watcherignore @@ -3,3 +3,6 @@ ./gpp1 ./watcher ./mongoose.o +./bundle.zip +./compile_flags.txt +./commit.h diff --git a/blog/blog-asset-packing-with-zip.c.md b/blog/blog-asset-packing-with-zip.c.md new file mode 100644 index 0000000..54f2766 --- /dev/null +++ b/blog/blog-asset-packing-with-zip.c.md @@ -0,0 +1,267 @@ +# Asset packing with [zip.c](https://raw.githubusercontent.com/kuba--/zip/refs/heads/master/src/zip.c) + +One of the technical decisions that I had to make when writing this website is the way that I'm going to +distribute the assets (images, GPP, page templates, etc.). I've decided to go with packing or baking-in +the assets into the main binary, so that I don't have to worry about updating an asset folder alongside +the application. + +One flaw of this approach is that as the website gets bigger, the binary will too. Fortunately I don't +have to worry about this too much right now, since I've reimplemented asset packing with the excellent +zip library by \`kuba--\` (Polska GUROM!!!111!!). In this article I'd like to demonstrate how I've +minimised the size of our assets using \`zip\` and \`incbin\`. + +## References + +- zip: https://raw.githubusercontent.com/kuba--/zip/refs/heads/master/src/zip.c +- incbin: https://github.com/graphitemaster/incbin +- zip format: https://en.wikipedia.org/wiki/ZIP_(file_format) +- the source code: https://git.kamkow1lair.pl/kamkow1/aboba + +## Generating the bundle - \`bundle.zip\` + +To compress our assets, first we need to generate a bundle file. I've decided to call it bundle.zip - simple +and descriptive. + +Generating the bundle requires some changes to our build program. See https://git.kamkow1lair.pl/kamkow1/aboba/src/branch/master/build.c for reference. + +\`\`\` + #define BUNDLED_FILES \\ + "./gpp1", \\ + "./tmpls/home.html", \\ + "./tmpls/page-missing.html", \\ + "./tmpls/template-blog.html", \\ + "./tmpls/blog.html", \\ + "./etc/hotreload.js", \\ + "./etc/theme.js", \\ + "./etc/simple.css", \\ + "./etc/highlight.js", \\ + "./etc/hljs-rainbow.css", \\ + "./etc/marked.js", \\ + "./etc/favicon.ico", \\ + "./etc/me.jpg", \\ + "./etc/tmoa-engine.jpg", \\ + "./etc/tmoa-garbage.jpg", \\ + "./blog/blog-welcome.md", \\ + "./blog/blog-weird-page.md", \\ + "./blog/blog-curious-case-of-gebs.md", \\ + "./blog/blog-the-making-of-aboba.md", \\ + "./blog/blog-asset-packing-with-zip.c.md" + + const char *bundle_zip_deps[] = { BUNDLED_FILES }; + + RULE_ARRAY("./bundle.zip", bundle_zip_deps) { + RULE("./gpp1", "./gpp/gpp.c") { + CMD("cc", "-DHAVE_STRDUP", "-DHAVE_FNMATCH_H", "-o", "gpp1", "gpp/gpp.c"); + } + + struct zip_t *zip = zip_open("./bundle.zip", BUNDLE_ZIP_COMPRESSION, 'w'); + defer { zip_close(zip); } + + for (size_t i = 0; i < sizeof(bundle_zip_deps)/sizeof(bundle_zip_deps[0]); i++) { + char *copy = strdup(bundle_zip_deps[i]); + defer { free(copy); } + char *name = basename(copy); + + String_Builder sb = {0}; + defer { sb_free(&sb); } + sb_read_file(&sb, bundle_zip_deps[i]); + + zip_entry_open(zip, name); + zip_entry_write(zip, sb.items, sb.count); + zip_entry_close(zip); + } + LOGI("Generated bundle.zip\\n"); + } + + RULE("./aboba", + "./main.c", + "./routes.c", + "./routes.h", + "./baked.c", + "./baked.h", + "./commit.h", + "./timer.c", + "./timer.h", + "./CONFIG.h", + "./locked.h", + + "./mongoose.o", + + "./bundle.zip", + BUNDLED_FILES + ) { + // build mongoose.o - skipped + + // Generate commit.h - skipped + + #define CC "cc" + #define TARGET "-o", "aboba" + #if MY_DEBUG + #define CFLAGS "-fsanitize=address", "-fPIC", "-ggdb" + #define DEFINES "-DMY_DEBUG=1", "-D_GNU_SOURCE", "-DGEBS_NO_PREFIX", "-DINCBIN_PREFIX=", "-DINCBIN_STYLE=INCBIN_STYLE_SNAKE", \\ + "-DGEBS_ENABLE_PTHREAD_FEATURES" + #define EXTRA_SOURCES "./cJSON/cJSON.c", "./zip/src/zip.c", "./md5-c/md5.c" + #else + #define CFLAGS "-fPIC" + #define DEFINES "-DMY_DEBUG=0", "-D_GNU_SOURCE", "-DGEBS_NO_PREFIX", "-DINCBIN_PREFIX=", "-DINCBIN_STYLE=INCBIN_STYLE_SNAKE", \\ + "-DGEBS_ENABLE_PTHREAD_FEATURES" + #define EXTRA_SOURCES "./cJSON/cJSON.c", "./zip/src/zip.c" + #endif + #define SOURCES "./main.c", "./routes.c", "./baked.c", "./timer.c" + #define OBJECTS "./mongoose.o" + #define LINK_FLAGS "-Wl,-z,execstack", "-lpthread" + #define INC_FLAGS "-I.", "-I./zip/src" + + CMD(CC, TARGET, CFLAGS, DEFINES, INC_FLAGS, SOURCES, OBJECTS, EXTRA_SOURCES, LINK_FLAGS); + + // generate compile_flags.txt - skipped + + // #undef macros - skipped + } +\`\`\` + +If you go through the commit history, you'll see that apart from just generating the bundle file, I've also cleaned up the +build commands a bit with \`#define\`s. Let's take a closer look at the bundle generation code. + +\`\`\` + const char *bundle_zip_deps[] = { BUNDLED_FILES }; + + RULE_ARRAY("./bundle.zip", bundle_zip_deps) { + RULE("./gpp1", "./gpp/gpp.c") { + CMD("cc", "-DHAVE_STRDUP", "-DHAVE_FNMATCH_H", "-o", "gpp1", "gpp/gpp.c"); + } + + struct zip_t *zip = zip_open("./bundle.zip", BUNDLE_ZIP_COMPRESSION, 'w'); + defer { zip_close(zip); } + + for (size_t i = 0; i < sizeof(bundle_zip_deps)/sizeof(bundle_zip_deps[0]); i++) { + char *copy = strdup(bundle_zip_deps[i]); + defer { free(copy); } + char *name = basename(copy); + + String_Builder sb = {0}; + defer { sb_free(&sb); } + sb_read_file(&sb, bundle_zip_deps[i]); + + zip_entry_open(zip, name); + zip_entry_write(zip, sb.items, sb.count); + zip_entry_close(zip); + } + LOGI("Generated bundle.zip\\n"); + } +\`\`\` + +We declare a dependency (using \`RULE\*()\` macro), which says that \`./bundle.zip\` depends on files defined by \`BUNDLED_FILES\`. +To generate the bundle we use \`zip_open()\` \`zip_close()\`. To call \`zip_open()\`, we have to provide a so called "compression level". +The zip library provides us only with \`ZIP_DEFAULT_COMPRESSION_LEVEL\`, which is a macro that evaluates to 6. I wasn't satisfied with +it, so after looking at \`miniz.h\` (a backing library that zip uses), I've found that zip uses \`MZ_DEFAULT_COMPRESSION_LEVEL\`, which +is 6, but we can use the value of \`MZ_UBER_COMPRESSION\`, which is 10. This way we can achieve the most size-efficient compression. + +I've decided to \`#define\` the compression level to avoid using arbitrary magic numbers and we can also use that definition both in +the application and in the build program. Here's how \`CONFIG.h\` looks like now: + +\`\`\` +#ifndef CONFIG_H_ +#define CONFIG_H_ + +#if MY_DEBUG +# define CONFIG_LISTEN_URL "http://localhost:8080" +#else +# define CONFIG_LISTEN_URL "http://localhost:5000" +#endif + +#define BUNDLE_ZIP_COMPRESSION 10 + +#endif // CONFIG_H_ +\`\`\` + +The only "downside" here is that, since we're compressing so hard, it's going to take more time to generate the bundle. I've put "downside" +in quotes for purpose, because this does not apply in our case. The files that we're packing are quite small already and there aren't +many of them. We're just doing this to sqeeze out *extra* spacial performance. + +Previously we were baking-in the assets like so: + +from https://git.kamkow1lair.pl/kamkow1/aboba/src/commit/447362c74dda85838d37d6ee3cb218697abf6106/baked.c + +\`\`\` +INCBIN(gpp1, "./gpp1"); + +INCBIN(home_html, "./tmpls/home.html"); +INCBIN(page_missing_html, "./tmpls/page-missing.html"); +INCBIN(template_blog_html, "./tmpls/template-blog.html"); +INCBIN(blog_html, "./tmpls/blog.html"); + +INCBIN(simple_css, "./etc/simple.css"); +INCBIN(favicon_ico, "./etc/favicon.ico"); +#if MY_DEBUG +INCBIN(hotreload_js, "./etc/hotreload.js"); +#endif +INCBIN(theme_js, "./etc/theme.js"); +INCBIN(highlight_js, "./etc/highlight.js"); +INCBIN(hljs_rainbow_css, "./etc/hljs-rainbow.css"); +INCBIN(marked_js, "./etc/marked.js"); +INCBIN(me_jpg, "./etc/me.jpg"); +INCBIN(tmoa_engine_jpg, "./etc/tmoa-engine.jpg"); +INCBIN(tmoa_garbage_jpg, "./etc/tmoa-garbage.jpg"); + +INCBIN(blog_welcome_md, "./blog/welcome.md"); +INCBIN(blog_weird_page_md, "./blog/weird-page.md"); +INCBIN(blog_curious_case_of_gebs_md, "./blog/curious-case-of-gebs.md"); +INCBIN(blog_the_making_of_aboba_md, "./blog/the-making-of-aboba.md"); +\`\`\` + +Now that we have our \`bundle.zip\`, we do it like this: + +\`\`\` +INCBIN(bundle_zip, "./bundle.zip"); +\`\`\` + +And there we go, we have our bundle! + +I've also had to slightly change the way we add the assets to the resource hash table: + +\`\`\` +void add_baked_resource(char *key, const uchar *data, size_t size) +{ + int fd = memfd_create(key, 0); + if (fd < 0) { + LOGE("Could not create resource %s. Aborting...\n", key); + abort(); + } + write(fd, data, size); + shput(baked_resources.value, key, ((Baked_Resource_Value){ .memfd = fd, .bufptr = (void *)data })); +} + +void init_baked_resources(void) +{ + struct zip_t *zip = zip_stream_open(bundle_zip_data, bundle_zip_size, BUNDLE_ZIP_COMPRESSION, 'r'); + size_t n = zip_entries_total(zip); + for (size_t i = 0; i < n; i++) { + zip_entry_openbyindex(zip, i); + + const char *name = strdup(zip_entry_name(zip)); + size_t size = zip_entry_size(zip); + char *buf = malloc(size); + zip_entry_noallocread(zip, buf, size); + + add_baked_resource((char *)name, buf, size); + + zip_entry_close(zip); + } + zip_stream_close(zip); +} +\`\`\` + +\`add_baked_resource()\` hasn't changed here, but \`init_baked_resources()\` has. Here we use one of zip's abilities, +which is unpacking an in-memory .zip file. We iterate each entry in the bundle, get the name and size, preallocate +a buffer and read said entry into the buffer. We can then add the resource to the hash table as we did previously. + +One question you may ask is, how much space are we saving? I don't remeber the exact sizes of the binary, but +I remember that before compression it was \`~1.2M\` and now after compression is implemented, it's \`~1.6M\`. How is +that an improvement if the binary gained weight? That \`~.4M\` is likely due to zip format overhead - metadata, file and +directory headers and so on. What we gain here is that the binary hasn't changed in size much despite adding more files, +like for eg. this article that I'm writing right now. It has stayed at \`~1.6M\` so far and the size doesn't go up. +*(So far)* We aren't even making byte-sized gains, which can be checked with \`stat --printf "%s" ./aboba\`. What +we're gaining here is slowed down size increase. Previously if I wanted to add a 30K jpeg, the binary would literally +go up by 30K. + diff --git a/build.c b/build.c index 01dd932..22127dc 100644 --- a/build.c +++ b/build.c @@ -37,7 +37,34 @@ int main(int argc, char ** argv) "./blog/blog-welcome.md", \ "./blog/blog-weird-page.md", \ "./blog/blog-curious-case-of-gebs.md", \ - "./blog/blog-the-making-of-aboba.md" + "./blog/blog-the-making-of-aboba.md", \ + "./blog/blog-asset-packing-with-zip.c.md" + + const char *bundle_zip_deps[] = { BUNDLED_FILES }; + + RULE_ARRAY("./bundle.zip", bundle_zip_deps) { + RULE("./gpp1", "./gpp/gpp.c") { + CMD("cc", "-DHAVE_STRDUP", "-DHAVE_FNMATCH_H", "-o", "gpp1", "gpp/gpp.c"); + } + + struct zip_t *zip = zip_open("./bundle.zip", BUNDLE_ZIP_COMPRESSION, 'w'); + defer { zip_close(zip); } + + for (size_t i = 0; i < sizeof(bundle_zip_deps)/sizeof(bundle_zip_deps[0]); i++) { + char *copy = strdup(bundle_zip_deps[i]); + defer { free(copy); } + char *name = basename(copy); + + String_Builder sb = {0}; + defer { sb_free(&sb); } + sb_read_file(&sb, bundle_zip_deps[i]); + + zip_entry_open(zip, name); + zip_entry_write(zip, sb.items, sb.count); + zip_entry_close(zip); + } + LOGI("Generated bundle.zip\n"); + } RULE("./aboba", "./main.c", @@ -95,32 +122,6 @@ int main(int argc, char ** argv) } } - const char *bundle_zip_deps[] = { BUNDLED_FILES }; - - RULE_ARRAY("./bundle.zip", bundle_zip_deps) { - RULE("./gpp1", "./gpp/gpp.c") { - CMD("cc", "-DHAVE_STRDUP", "-DHAVE_FNMATCH_H", "-o", "gpp1", "gpp/gpp.c"); - } - - struct zip_t *zip = zip_open("./bundle.zip", BUNDLE_ZIP_COMPRESSION, 'w'); - defer { zip_close(zip); } - - for (size_t i = 0; i < sizeof(bundle_zip_deps)/sizeof(bundle_zip_deps[0]); i++) { - char *copy = strdup(bundle_zip_deps[i]); - defer { free(copy); } - char *name = basename(copy); - - String_Builder sb = {0}; - defer { sb_free(&sb); } - sb_read_file(&sb, bundle_zip_deps[i]); - - zip_entry_open(zip, name); - zip_entry_write(zip, sb.items, sb.count); - zip_entry_close(zip); - } - LOGI("Generated bundle.zip\n"); - } - #define CC "cc" #define TARGET "-o", "aboba" #if MY_DEBUG