Compress base.img using FastLZ library
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@ -3,3 +3,4 @@
|
||||
*.iso
|
||||
*.img
|
||||
*.hdd
|
||||
*.6pack
|
||||
|
||||
19
FastLZ/.editorconfig
Normal file
19
FastLZ/.editorconfig
Normal file
@ -0,0 +1,19 @@
|
||||
# top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
# Unix-style newlines with a newline ending every file
|
||||
[*]
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
[*.{c,h}]
|
||||
indent_size = 2
|
||||
indent_style = space
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[Makefile.win]
|
||||
indent_style = tab
|
||||
32
FastLZ/.github/workflows/amd64_linux_clang.yml
vendored
Normal file
32
FastLZ/.github/workflows/amd64_linux_clang.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: amd64_linux_clang
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_linux_clang:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: clang
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt install -y make clang
|
||||
- run: clang --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
32
FastLZ/.github/workflows/amd64_linux_gcc.yml
vendored
Normal file
32
FastLZ/.github/workflows/amd64_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: amd64_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt install -y make gcc
|
||||
- run: gcc --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
32
FastLZ/.github/workflows/amd64_linux_tcc.yml
vendored
Normal file
32
FastLZ/.github/workflows/amd64_linux_tcc.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: amd64_linux_tcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_linux_tcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: tcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt install -y make tcc
|
||||
- run: tcc -v
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
29
FastLZ/.github/workflows/amd64_macos_clang.yml
vendored
Normal file
29
FastLZ/.github/workflows/amd64_macos_clang.yml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
name: amd64_macos_clang
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_macos_clang:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: cc --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
32
FastLZ/.github/workflows/amd64_macos_gcc.yml
vendored
Normal file
32
FastLZ/.github/workflows/amd64_macos_gcc.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: amd64_macos_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_macos_gcc:
|
||||
runs-on: macos-12
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: gcc-9
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: brew install gcc@9
|
||||
- run: gcc-9 --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
34
FastLZ/.github/workflows/amd64_windows_clang.yml
vendored
Normal file
34
FastLZ/.github/workflows/amd64_windows_clang.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: amd64_windows_clang
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_windows_clang:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: clang
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
install: make mingw-w64-x86_64-clang
|
||||
- run: clang --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
34
FastLZ/.github/workflows/amd64_windows_gcc.yml
vendored
Normal file
34
FastLZ/.github/workflows/amd64_windows_gcc.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: amd64_windows_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_windows_gcc:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
install: gcc make
|
||||
- run: gcc --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
35
FastLZ/.github/workflows/amd64_windows_tcc.yml
vendored
Normal file
35
FastLZ/.github/workflows/amd64_windows_tcc.yml
vendored
Normal file
@ -0,0 +1,35 @@
|
||||
name: amd64_windows_tcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_windows_tcc:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- name: Install tcc
|
||||
run: |
|
||||
echo "5a3979bd5044b795547a4948a5625a12 tcc.zip" > checksum.md5
|
||||
dos2unix checksum.md5
|
||||
curl -L -o tcc.zip https://archive.org/download/tinyccompiler/tcc-0.9.27-win32-bin.zip
|
||||
md5sum -c checksum.md5 && unzip -q tcc.zip
|
||||
- run: tcc\x86_64-win32-tcc.exe -v
|
||||
- run: cd tests && make roundtrip CC=..\tcc\x86_64-win32-tcc.exe
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make CC=..\tcc\x86_64-win32-tcc.exe
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
30
FastLZ/.github/workflows/amd64_windows_vs2019.yml
vendored
Normal file
30
FastLZ/.github/workflows/amd64_windows_vs2019.yml
vendored
Normal file
@ -0,0 +1,30 @@
|
||||
name: amd64_windows_vs2019
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
amd64_windows_vs2019:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- uses: ilammy/msvc-dev-cmd@v1
|
||||
- run: cl
|
||||
- run: cd tests && mingw32-make -f Makefile.win roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make -f Makefile.win
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/arm64_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/arm64_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: arm64_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
arm64_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/aarch64-linux-musl-cross.tgz
|
||||
tar xzf aarch64-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-aarch64 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-aarch64 ./6pack -v
|
||||
qemu-aarch64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-aarch64 ./6unpack -v
|
||||
qemu-aarch64 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/armhf_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/armhf_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: armhf_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
armhf_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/armel-linux-musleabihf-cross/bin/armel-linux-musleabihf-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O https://musl.cc/armel-linux-musleabihf-cross.tgz
|
||||
tar xzf armel-linux-musleabihf-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/armel-linux-musleabihf-cross/bin/armel-linux-musleabihf-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-arm ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-arm ./6pack -v
|
||||
qemu-arm ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-arm ./6unpack -v
|
||||
qemu-arm ./6unpack archive.6pk
|
||||
29
FastLZ/.github/workflows/asan.yml
vendored
Normal file
29
FastLZ/.github/workflows/asan.yml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
name: Address Sanitizer
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
asan:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt install -y make gcc
|
||||
- run: gcc --version
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests
|
||||
env:
|
||||
CFLAGS: "-g -fno-omit-frame-pointer -fsanitize=address"
|
||||
- run: cd tests && make roundtrip
|
||||
name: Perform round-trip tests with FASTLZ_USE_MEMMOVE=0
|
||||
env:
|
||||
CFLAGS: "-g -fno-omit-frame-pointer -fsanitize=address -DFASTLZ_USE_MEMMOVE=0"
|
||||
19
FastLZ/.github/workflows/codestyle.yml
vendored
Normal file
19
FastLZ/.github/workflows/codestyle.yml
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
name: Code style
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
codestyle:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- run: sudo apt install -y clang-format-6.0
|
||||
name: Install clang-format
|
||||
- run: clang-format-6.0 --version
|
||||
- run: bash tools/format-code.sh
|
||||
name: Run code formatter
|
||||
- run: git diff
|
||||
- run: git diff --quiet HEAD
|
||||
name: Check if the styling guide is followed
|
||||
52
FastLZ/.github/workflows/i586_dos_gcc_cross.yml
vendored
Normal file
52
FastLZ/.github/workflows/i586_dos_gcc_cross.yml
vendored
Normal file
@ -0,0 +1,52 @@
|
||||
name: i586_dos_gcc_cross
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
i586_dos_gcc_cross:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
CC: /opt/djgpp/bin/i586-pc-msdosdjgpp-gcc
|
||||
LDFLAGS: -static
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -OL https://github.com/andrewwutw/build-djgpp/releases/download/v3.3/djgpp-linux64-gcc1210.tar.bz2
|
||||
tar xf djgpp-linux64-gcc1210.tar.bz2 -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/djgpp/bin/i586-pc-msdosdjgpp-gcc --version
|
||||
- name: Install DOSEMU2
|
||||
run: |
|
||||
sudo add-apt-repository -y ppa:dosemu2/ppa
|
||||
sudo apt update -y
|
||||
sudo apt install -y dosemu2
|
||||
- run: dosemu --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
ln -s ../compression-corpus/ corpus
|
||||
make test_roundtrip TEST_ROUNDTRIP=testrr
|
||||
file ./testrr.exe
|
||||
dosemu -dumb -K . -t -E "testrr corpus/"
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
ln -s ../compression-corpus/enwik/enwik8.txt enwik8.txt
|
||||
make
|
||||
file ./6pack.exe
|
||||
dosemu -K . -t -E "6pack.exe -v"
|
||||
dosemu -K . -t -E "6pack.exe enwik8.txt archive.6pk"
|
||||
mv enwik8.txt enwik8.txt.orig
|
||||
dosemu -K . -t -E "6unpack.exe -v"
|
||||
dosemu -K . -t -E "6unpack.exe archive.6pk"
|
||||
ls -l enwik8*
|
||||
36
FastLZ/.github/workflows/i686_linux_clang.yml
vendored
Normal file
36
FastLZ/.github/workflows/i686_linux_clang.yml
vendored
Normal file
@ -0,0 +1,36 @@
|
||||
name: i686_linux_clang
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
i686_linux_clang:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: clang
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y make clang gcc-multilib qemu-user
|
||||
- run: clang --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS="-static -m32"
|
||||
file ./test_roundtrip
|
||||
qemu-i386 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS="-static -m32"
|
||||
qemu-i386 ./6pack -v
|
||||
qemu-i386 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-i386 ./6unpack -v
|
||||
qemu-i386 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/i686_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/i686_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: i686_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
i686_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/i686-linux-musl-cross/bin/i686-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O https://musl.cc/i686-linux-musl-cross.tgz
|
||||
tar xzf i686-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/i686-linux-musl-cross/bin/i686-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-i386 ./test_roundtrip
|
||||
- name: 'Build and run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-i386 ./6pack -v
|
||||
qemu-i386 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-i386 ./6unpack -v
|
||||
qemu-i386 ./6unpack archive.6pk
|
||||
35
FastLZ/.github/workflows/i686_windows_tcc.yml
vendored
Normal file
35
FastLZ/.github/workflows/i686_windows_tcc.yml
vendored
Normal file
@ -0,0 +1,35 @@
|
||||
name: i686_windows_tcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
i686_windows_tcc:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- name: Install tcc
|
||||
run: |
|
||||
echo "D73CF66CEC8C761DE38C7A3D16C9EB0D tcc.zip" > checksum.md5
|
||||
dos2unix checksum.md5
|
||||
curl -L -o tcc.zip https://archive.org/download/tinyccompiler/tcc-0.9.27-win64-bin.zip
|
||||
md5sum -c checksum.md5 && unzip -q tcc.zip
|
||||
- run: tcc\i386-win32-tcc.exe -v
|
||||
- run: cd tests && make roundtrip CC=..\tcc\i386-win32-tcc.exe
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make CC=..\tcc\i386-win32-tcc.exe
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
32
FastLZ/.github/workflows/i686_windows_vs2019.yml
vendored
Normal file
32
FastLZ/.github/workflows/i686_windows_vs2019.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: i686_windows_vs2019
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
i686_windows_vs2019:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- uses: ilammy/msvc-dev-cmd@v1
|
||||
with:
|
||||
arch: x86
|
||||
- run: cl
|
||||
- run: cd tests && mingw32-make -f Makefile.win roundtrip
|
||||
name: Perform round-trip tests
|
||||
- name: 'Build examples: 6pack and 6unpack'
|
||||
run: cd examples && make -f Makefile.win
|
||||
- name: 'Run examples: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
./6pack -v
|
||||
./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
./6unpack -v
|
||||
./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/mips64_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/mips64_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: mips64_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
mips64_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/mips64-linux-musl-cross/bin/mips64-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/mips64-linux-musl-cross.tgz
|
||||
tar xzf mips64-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/mips64-linux-musl-cross/bin/mips64-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-mips64 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-mips64 ./6pack -v
|
||||
qemu-mips64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-mips64 ./6unpack -v
|
||||
qemu-mips64 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/mips64el_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/mips64el_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: mips64el_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
mips64el_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/mips64el-linux-musl-cross/bin/mips64el-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/mips64el-linux-musl-cross.tgz
|
||||
tar xzf mips64el-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/mips64el-linux-musl-cross/bin/mips64el-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-mips64el ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-mips64el ./6pack -v
|
||||
qemu-mips64el ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-mips64el ./6unpack -v
|
||||
qemu-mips64el ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/mips_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/mips_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: mips_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
mips_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/mips-linux-musl-cross/bin/mips-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/mips-linux-musl-cross.tgz
|
||||
tar xzf mips-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/mips-linux-musl-cross/bin/mips-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-mips ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-mips ./6pack -v
|
||||
qemu-mips ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-mips ./6unpack -v
|
||||
qemu-mips ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/mipsel_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/mipsel_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: mipsel_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
mipsel_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/mipsel-linux-musl-cross/bin/mipsel-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/mipsel-linux-musl-cross.tgz
|
||||
tar xzf mipsel-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/mipsel-linux-musl-cross/bin/mipsel-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-mipsel ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-mipsel ./6pack -v
|
||||
qemu-mipsel ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-mipsel ./6unpack -v
|
||||
qemu-mipsel ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/powerpc_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/powerpc_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: powerpc_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
powerpc_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/powerpc-linux-musl-cross/bin/powerpc-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/powerpc-linux-musl-cross.tgz
|
||||
tar xzf powerpc-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/powerpc-linux-musl-cross/bin/powerpc-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-ppc ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-ppc ./6pack -v
|
||||
qemu-ppc ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-ppc ./6unpack -v
|
||||
qemu-ppc ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/ppc64_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/ppc64_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: ppc64_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
ppc64_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/powerpc64-linux-musl-cross/bin/powerpc64-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/powerpc64-linux-musl-cross.tgz
|
||||
tar xzf powerpc64-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/powerpc64-linux-musl-cross/bin/powerpc64-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-ppc64 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-ppc64 ./6pack -v
|
||||
qemu-ppc64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-ppc64 ./6unpack -v
|
||||
qemu-ppc64 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/ppc64le_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/ppc64le_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: ppc64le_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
ppc64le_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/powerpc64le-linux-musl-cross/bin/powerpc64le-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/powerpc64le-linux-musl-cross.tgz
|
||||
tar xzf powerpc64le-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/powerpc64le-linux-musl-cross/bin/powerpc64le-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-ppc64le ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-ppc64le ./6pack -v
|
||||
qemu-ppc64le ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-ppc64le ./6unpack -v
|
||||
qemu-ppc64le ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/riscv64_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/riscv64_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: riscv64_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
riscv64_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/riscv64-linux-musl-cross/bin/riscv64-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/riscv64-linux-musl-cross.tgz
|
||||
tar xzf riscv64-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/riscv64-linux-musl-cross/bin/riscv64-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-riscv64 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-riscv64 ./6pack -v
|
||||
qemu-riscv64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-riscv64 ./6unpack -v
|
||||
qemu-riscv64 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/riscv_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/riscv_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: riscv_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
riscv_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/riscv32-linux-musl-cross/bin/riscv32-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/riscv32-linux-musl-cross.tgz
|
||||
tar xzf riscv32-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/riscv32-linux-musl-cross/bin/riscv32-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-riscv32 ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-riscv32 ./6pack -v
|
||||
qemu-riscv32 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-riscv32 ./6unpack -v
|
||||
qemu-riscv32 ./6unpack archive.6pk
|
||||
41
FastLZ/.github/workflows/s390x_linux_gcc.yml
vendored
Normal file
41
FastLZ/.github/workflows/s390x_linux_gcc.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: s390x_linux_gcc
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
|
||||
s390x_linux_gcc:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CC: /opt/s390x-linux-musl-cross/bin/s390x-linux-musl-gcc
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Retrieve test compression corpus
|
||||
run: |
|
||||
git clone https://github.com/ariya/compression-corpus.git
|
||||
cd compression-corpus
|
||||
cd enwik
|
||||
unzip enwik8.zip
|
||||
- run: sudo apt-get -y -qq update
|
||||
- run: sudo apt install -y qemu-user
|
||||
- name: Prepare cross-compiler
|
||||
run: |
|
||||
curl -O http://musl.cc/s390x-linux-musl-cross.tgz
|
||||
tar xzf s390x-linux-musl-cross.tgz -C /opt
|
||||
- name: Verify compiler version
|
||||
run: /opt/s390x-linux-musl-cross/bin/s390x-linux-musl-gcc --version
|
||||
- name: Perform round-trip tests
|
||||
run: |
|
||||
cd tests
|
||||
make test_roundtrip CFLAGS=-static
|
||||
file ./test_roundtrip
|
||||
qemu-s390x ./test_roundtrip
|
||||
- name: 'Build and run example: 6pack and 6unpack'
|
||||
run: |
|
||||
cd examples
|
||||
make CFLAGS=-static
|
||||
qemu-s390x ./6pack -v
|
||||
qemu-s390x ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk
|
||||
qemu-s390x ./6unpack -v
|
||||
qemu-s390x ./6unpack archive.6pk
|
||||
1
FastLZ/.gitignore
vendored
Normal file
1
FastLZ/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.o
|
||||
3
FastLZ/.gitmodules
vendored
Normal file
3
FastLZ/.gitmodules
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
[submodule "compression-corpus"]
|
||||
path = compression-corpus
|
||||
url = https://github.com/ariya/compression-corpus.git
|
||||
8
FastLZ/ChangeLog
Normal file
8
FastLZ/ChangeLog
Normal file
@ -0,0 +1,8 @@
|
||||
2020-02-02: Version 0.5.0
|
||||
|
||||
Minor speed improvement on the decompressor.
|
||||
Prevent memory violation when decompressing corrupted input.
|
||||
|
||||
2020-01-10: Version 0.4.0
|
||||
|
||||
Only code & infrastructure clean-up, no new functionality.
|
||||
21
FastLZ/LICENSE.MIT
Normal file
21
FastLZ/LICENSE.MIT
Normal file
@ -0,0 +1,21 @@
|
||||
FastLZ - Byte-aligned LZ77 compression library
|
||||
Copyright (C) 2005-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
207
FastLZ/README.md
Normal file
207
FastLZ/README.md
Normal file
@ -0,0 +1,207 @@
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ariya/fastlz/actions)
|
||||
[](https://github.com/ariya/fastlz/actions)
|
||||
|
||||
## Overview
|
||||
|
||||
FastLZ (MIT license) is an ANSI C/C90 implementation of [Lempel-Ziv 77 algorithm](https://en.wikipedia.org/wiki/LZ77_and_LZ78#LZ77) (LZ77) of lossless data compression. It is suitable to compress series of text/paragraphs, sequences of raw pixel data, or any other blocks of data with lots of repetition. It is not intended to be used on images, videos, and other formats of data typically already in an optimal compressed form.
|
||||
|
||||
The focus for FastLZ is a very fast compression and decompression, doing that at the cost of the compression ratio. As an illustration, the comparison with zlib when compressing [enwik8](http://www.mattmahoney.net/dc/textdata.html) (also in [more details](https://github.com/inikep/lzbench)):
|
||||
|
||||
||Ratio|Compression|Decompression
|
||||
|--|--|--|--|
|
||||
|FastLZ |54.2%|159 MB/s|305 MB/s|
|
||||
|zlib -1|42.3%|50 MB/s|184 MB/s|
|
||||
|zlib -9|36.5%|11 MB/s|185 MB/s|
|
||||
|
||||
FastLZ is used by many software products, from a number of games (such as [Death Stranding](https://en.wikipedia.org/wiki/Death_Stranding)) to various open-source projects ([Godot Engine](https://godotengine.org/), [Facebook HHVM](https://hhvm.com/), [Apache Traffic Server](https://trafficserver.apache.org/), [Calligra Office](https://www.calligra.org/), [OSv](http://osv.io/), [Netty](https://netty.io/), etc). It even serves as the basis for other compression projects like [BLOSC](https://blosc.org/).
|
||||
|
||||
For other implementations of byte-aligned LZ77, take a look at [LZ4](https://lz4.github.io/lz4/), [Snappy](http://google.github.io/snappy/), [Density](https://github.com/centaurean/density), [LZO](http://www.oberhumer.com/opensource/lzo/), [LZF](http://oldhome.schmorp.de/marc/liblzf.html), [LZJB](https://en.wikipedia.org/wiki/LZJB), [LZRW](http://www.ross.net/compression/lzrw1.html), etc.
|
||||
|
||||
## Usage
|
||||
|
||||
FastLZ can be used directly in any C/C++ applications. For other programming languages/environments, use the corresponding binding:
|
||||
|
||||
* [Rust](https://crates.io/crates/fastlz), available on Crates: `cargo install fastlz`
|
||||
* [Python](https://pypi.org/project/fastlz/), available on PyPi: `pip install fastlz`
|
||||
* [JavaScript](https://www.npmjs.com/package/fastlz), available on npm: `npm install fastlz`
|
||||
* [Ruby](https://rubygems.org/gems/fastlz), available on Rubygems: `gem install fastlz`
|
||||
* Lua via [github.com/oneoo/lua-fastlz](https://github.com/oneoo/lua-fastlz)
|
||||
|
||||
FastLZ consists of only two files: `fastlz.h` and `fastlz.c`. Just add these files to your project in order to use FastLZ. For the detailed information on the API to perform compression and decompression, see `fastlz.h`.
|
||||
|
||||
For [Vcpkg](https://github.com/microsoft/vcpkg) users, FastLZ is [already available](https://github.com/microsoft/vcpkg): `vcpkg install fastlz`.
|
||||
|
||||
A simple file compressor called `6pack` is included as an example on how to use FastLZ. The corresponding decompressor is `6unpack`.
|
||||
|
||||
FastLZ supports any standard-conforming ANSI C/C90 compiler, including the popular ones such as [GCC](https://gcc.gnu.org/), [Clang](https://clang.llvm.org/), [Visual Studio](https://visualstudio.microsoft.com/vs/features/cplusplus/), and even [Tiny CC](https://bellard.org/tcc/). FastLZ works well on a number of architectures (32-bit and 64-bit, big endian and little endian), from Intel/AMD, PowerPC, System z, ARM, MIPS, and RISC-V.
|
||||
|
||||
The continuous integration system runs an extensive set of compression-decompression round trips on the following systems:
|
||||
|
||||
|
||||
For more details, check the corresponding [GitHub Actions build logs](https://github.com/ariya/FastLZ/actions).
|
||||
|
||||
| | | | |
|
||||
|----------------------|--------------------------------------------------------------------------------------------------------:|--------------------------------------------------------------------------------------------------:|--------------------------------------------------------------------------------------------:|
|
||||
| **amd64** | **Linux** | **Windows** | **macOS** |
|
||||
| GCC |  |  |  |
|
||||
| Clang |  |  |  |
|
||||
| TinyCC |  |  | |
|
||||
| VS 2019 | |  | |
|
||||
| **i686** | **Linux** | **Windows** | **macOS** |
|
||||
| GCC |  | | |
|
||||
| Clang |  | | |
|
||||
| TinyCC | |  | |
|
||||
| VS 2019 | |  | |
|
||||
| **i586** | **Linux** | **DOS** | |
|
||||
| GCC | |  | |
|
||||
| | **Linux** | | |
|
||||
| **powerpc** | | | |
|
||||
| GCC |  | | |
|
||||
| **ppc64(le)** | | | |
|
||||
| GCC |  | | |
|
||||
| GCC |  | | |
|
||||
| **s390x** | | | |
|
||||
| GCC |  | | |
|
||||
| **armhf** | | | |
|
||||
| GCC |  | | |
|
||||
| **arm64** | | | |
|
||||
| GCC |  | | |
|
||||
| **mips(el)** | | | |
|
||||
| GCC |  | | |
|
||||
| GCC |  | | |
|
||||
| **mips64(el)** | | | |
|
||||
| GCC |  | | |
|
||||
| GCC |  | | |
|
||||
| **riscv** | | | |
|
||||
| GCC |  | | |
|
||||
| **riscv64** | | | |
|
||||
| GCC |  | | |
|
||||
|
||||
|
||||
|
||||
## Block Format
|
||||
|
||||
Let us assume that FastLZ compresses an array of bytes, called the _uncompressed block_, into another array of bytes, called the _compressed block_. To understand what will be stored in the compressed block, it is illustrative to demonstrate how FastLZ will _decompress_ the block to retrieve the original uncompressed block.
|
||||
|
||||
The first 3-bit of the block, i.e. the 3 most-significant bits of the first byte, is the **block tag**. Currently the block tag determines the compression level used to produce the compressed block.
|
||||
|
||||
|Block tag|Compression level|
|
||||
|---------|-----------------|
|
||||
| 0 | Level 1 |
|
||||
| 1 | Level 2 |
|
||||
|
||||
The content of the block will vary depending on the compression level.
|
||||
|
||||
### Block Format for Level 1
|
||||
|
||||
FastLZ Level 1 implements LZ77 compression algorithm with 8 KB sliding window and up to 264 bytes of match length.
|
||||
|
||||
The compressed block consists of one or more **instructions**.
|
||||
Each instruction starts with a 1-byte opcode, 2-byte opcode, or 3-byte opcode.
|
||||
|
||||
| Instruction type | Opcode[0] | Opcode[1] | Opcode[2]
|
||||
|-----------|------------------|--------------------|--|
|
||||
| Literal run | `000`, L₄-L₀ | -|- |
|
||||
| Short match | M₂-M₀, R₁₂-R₈ | R₇-R₀ | - |
|
||||
| Long match | `111`, R₁₂-R₈ | M₇-M₀ | R₇-R₀ |
|
||||
|
||||
Note that the _very first_ instruction in a compressed block is always a literal run.
|
||||
|
||||
#### Literal run instruction
|
||||
|
||||
For the literal run instruction, there is one or more bytes following the code. This is called the literal run.
|
||||
|
||||
The 5 least-significant bits of `opcode[0]`, _L_, determines the **number of literals** following the opcode. The value of 0 indicates a 1-byte literal run, 1 indicates a 2-byte literal run, and so on. The minimum literal run is 1 and the maximum literal run is 32.
|
||||
|
||||
The decompressor copies (_L + 1_) bytes of literal run, starting from the first one right after opcode.
|
||||
|
||||
_Example_: If the compressed block is a 4-byte array of `[0x02, 0x41, 0x42, 0x43]`, then the opcode is `0x02` and that means a literal run of 3 bytes. The decompressor will then copy the subsequent 3 bytes, `[0x41, 0x42, 0x43]`, to the output buffer. The output buffer now represents the (original) uncompressed block, `[0x41, 0x42, 0x43]`.
|
||||
|
||||
#### Short match instruction
|
||||
|
||||
The 3 most-significant bits of `opcode[0]`, _M_, determines the **match length**. The value of 1 indicates a 3-byte match, 2 indicates a 4-byte match and so on. The minimum match length is 3 and the maximum match length is 8.
|
||||
|
||||
The 5 least-significant bits of `opcode[0]` combined with the 8 bits of the `opcode[1]`, _R_, determines the **reference offset**. Since the offset is encoded in 13 bits, the minimum is 0 and the maximum is 8191.
|
||||
|
||||
The following C code retrieves the match length and reference offset:
|
||||
|
||||
```c
|
||||
M = opcode[0] >> 5;
|
||||
R = 256 * (opcode[0] << 5) + opcode[1];
|
||||
```
|
||||
|
||||
The decompressor copies _(M+2)_ bytes, starting from the location offsetted by _R_ in the output buffer. Note that _R_ is a *back reference*, i.e. the value of 0 corresponds the last byte in the output buffer, 1 is the second to last byte, and so forth.
|
||||
|
||||
_Example 1_: If the compressed block is a 7-byte array of `[0x03, 0x41, 0x42, 0x43, 0x44, 0x20, 0x02]`, then there are two instructions in the there. The first instruction is the literal run of 4 bytes (due to _L = 3_). Thus, the decompressor copies 4 bytes to the output buffer, resulting in `[0x41, 0x42, 0x43, 0x44]`. The second instruction is the short match of 3 bytes (from _M = 1_, i.e `0x20 >> 5`) and the offset of 2. Therefore, the compressor goes back 2 bytes from the last position, copies 3 bytes (`[0x42, 0x43, 0x44]`), and appends them to the output buffer. The output buffer now represents the complete uncompressed data, `[0x41, 0x42, 0x43, 0x44, 0x42, 0x43, 0x44]`.
|
||||
|
||||
_Example 2_: If the compressed block is a 4-byte array of `[0x00, 0x61, 0x40, 0x00]`, then there are two instructions in there. The first instruction is the literal run of just 1 byte (_L = 0_). Thus, the decompressor copies the byte (`0x61`) to the output buffer. The output buffer now becomes `[0x61]`. The second instruction is the short match of 4 bytes (from _M = 2_, i.e. `0x40 >> 5`) and the offset of 0. Therefore, the decompressor copies 4 bytes starting using the back reference of 0 (i.e. the position of `0x61`). The output buffer now represents the complete uncompressed data, `[0x61, 0x61, 0x61, 0x61, 0x61]`.
|
||||
|
||||
#### Long match instruction
|
||||
|
||||
The value of `opcode[1]`, _M_, determines the **match length**. The value of 0 indicates a 9-byte match, 1 indicates a 10-byte match and so on. The minimum match length is 9 and the maximum match length is 264.
|
||||
|
||||
The 5 least-significant bits of `opcode[0]` combined with the 8 bits of `opcode[2]`, _R_, determines the **reference offset**. Since the offset is encoded in 13 bits, the minimum is 0 and the maximum is 8191.
|
||||
|
||||
The following C code retrieves the match length and reference offset:
|
||||
|
||||
```c
|
||||
M = opcode[1];
|
||||
R = 256 * (opcode[0] << 5) + opcode[2];
|
||||
```
|
||||
The decompressor copies _(M+9)_ bytes, starting from the location offsetted by _R_ in the output buffer. Note that _R_ is a *back reference*, i.e. the value of 0 corresponds to the last byte in the output buffer, 1 is for the second to last byte, and so forth.
|
||||
|
||||
_Example_: If the compressed block is a 4-byte array of `[0x01, 0x44, 0x45, 0xE0, 0x01, 0x01]`, then there are two instructions in there. The first instruction is the literal run with the length of 2 (due to _L = 1_). Thus, the decompressor copies the 2-byte literal run (`[0x44, 0x45]`) to the output buffer. The second instruction is the long match with the match length of 10 (from _M = 1_) and the offset of 1. Therefore, the decompressor copies 10 bytes starting using the back reference of 1 (i.e. the position of `0x44`). The output buffer now represents the complete uncompressed data, `[0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45]`.
|
||||
|
||||
#### Decompressor Reference Implementation
|
||||
|
||||
The following 40-line C function implements a fully-functional decompressor for the above block format. Note that it is intended to be educational, e.g. no bound check is implemented, and therefore it is absolutely **unsafe** for production.
|
||||
|
||||
```c
|
||||
void fastlz_level1_decompress(const uint8_t* input, int length, uint8_t* output) {
|
||||
int src = 0;
|
||||
int dest = 0;
|
||||
while (src < length) {
|
||||
int type = input[src] >> 5;
|
||||
if (type == 0) {
|
||||
/* literal run */
|
||||
int run = 1 + input[src];
|
||||
src = src + 1;
|
||||
while (run > 0) {
|
||||
output[dest] = input[src];
|
||||
src = src + 1;
|
||||
dest = dest + 1;
|
||||
run = run - 1;
|
||||
}
|
||||
} else if (type < 7) {
|
||||
/* short match */
|
||||
int ofs = 256 * (input[src] & 31) + input[src + 1];
|
||||
int len = 2 + (input[src] >> 5);
|
||||
src = src + 2;
|
||||
int ref = dest - ofs - 1;
|
||||
while (len > 0) {
|
||||
output[dest] = output[ref];
|
||||
ref = ref + 1;
|
||||
dest = dest + 1;
|
||||
len = len - 1;
|
||||
}
|
||||
} else {
|
||||
/* long match */
|
||||
int ofs = 256 * (input[src] & 31) + input[src + 2];
|
||||
int len = 9 + input[src + 1];
|
||||
src = src + 3;
|
||||
int ref = dest - ofs - 1;
|
||||
while (len > 0) {
|
||||
output[dest] = output[ref];
|
||||
ref = ref + 1;
|
||||
dest = dest + 1;
|
||||
len = len - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Block Format for Level 2
|
||||
|
||||
(To be written)
|
||||
1
FastLZ/_config.yml
Normal file
1
FastLZ/_config.yml
Normal file
@ -0,0 +1 @@
|
||||
theme: jekyll-theme-tactile
|
||||
2
FastLZ/examples/.gitignore
vendored
Normal file
2
FastLZ/examples/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
6pack
|
||||
6unpack
|
||||
586
FastLZ/examples/6pack.c
Normal file
586
FastLZ/examples/6pack.c
Normal file
@ -0,0 +1,586 @@
|
||||
/*
|
||||
6PACK - file compressor using FastLZ (lightning-fast compression library)
|
||||
Copyright (C) 2007-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define SIXPACK_VERSION_MAJOR 0
|
||||
#define SIXPACK_VERSION_MINOR 1
|
||||
#define SIXPACK_VERSION_REVISION 0
|
||||
#define SIXPACK_VERSION_STRING "snapshot 20070615"
|
||||
|
||||
#include "fastlz.h"
|
||||
|
||||
#undef PATH_SEPARATOR
|
||||
|
||||
#if defined(MSDOS) || defined(__MSDOS__) || defined(MSDOS)
|
||||
#define PATH_SEPARATOR '\\'
|
||||
#endif
|
||||
|
||||
#if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__)
|
||||
#define PATH_SEPARATOR '\\'
|
||||
#endif
|
||||
|
||||
#ifndef PATH_SEPARATOR
|
||||
#define PATH_SEPARATOR '/'
|
||||
#endif
|
||||
|
||||
#undef SIXPACK_BENCHMARK_WIN32
|
||||
#if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__)
|
||||
#if defined(_MSC_VER) || defined(__GNUC__)
|
||||
#define SIXPACK_BENCHMARK_WIN32
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* magic identifier for 6pack file */
|
||||
static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10};
|
||||
|
||||
#define BLOCK_SIZE (2 * 64 * 1024)
|
||||
|
||||
/* prototypes */
|
||||
static unsigned long update_adler32(unsigned long checksum, const void* buf, int len);
|
||||
void usage(void);
|
||||
int detect_magic(FILE* f);
|
||||
void write_magic(FILE* f);
|
||||
void write_chunk_header(FILE* f, int id, int options, unsigned long size, unsigned long checksum, unsigned long extra);
|
||||
unsigned long block_compress(const unsigned char* input, unsigned long length, unsigned char* output);
|
||||
int pack_file_compressed(const char* input_file, int method, int level, FILE* f);
|
||||
int pack_file(int compress_level, const char* input_file, const char* output_file);
|
||||
|
||||
/* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */
|
||||
#define ADLER32_BASE 65521
|
||||
static unsigned long update_adler32(unsigned long checksum, const void* buf, int len) {
|
||||
const unsigned char* ptr = (const unsigned char*)buf;
|
||||
unsigned long s1 = checksum & 0xffff;
|
||||
unsigned long s2 = (checksum >> 16) & 0xffff;
|
||||
|
||||
while (len > 0) {
|
||||
unsigned k = len < 5552 ? len : 5552;
|
||||
len -= k;
|
||||
|
||||
while (k >= 8) {
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
k -= 8;
|
||||
}
|
||||
|
||||
while (k-- > 0) {
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
}
|
||||
s1 = s1 % ADLER32_BASE;
|
||||
s2 = s2 % ADLER32_BASE;
|
||||
}
|
||||
return (s2 << 16) + s1;
|
||||
}
|
||||
|
||||
void usage(void) {
|
||||
printf("6pack: high-speed file compression tool\n");
|
||||
printf("Copyright (C) Ariya Hidayat\n");
|
||||
printf("\n");
|
||||
printf("Usage: 6pack [options] input-file output-file\n");
|
||||
printf("\n");
|
||||
printf("Options:\n");
|
||||
printf(" -1 compress faster\n");
|
||||
printf(" -2 compress better\n");
|
||||
printf(" -v show program version\n");
|
||||
#ifdef SIXPACK_BENCHMARK_WIN32
|
||||
printf(" -mem check in-memory compression speed\n");
|
||||
#endif
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
/* return non-zero if magic sequence is detected */
|
||||
/* warning: reset the read pointer to the beginning of the file */
|
||||
int detect_magic(FILE* f) {
|
||||
unsigned char buffer[8];
|
||||
size_t bytes_read;
|
||||
int c;
|
||||
|
||||
fseek(f, SEEK_SET, 0);
|
||||
bytes_read = fread(buffer, 1, 8, f);
|
||||
fseek(f, SEEK_SET, 0);
|
||||
if (bytes_read < 8) return 0;
|
||||
|
||||
for (c = 0; c < 8; c++)
|
||||
if (buffer[c] != sixpack_magic[c]) return 0;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
void write_magic(FILE* f) { fwrite(sixpack_magic, 8, 1, f); }
|
||||
|
||||
void write_chunk_header(FILE* f, int id, int options, unsigned long size, unsigned long checksum, unsigned long extra) {
|
||||
unsigned char buffer[16];
|
||||
|
||||
buffer[0] = id & 255;
|
||||
buffer[1] = id >> 8;
|
||||
buffer[2] = options & 255;
|
||||
buffer[3] = options >> 8;
|
||||
buffer[4] = size & 255;
|
||||
buffer[5] = (size >> 8) & 255;
|
||||
buffer[6] = (size >> 16) & 255;
|
||||
buffer[7] = (size >> 24) & 255;
|
||||
buffer[8] = checksum & 255;
|
||||
buffer[9] = (checksum >> 8) & 255;
|
||||
buffer[10] = (checksum >> 16) & 255;
|
||||
buffer[11] = (checksum >> 24) & 255;
|
||||
buffer[12] = extra & 255;
|
||||
buffer[13] = (extra >> 8) & 255;
|
||||
buffer[14] = (extra >> 16) & 255;
|
||||
buffer[15] = (extra >> 24) & 255;
|
||||
|
||||
fwrite(buffer, 16, 1, f);
|
||||
}
|
||||
|
||||
int pack_file_compressed(const char* input_file, int method, int level, FILE* f) {
|
||||
FILE* in;
|
||||
unsigned long fsize;
|
||||
unsigned long checksum;
|
||||
const char* shown_name;
|
||||
unsigned char buffer[BLOCK_SIZE];
|
||||
unsigned char result[BLOCK_SIZE * 2]; /* FIXME twice is too large */
|
||||
unsigned char progress[20];
|
||||
int c;
|
||||
unsigned long percent;
|
||||
unsigned long total_read;
|
||||
unsigned long total_compressed;
|
||||
int chunk_size;
|
||||
|
||||
/* sanity check */
|
||||
in = fopen(input_file, "rb");
|
||||
if (!in) {
|
||||
printf("Error: could not open %s\n", input_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* find size of the file */
|
||||
fseek(in, 0, SEEK_END);
|
||||
fsize = ftell(in);
|
||||
fseek(in, 0, SEEK_SET);
|
||||
|
||||
/* already a 6pack archive? */
|
||||
if (detect_magic(in)) {
|
||||
printf("Error: file %s is already a 6pack archive!\n", input_file);
|
||||
fclose(in);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */
|
||||
shown_name = input_file + strlen(input_file) - 1;
|
||||
while (shown_name > input_file)
|
||||
if (*(shown_name - 1) == PATH_SEPARATOR)
|
||||
break;
|
||||
else
|
||||
shown_name--;
|
||||
|
||||
/* chunk for File Entry */
|
||||
buffer[0] = fsize & 255;
|
||||
buffer[1] = (fsize >> 8) & 255;
|
||||
buffer[2] = (fsize >> 16) & 255;
|
||||
buffer[3] = (fsize >> 24) & 255;
|
||||
#if 0
|
||||
buffer[4] = (fsize >> 32) & 255;
|
||||
buffer[5] = (fsize >> 40) & 255;
|
||||
buffer[6] = (fsize >> 48) & 255;
|
||||
buffer[7] = (fsize >> 56) & 255;
|
||||
#else
|
||||
/* because fsize is only 32-bit */
|
||||
buffer[4] = 0;
|
||||
buffer[5] = 0;
|
||||
buffer[6] = 0;
|
||||
buffer[7] = 0;
|
||||
#endif
|
||||
buffer[8] = (strlen(shown_name) + 1) & 255;
|
||||
buffer[9] = (strlen(shown_name) + 1) >> 8;
|
||||
checksum = 1L;
|
||||
checksum = update_adler32(checksum, buffer, 10);
|
||||
checksum = update_adler32(checksum, shown_name, strlen(shown_name) + 1);
|
||||
write_chunk_header(f, 1, 0, 10 + strlen(shown_name) + 1, checksum, 0);
|
||||
fwrite(buffer, 10, 1, f);
|
||||
fwrite(shown_name, strlen(shown_name) + 1, 1, f);
|
||||
total_compressed = 16 + 10 + strlen(shown_name) + 1;
|
||||
|
||||
/* for progress status */
|
||||
memset(progress, ' ', 20);
|
||||
if (strlen(shown_name) < 16)
|
||||
for (c = 0; c < (int)strlen(shown_name); c++) progress[c] = shown_name[c];
|
||||
else {
|
||||
for (c = 0; c < 13; c++) progress[c] = shown_name[c];
|
||||
progress[13] = '.';
|
||||
progress[14] = '.';
|
||||
progress[15] = ' ';
|
||||
}
|
||||
progress[16] = '[';
|
||||
progress[17] = 0;
|
||||
printf("%s", progress);
|
||||
for (c = 0; c < 50; c++) printf(".");
|
||||
printf("]\r");
|
||||
printf("%s", progress);
|
||||
|
||||
/* read file and place in archive */
|
||||
total_read = 0;
|
||||
percent = 0;
|
||||
for (;;) {
|
||||
int compress_method = method;
|
||||
int last_percent = (int)percent;
|
||||
size_t bytes_read = fread(buffer, 1, BLOCK_SIZE, in);
|
||||
if (bytes_read == 0) break;
|
||||
total_read += bytes_read;
|
||||
|
||||
/* for progress */
|
||||
if (fsize < (1 << 24))
|
||||
percent = total_read * 100 / fsize;
|
||||
else
|
||||
percent = total_read / 256 * 100 / (fsize >> 8);
|
||||
percent >>= 1;
|
||||
while (last_percent < (int)percent) {
|
||||
printf("#");
|
||||
last_percent++;
|
||||
}
|
||||
|
||||
/* too small, don't bother to compress */
|
||||
if (bytes_read < 32) compress_method = 0;
|
||||
|
||||
/* write to output */
|
||||
switch (compress_method) {
|
||||
/* FastLZ */
|
||||
case 1:
|
||||
chunk_size = fastlz_compress_level(level, buffer, bytes_read, result);
|
||||
checksum = update_adler32(1L, result, chunk_size);
|
||||
write_chunk_header(f, 17, 1, chunk_size, checksum, bytes_read);
|
||||
fwrite(result, 1, chunk_size, f);
|
||||
total_compressed += 16;
|
||||
total_compressed += chunk_size;
|
||||
break;
|
||||
|
||||
/* uncompressed, also fallback method */
|
||||
case 0:
|
||||
default:
|
||||
checksum = 1L;
|
||||
checksum = update_adler32(checksum, buffer, bytes_read);
|
||||
write_chunk_header(f, 17, 0, bytes_read, checksum, bytes_read);
|
||||
fwrite(buffer, 1, bytes_read, f);
|
||||
total_compressed += 16;
|
||||
total_compressed += bytes_read;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(in);
|
||||
if (total_read != fsize) {
|
||||
printf("\n");
|
||||
printf("Error: reading %s failed!\n", input_file);
|
||||
return -1;
|
||||
} else {
|
||||
printf("] ");
|
||||
if (total_compressed < fsize) {
|
||||
if (fsize < (1 << 20))
|
||||
percent = total_compressed * 1000 / fsize;
|
||||
else
|
||||
percent = total_compressed / 256 * 1000 / (fsize >> 8);
|
||||
percent = 1000 - percent;
|
||||
printf("%2d.%d%% saved", (int)percent / 10, (int)percent % 10);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int pack_file(int compress_level, const char* input_file, const char* output_file) {
|
||||
FILE* f;
|
||||
int result;
|
||||
|
||||
f = fopen(output_file, "rb");
|
||||
if (f) {
|
||||
fclose(f);
|
||||
printf("Error: file %s already exists. Aborted.\n\n", output_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
f = fopen(output_file, "wb");
|
||||
if (!f) {
|
||||
printf("Error: could not create %s. Aborted.\n\n", output_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
write_magic(f);
|
||||
|
||||
result = pack_file_compressed(input_file, 1, compress_level, f);
|
||||
fclose(f);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#ifdef SIXPACK_BENCHMARK_WIN32
|
||||
int benchmark_speed(int compress_level, const char* input_file);
|
||||
|
||||
int benchmark_speed(int compress_level, const char* input_file) {
|
||||
FILE* in;
|
||||
unsigned long fsize;
|
||||
unsigned long maxout;
|
||||
const char* shown_name;
|
||||
unsigned char* buffer;
|
||||
unsigned char* result;
|
||||
size_t bytes_read;
|
||||
|
||||
/* sanity check */
|
||||
in = fopen(input_file, "rb");
|
||||
if (!in) {
|
||||
printf("Error: could not open %s\n", input_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* find size of the file */
|
||||
fseek(in, 0, SEEK_END);
|
||||
fsize = ftell(in);
|
||||
fseek(in, 0, SEEK_SET);
|
||||
|
||||
/* already a 6pack archive? */
|
||||
if (detect_magic(in)) {
|
||||
printf("Error: no benchmark for 6pack archive!\n");
|
||||
fclose(in);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */
|
||||
shown_name = input_file + strlen(input_file) - 1;
|
||||
while (shown_name > input_file)
|
||||
if (*(shown_name - 1) == PATH_SEPARATOR)
|
||||
break;
|
||||
else
|
||||
shown_name--;
|
||||
|
||||
maxout = 1.05 * fsize;
|
||||
maxout = (maxout < 66) ? 66 : maxout;
|
||||
buffer = (unsigned char*)malloc(fsize);
|
||||
result = (unsigned char*)malloc(maxout);
|
||||
if (!buffer || !result) {
|
||||
printf("Error: not enough memory!\n");
|
||||
free(buffer);
|
||||
free(result);
|
||||
fclose(in);
|
||||
return -1;
|
||||
}
|
||||
|
||||
printf("Reading source file....\n");
|
||||
bytes_read = fread(buffer, 1, fsize, in);
|
||||
if (bytes_read != fsize) {
|
||||
printf("Error reading file %s!\n", shown_name);
|
||||
printf("Read %d bytes, expecting %d bytes\n", bytes_read, fsize);
|
||||
free(buffer);
|
||||
free(result);
|
||||
fclose(in);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* shamelessly copied from QuickLZ 1.20 test program */
|
||||
{
|
||||
unsigned int j, y;
|
||||
size_t i, u = 0;
|
||||
double mbs, fastest;
|
||||
unsigned long compressed_size;
|
||||
|
||||
printf("Setting HIGH_PRIORITY_CLASS...\n");
|
||||
SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS);
|
||||
|
||||
printf("Benchmarking FastLZ Level %d, please wait...\n", compress_level);
|
||||
|
||||
i = bytes_read;
|
||||
fastest = 0.0;
|
||||
for (j = 0; j < 3; j++) {
|
||||
y = 0;
|
||||
mbs = GetTickCount();
|
||||
while (GetTickCount() == mbs)
|
||||
;
|
||||
mbs = GetTickCount();
|
||||
while (GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */
|
||||
{
|
||||
u = fastlz_compress_level(compress_level, buffer, bytes_read, result);
|
||||
y++;
|
||||
}
|
||||
|
||||
mbs = ((double)i * (double)y) / ((double)(GetTickCount() - mbs) / 1000.) / 1000000.;
|
||||
/*printf(" %.1f Mbyte/s ", mbs);*/
|
||||
if (fastest < mbs) fastest = mbs;
|
||||
}
|
||||
|
||||
printf("\nCompressed %d bytes into %d bytes (%.1f%%) at %.1f Mbyte/s.\n", (unsigned int)i, (unsigned int)u,
|
||||
(double)u / (double)i * 100., fastest);
|
||||
|
||||
#if 1
|
||||
fastest = 0.0;
|
||||
compressed_size = u;
|
||||
for (j = 0; j < 3; j++) {
|
||||
y = 0;
|
||||
mbs = GetTickCount();
|
||||
while (GetTickCount() == mbs)
|
||||
;
|
||||
mbs = GetTickCount();
|
||||
while (GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */
|
||||
{
|
||||
u = fastlz_decompress(result, compressed_size, buffer, bytes_read);
|
||||
y++;
|
||||
}
|
||||
|
||||
mbs = ((double)i * (double)y) / ((double)(GetTickCount() - mbs) / 1000.) / 1000000.;
|
||||
/*printf(" %.1f Mbyte/s ", mbs);*/
|
||||
if (fastest < mbs) fastest = mbs;
|
||||
}
|
||||
|
||||
printf("\nDecompressed at %.1f Mbyte/s.\n\n(1 MB = 1000000 byte)\n", fastest);
|
||||
#endif
|
||||
}
|
||||
|
||||
fclose(in);
|
||||
return 0;
|
||||
}
|
||||
#endif /* SIXPACK_BENCHMARK_WIN32 */
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
int i;
|
||||
int compress_level;
|
||||
int benchmark;
|
||||
char* input_file;
|
||||
char* output_file;
|
||||
|
||||
/* show help with no argument at all*/
|
||||
if (argc == 1) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* default compression level, not the fastest */
|
||||
compress_level = 2;
|
||||
|
||||
/* do benchmark only when explicitly specified */
|
||||
benchmark = 0;
|
||||
|
||||
/* no file is specified */
|
||||
input_file = 0;
|
||||
output_file = 0;
|
||||
|
||||
for (i = 1; i <= argc; i++) {
|
||||
char* argument = argv[i];
|
||||
|
||||
if (!argument) continue;
|
||||
|
||||
/* display help on usage */
|
||||
if (!strcmp(argument, "-h") || !strcmp(argument, "--help")) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* check for version information */
|
||||
if (!strcmp(argument, "-v") || !strcmp(argument, "--version")) {
|
||||
printf("6pack: high-speed file compression tool\n");
|
||||
printf("Version %s (using FastLZ %s)\n", SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING);
|
||||
printf("Copyright (C) Ariya Hidayat\n");
|
||||
printf("\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* test compression speed? */
|
||||
if (!strcmp(argument, "-mem")) {
|
||||
benchmark = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* compression level */
|
||||
if (!strcmp(argument, "-1") || !strcmp(argument, "--fastest")) {
|
||||
compress_level = 1;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argument, "-2")) {
|
||||
compress_level = 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* unknown option */
|
||||
if (argument[0] == '-') {
|
||||
printf("Error: unknown option %s\n\n", argument);
|
||||
printf("To get help on usage:\n");
|
||||
printf(" 6pack --help\n\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* first specified file is input */
|
||||
if (!input_file) {
|
||||
input_file = argument;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* next specified file is output */
|
||||
if (!output_file) {
|
||||
output_file = argument;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* files are already specified */
|
||||
printf("Error: unknown option %s\n\n", argument);
|
||||
printf("To get help on usage:\n");
|
||||
printf(" 6pack --help\n\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!input_file) {
|
||||
printf("Error: input file is not specified.\n\n");
|
||||
printf("To get help on usage:\n");
|
||||
printf(" 6pack --help\n\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!output_file && !benchmark) {
|
||||
printf("Error: output file is not specified.\n\n");
|
||||
printf("To get help on usage:\n");
|
||||
printf(" 6pack --help\n\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
#ifdef SIXPACK_BENCHMARK_WIN32
|
||||
if (benchmark)
|
||||
return benchmark_speed(compress_level, input_file);
|
||||
else
|
||||
#endif
|
||||
return pack_file(compress_level, input_file, output_file);
|
||||
|
||||
/* unreachable */
|
||||
return 0;
|
||||
}
|
||||
425
FastLZ/examples/6unpack.c
Normal file
425
FastLZ/examples/6unpack.c
Normal file
@ -0,0 +1,425 @@
|
||||
/*
|
||||
6PACK - file compressor using FastLZ (lightning-fast compression library)
|
||||
Copyright (C) 2007-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define SIXPACK_VERSION_MAJOR 0
|
||||
#define SIXPACK_VERSION_MINOR 1
|
||||
#define SIXPACK_VERSION_REVISION 0
|
||||
#define SIXPACK_VERSION_STRING "0.1.0"
|
||||
|
||||
#include "fastlz.h"
|
||||
|
||||
/* magic identifier for 6pack file */
|
||||
static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10};
|
||||
|
||||
#define BLOCK_SIZE 65536
|
||||
|
||||
/* prototypes */
|
||||
static unsigned long update_adler32(unsigned long checksum, const void* buf, int len);
|
||||
void usage(void);
|
||||
int detect_magic(FILE* f);
|
||||
static unsigned long readU16(const unsigned char* ptr);
|
||||
static unsigned long readU32(const unsigned char* ptr);
|
||||
void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size, unsigned long* checksum,
|
||||
unsigned long* extra);
|
||||
int unpack_file(const char* archive_file);
|
||||
|
||||
/* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */
|
||||
#define ADLER32_BASE 65521
|
||||
static unsigned long update_adler32(unsigned long checksum, const void* buf, int len) {
|
||||
const unsigned char* ptr = (const unsigned char*)buf;
|
||||
unsigned long s1 = checksum & 0xffff;
|
||||
unsigned long s2 = (checksum >> 16) & 0xffff;
|
||||
|
||||
while (len > 0) {
|
||||
unsigned k = len < 5552 ? len : 5552;
|
||||
len -= k;
|
||||
|
||||
while (k >= 8) {
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
k -= 8;
|
||||
}
|
||||
|
||||
while (k-- > 0) {
|
||||
s1 += *ptr++;
|
||||
s2 += s1;
|
||||
}
|
||||
s1 = s1 % ADLER32_BASE;
|
||||
s2 = s2 % ADLER32_BASE;
|
||||
}
|
||||
return (s2 << 16) + s1;
|
||||
}
|
||||
|
||||
void usage(void) {
|
||||
printf("6unpack: uncompress 6pack archive\n");
|
||||
printf("Copyright (C) Ariya Hidayat\n");
|
||||
printf("\n");
|
||||
printf("Usage: 6unpack archive-file\n");
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
/* return non-zero if magic sequence is detected */
|
||||
/* warning: reset the read pointer to the beginning of the file */
|
||||
int detect_magic(FILE* f) {
|
||||
unsigned char buffer[8];
|
||||
size_t bytes_read;
|
||||
int c;
|
||||
|
||||
fseek(f, SEEK_SET, 0);
|
||||
bytes_read = fread(buffer, 1, 8, f);
|
||||
fseek(f, SEEK_SET, 0);
|
||||
if (bytes_read < 8) return 0;
|
||||
|
||||
for (c = 0; c < 8; c++)
|
||||
if (buffer[c] != sixpack_magic[c]) return 0;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
static unsigned long readU16(const unsigned char* ptr) { return ptr[0] + (ptr[1] << 8); }
|
||||
|
||||
static unsigned long readU32(const unsigned char* ptr) {
|
||||
return ptr[0] + (ptr[1] << 8) + (ptr[2] << 16) + (ptr[3] << 24);
|
||||
}
|
||||
|
||||
void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size, unsigned long* checksum,
|
||||
unsigned long* extra) {
|
||||
unsigned char buffer[16];
|
||||
fread(buffer, 1, 16, f);
|
||||
|
||||
*id = readU16(buffer) & 0xffff;
|
||||
*options = readU16(buffer + 2) & 0xffff;
|
||||
*size = readU32(buffer + 4) & 0xffffffff;
|
||||
*checksum = readU32(buffer + 8) & 0xffffffff;
|
||||
*extra = readU32(buffer + 12) & 0xffffffff;
|
||||
}
|
||||
|
||||
int unpack_file(const char* input_file) {
|
||||
FILE* in;
|
||||
unsigned long fsize;
|
||||
int c;
|
||||
unsigned long percent;
|
||||
unsigned char progress[20];
|
||||
int chunk_id;
|
||||
int chunk_options;
|
||||
unsigned long chunk_size;
|
||||
unsigned long chunk_checksum;
|
||||
unsigned long chunk_extra;
|
||||
unsigned char buffer[BLOCK_SIZE];
|
||||
unsigned long checksum;
|
||||
|
||||
unsigned long decompressed_size;
|
||||
unsigned long total_extracted;
|
||||
int name_length;
|
||||
char* output_file;
|
||||
FILE* f;
|
||||
|
||||
unsigned char* compressed_buffer;
|
||||
unsigned char* decompressed_buffer;
|
||||
unsigned long compressed_bufsize;
|
||||
unsigned long decompressed_bufsize;
|
||||
|
||||
/* sanity check */
|
||||
in = fopen(input_file, "rb");
|
||||
if (!in) {
|
||||
printf("Error: could not open %s\n", input_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* find size of the file */
|
||||
fseek(in, 0, SEEK_END);
|
||||
fsize = ftell(in);
|
||||
fseek(in, 0, SEEK_SET);
|
||||
|
||||
/* not a 6pack archive? */
|
||||
if (!detect_magic(in)) {
|
||||
fclose(in);
|
||||
printf("Error: file %s is not a 6pack archive!\n", input_file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
printf("Archive: %s", input_file);
|
||||
|
||||
/* position of first chunk */
|
||||
fseek(in, 8, SEEK_SET);
|
||||
|
||||
/* initialize */
|
||||
output_file = 0;
|
||||
f = 0;
|
||||
total_extracted = 0;
|
||||
decompressed_size = 0;
|
||||
percent = 0;
|
||||
compressed_buffer = 0;
|
||||
decompressed_buffer = 0;
|
||||
compressed_bufsize = 0;
|
||||
decompressed_bufsize = 0;
|
||||
|
||||
/* main loop */
|
||||
for (;;) {
|
||||
/* end of file? */
|
||||
size_t pos = ftell(in);
|
||||
if (pos >= fsize) break;
|
||||
|
||||
read_chunk_header(in, &chunk_id, &chunk_options, &chunk_size, &chunk_checksum, &chunk_extra);
|
||||
|
||||
if ((chunk_id == 1) && (chunk_size > 10) && (chunk_size < BLOCK_SIZE)) {
|
||||
/* close current file, if any */
|
||||
printf("\n");
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
if (f) fclose(f);
|
||||
|
||||
/* file entry */
|
||||
fread(buffer, 1, chunk_size, in);
|
||||
checksum = update_adler32(1L, buffer, chunk_size);
|
||||
if (checksum != chunk_checksum) {
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
fclose(in);
|
||||
printf("\nError: checksum mismatch!\n");
|
||||
printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
|
||||
return -1;
|
||||
}
|
||||
|
||||
decompressed_size = readU32(buffer);
|
||||
total_extracted = 0;
|
||||
percent = 0;
|
||||
|
||||
/* get file to extract */
|
||||
name_length = (int)readU16(buffer + 8);
|
||||
if (name_length > (int)chunk_size - 10) name_length = chunk_size - 10;
|
||||
output_file = (char*)malloc(name_length + 1);
|
||||
memset(output_file, 0, name_length + 1);
|
||||
for (c = 0; c < name_length; c++) output_file[c] = buffer[10 + c];
|
||||
|
||||
/* check if already exists */
|
||||
f = fopen(output_file, "rb");
|
||||
if (f) {
|
||||
fclose(f);
|
||||
printf("File %s already exists. Skipped.\n", output_file);
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
f = 0;
|
||||
} else {
|
||||
/* create the file */
|
||||
f = fopen(output_file, "wb");
|
||||
if (!f) {
|
||||
printf("Can't create file %s. Skipped.\n", output_file);
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
f = 0;
|
||||
} else {
|
||||
/* for progress status */
|
||||
printf("\n");
|
||||
memset(progress, ' ', 20);
|
||||
if (strlen(output_file) < 16)
|
||||
for (c = 0; c < (int)strlen(output_file); c++) progress[c] = output_file[c];
|
||||
else {
|
||||
for (c = 0; c < 13; c++) progress[c] = output_file[c];
|
||||
progress[13] = '.';
|
||||
progress[14] = '.';
|
||||
progress[15] = ' ';
|
||||
}
|
||||
progress[16] = '[';
|
||||
progress[17] = 0;
|
||||
printf("%s", progress);
|
||||
for (c = 0; c < 50; c++) printf(".");
|
||||
printf("]\r");
|
||||
printf("%s", progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((chunk_id == 17) && f && output_file && decompressed_size) {
|
||||
unsigned long remaining;
|
||||
|
||||
/* uncompressed */
|
||||
switch (chunk_options) {
|
||||
/* stored, simply copy to output */
|
||||
case 0:
|
||||
/* read one block at at time, write and update checksum */
|
||||
total_extracted += chunk_size;
|
||||
remaining = chunk_size;
|
||||
checksum = 1L;
|
||||
for (;;) {
|
||||
unsigned long r = (BLOCK_SIZE < remaining) ? BLOCK_SIZE : remaining;
|
||||
size_t bytes_read = fread(buffer, 1, r, in);
|
||||
if (bytes_read == 0) break;
|
||||
fwrite(buffer, 1, bytes_read, f);
|
||||
checksum = update_adler32(checksum, buffer, bytes_read);
|
||||
remaining -= bytes_read;
|
||||
}
|
||||
|
||||
/* verify everything is written correctly */
|
||||
if (checksum != chunk_checksum) {
|
||||
fclose(f);
|
||||
f = 0;
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
printf("\nError: checksum mismatch. Aborted.\n");
|
||||
printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
|
||||
}
|
||||
break;
|
||||
|
||||
/* compressed using FastLZ */
|
||||
case 1:
|
||||
/* enlarge input buffer if necessary */
|
||||
if (chunk_size > compressed_bufsize) {
|
||||
compressed_bufsize = chunk_size;
|
||||
free(compressed_buffer);
|
||||
compressed_buffer = (unsigned char*)malloc(compressed_bufsize);
|
||||
}
|
||||
|
||||
/* enlarge output buffer if necessary */
|
||||
if (chunk_extra > decompressed_bufsize) {
|
||||
decompressed_bufsize = chunk_extra;
|
||||
free(decompressed_buffer);
|
||||
decompressed_buffer = (unsigned char*)malloc(decompressed_bufsize);
|
||||
}
|
||||
|
||||
/* read and check checksum */
|
||||
fread(compressed_buffer, 1, chunk_size, in);
|
||||
checksum = update_adler32(1L, compressed_buffer, chunk_size);
|
||||
total_extracted += chunk_extra;
|
||||
|
||||
/* verify that the chunk data is correct */
|
||||
if (checksum != chunk_checksum) {
|
||||
fclose(f);
|
||||
f = 0;
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
printf("\nError: checksum mismatch. Skipped.\n");
|
||||
printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum);
|
||||
} else {
|
||||
/* decompress and verify */
|
||||
remaining = fastlz_decompress(compressed_buffer, chunk_size, decompressed_buffer, chunk_extra);
|
||||
if (remaining != chunk_extra) {
|
||||
fclose(f);
|
||||
f = 0;
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
printf("\nError: decompression failed. Skipped.\n");
|
||||
} else
|
||||
fwrite(decompressed_buffer, 1, chunk_extra, f);
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("\nError: unknown compression method (%d)\n", chunk_options);
|
||||
fclose(f);
|
||||
f = 0;
|
||||
free(output_file);
|
||||
output_file = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
/* for progress, if everything is fine */
|
||||
if (f) {
|
||||
int last_percent = (int)percent;
|
||||
if (decompressed_size < (1 << 24))
|
||||
percent = total_extracted * 100 / decompressed_size;
|
||||
else
|
||||
percent = total_extracted / 256 * 100 / (decompressed_size >> 8);
|
||||
percent >>= 1;
|
||||
while (last_percent < (int)percent) {
|
||||
printf("#");
|
||||
last_percent++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* position of next chunk */
|
||||
fseek(in, pos + 16 + chunk_size, SEEK_SET);
|
||||
}
|
||||
printf("\n\n");
|
||||
|
||||
/* free allocated stuff */
|
||||
free(compressed_buffer);
|
||||
free(decompressed_buffer);
|
||||
free(output_file);
|
||||
|
||||
/* close working files */
|
||||
if (f) fclose(f);
|
||||
fclose(in);
|
||||
|
||||
/* so far so good */
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
int i;
|
||||
const char* archive_file;
|
||||
|
||||
/* show help with no argument at all*/
|
||||
if (argc == 1) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* check for help on usage */
|
||||
for (i = 1; i <= argc; i++)
|
||||
if (argv[i])
|
||||
if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* check for version information */
|
||||
for (i = 1; i <= argc; i++)
|
||||
if (argv[i])
|
||||
if (!strcmp(argv[i], "-v") || !strcmp(argv[i], "--version")) {
|
||||
printf("6unpack: high-speed file compression tool\n");
|
||||
printf("Version %s (using FastLZ %s)\n", SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING);
|
||||
printf("Copyright (C) Ariya Hidayat\n");
|
||||
printf("\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* needs at least two arguments */
|
||||
if (argc <= 1) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
archive_file = argv[1];
|
||||
|
||||
return unpack_file(archive_file);
|
||||
}
|
||||
12
FastLZ/examples/Makefile
Normal file
12
FastLZ/examples/Makefile
Normal file
@ -0,0 +1,12 @@
|
||||
CFLAGS?=-Wall -std=c90
|
||||
|
||||
all: 6pack 6unpack
|
||||
|
||||
6pack: 6pack.c ../fastlz.c
|
||||
$(CC) -o 6pack $(CFLAGS) -I.. 6pack.c ../fastlz.c
|
||||
|
||||
6unpack: 6unpack.c ../fastlz.c
|
||||
$(CC) -o 6unpack $(CFLAGS) -I.. 6unpack.c ../fastlz.c
|
||||
|
||||
clean :
|
||||
$(RM) 6pack 6unpack *.o
|
||||
14
FastLZ/examples/Makefile.win
Executable file
14
FastLZ/examples/Makefile.win
Executable file
@ -0,0 +1,14 @@
|
||||
CC=cl.exe
|
||||
CFLAGS=/Wall
|
||||
RM=del
|
||||
|
||||
all: 6pack 6unpack
|
||||
|
||||
6pack: 6pack.c ../fastlz.c
|
||||
$(CC) -o 6pack $(CFLAGS) -I.. 6pack.c ../fastlz.c
|
||||
|
||||
6unpack: 6unpack.c ../fastlz.c
|
||||
$(CC) -o 6unpack $(CFLAGS) -I.. 6unpack.c ../fastlz.c
|
||||
|
||||
clean :
|
||||
$(RM) 6pack.exe 6unpack.exe *.obj
|
||||
508
FastLZ/fastlz.c
Normal file
508
FastLZ/fastlz.c
Normal file
@ -0,0 +1,508 @@
|
||||
/*
|
||||
FastLZ - Byte-aligned LZ77 compression library
|
||||
Copyright (C) 2005-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "fastlz.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
|
||||
|
||||
/*
|
||||
* Give hints to the compiler for branch prediction optimization.
|
||||
*/
|
||||
#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2))
|
||||
#define FASTLZ_LIKELY(c) (__builtin_expect(!!(c), 1))
|
||||
#define FASTLZ_UNLIKELY(c) (__builtin_expect(!!(c), 0))
|
||||
#else
|
||||
#define FASTLZ_LIKELY(c) (c)
|
||||
#define FASTLZ_UNLIKELY(c) (c)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Specialize custom 64-bit implementation for speed improvements.
|
||||
*/
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__)
|
||||
#define FLZ_ARCH64
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Workaround for DJGPP to find uint8_t, uint16_t, etc.
|
||||
*/
|
||||
#if defined(__MSDOS__) && defined(__GNUC__)
|
||||
#include <stdint-gcc.h>
|
||||
#endif
|
||||
|
||||
#if defined(FASTLZ_USE_MEMMOVE) && (FASTLZ_USE_MEMMOVE == 0)
|
||||
|
||||
static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) {
|
||||
do {
|
||||
*dest++ = *src++;
|
||||
} while (--count);
|
||||
}
|
||||
|
||||
static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) {
|
||||
return fastlz_memmove(dest, src, count);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#include <string.h>
|
||||
|
||||
static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) {
|
||||
if ((count > 4) && (dest >= src + count)) {
|
||||
memmove(dest, src, count);
|
||||
} else {
|
||||
switch (count) {
|
||||
default:
|
||||
do {
|
||||
*dest++ = *src++;
|
||||
} while (--count);
|
||||
break;
|
||||
case 3:
|
||||
*dest++ = *src++;
|
||||
case 2:
|
||||
*dest++ = *src++;
|
||||
case 1:
|
||||
*dest++ = *src++;
|
||||
case 0:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { memcpy(dest, src, count); }
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(FLZ_ARCH64)
|
||||
|
||||
static uint32_t flz_readu32(const void* ptr) { return *(const uint32_t*)ptr; }
|
||||
|
||||
static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) {
|
||||
const uint8_t* start = p;
|
||||
|
||||
if (flz_readu32(p) == flz_readu32(q)) {
|
||||
p += 4;
|
||||
q += 4;
|
||||
}
|
||||
while (q < r)
|
||||
if (*p++ != *q++) break;
|
||||
return p - start;
|
||||
}
|
||||
|
||||
#endif /* FLZ_ARCH64 */
|
||||
|
||||
#if !defined(FLZ_ARCH64)
|
||||
|
||||
static uint32_t flz_readu32(const void* ptr) {
|
||||
const uint8_t* p = (const uint8_t*)ptr;
|
||||
return (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0];
|
||||
}
|
||||
|
||||
static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) {
|
||||
const uint8_t* start = p;
|
||||
while (q < r)
|
||||
if (*p++ != *q++) break;
|
||||
return p - start;
|
||||
}
|
||||
|
||||
#endif /* !FLZ_ARCH64 */
|
||||
|
||||
#define MAX_COPY 32
|
||||
#define MAX_LEN 264 /* 256 + 8 */
|
||||
#define MAX_L1_DISTANCE 8192
|
||||
#define MAX_L2_DISTANCE 8191
|
||||
#define MAX_FARDISTANCE (65535 + MAX_L2_DISTANCE - 1)
|
||||
|
||||
#define HASH_LOG 13
|
||||
#define HASH_SIZE (1 << HASH_LOG)
|
||||
#define HASH_MASK (HASH_SIZE - 1)
|
||||
|
||||
static uint16_t flz_hash(uint32_t v) {
|
||||
uint32_t h = (v * 2654435769LL) >> (32 - HASH_LOG);
|
||||
return h & HASH_MASK;
|
||||
}
|
||||
|
||||
/* special case of memcpy: at most MAX_COPY bytes */
|
||||
static void flz_smallcopy(uint8_t* dest, const uint8_t* src, uint32_t count) {
|
||||
#if defined(FLZ_ARCH64)
|
||||
if (count >= 4) {
|
||||
const uint32_t* p = (const uint32_t*)src;
|
||||
uint32_t* q = (uint32_t*)dest;
|
||||
while (count > 4) {
|
||||
*q++ = *p++;
|
||||
count -= 4;
|
||||
dest += 4;
|
||||
src += 4;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
fastlz_memcpy(dest, src, count);
|
||||
}
|
||||
|
||||
/* special case of memcpy: exactly MAX_COPY bytes */
|
||||
static void flz_maxcopy(void* dest, const void* src) {
|
||||
#if defined(FLZ_ARCH64)
|
||||
const uint32_t* p = (const uint32_t*)src;
|
||||
uint32_t* q = (uint32_t*)dest;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
*q++ = *p++;
|
||||
#else
|
||||
fastlz_memcpy(dest, src, MAX_COPY);
|
||||
#endif
|
||||
}
|
||||
|
||||
static uint8_t* flz_literals(uint32_t runs, const uint8_t* src, uint8_t* dest) {
|
||||
while (runs >= MAX_COPY) {
|
||||
*dest++ = MAX_COPY - 1;
|
||||
flz_maxcopy(dest, src);
|
||||
src += MAX_COPY;
|
||||
dest += MAX_COPY;
|
||||
runs -= MAX_COPY;
|
||||
}
|
||||
if (runs > 0) {
|
||||
*dest++ = runs - 1;
|
||||
flz_smallcopy(dest, src, runs);
|
||||
dest += runs;
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
|
||||
static uint8_t* flz1_match(uint32_t len, uint32_t distance, uint8_t* op) {
|
||||
--distance;
|
||||
if (FASTLZ_UNLIKELY(len > MAX_LEN - 2))
|
||||
while (len > MAX_LEN - 2) {
|
||||
*op++ = (7 << 5) + (distance >> 8);
|
||||
*op++ = MAX_LEN - 2 - 7 - 2;
|
||||
*op++ = (distance & 255);
|
||||
len -= MAX_LEN - 2;
|
||||
}
|
||||
if (len < 7) {
|
||||
*op++ = (len << 5) + (distance >> 8);
|
||||
*op++ = (distance & 255);
|
||||
} else {
|
||||
*op++ = (7 << 5) + (distance >> 8);
|
||||
*op++ = len - 7;
|
||||
*op++ = (distance & 255);
|
||||
}
|
||||
return op;
|
||||
}
|
||||
|
||||
#define FASTLZ_BOUND_CHECK(cond) \
|
||||
if (FASTLZ_UNLIKELY(!(cond))) return 0;
|
||||
|
||||
static int fastlz1_compress(const void* input, int length, void* output) {
|
||||
const uint8_t* ip = (const uint8_t*)input;
|
||||
const uint8_t* ip_start = ip;
|
||||
const uint8_t* ip_bound = ip + length - 4; /* because readU32 */
|
||||
const uint8_t* ip_limit = ip + length - 12 - 1;
|
||||
uint8_t* op = (uint8_t*)output;
|
||||
|
||||
uint32_t htab[HASH_SIZE];
|
||||
uint32_t seq, hash;
|
||||
|
||||
/* initializes hash table */
|
||||
for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0;
|
||||
|
||||
/* we start with literal copy */
|
||||
const uint8_t* anchor = ip;
|
||||
ip += 2;
|
||||
|
||||
/* main loop */
|
||||
while (FASTLZ_LIKELY(ip < ip_limit)) {
|
||||
const uint8_t* ref;
|
||||
uint32_t distance, cmp;
|
||||
|
||||
/* find potential match */
|
||||
do {
|
||||
seq = flz_readu32(ip) & 0xffffff;
|
||||
hash = flz_hash(seq);
|
||||
ref = ip_start + htab[hash];
|
||||
htab[hash] = ip - ip_start;
|
||||
distance = ip - ref;
|
||||
cmp = FASTLZ_LIKELY(distance < MAX_L1_DISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000;
|
||||
if (FASTLZ_UNLIKELY(ip >= ip_limit)) break;
|
||||
++ip;
|
||||
} while (seq != cmp);
|
||||
|
||||
if (FASTLZ_UNLIKELY(ip >= ip_limit)) break;
|
||||
--ip;
|
||||
|
||||
if (FASTLZ_LIKELY(ip > anchor)) {
|
||||
op = flz_literals(ip - anchor, anchor, op);
|
||||
}
|
||||
|
||||
uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound);
|
||||
op = flz1_match(len, distance, op);
|
||||
|
||||
/* update the hash at match boundary */
|
||||
ip += len;
|
||||
seq = flz_readu32(ip);
|
||||
hash = flz_hash(seq & 0xffffff);
|
||||
htab[hash] = ip++ - ip_start;
|
||||
seq >>= 8;
|
||||
hash = flz_hash(seq);
|
||||
htab[hash] = ip++ - ip_start;
|
||||
|
||||
anchor = ip;
|
||||
}
|
||||
|
||||
uint32_t copy = (uint8_t*)input + length - anchor;
|
||||
op = flz_literals(copy, anchor, op);
|
||||
|
||||
return op - (uint8_t*)output;
|
||||
}
|
||||
|
||||
static int fastlz1_decompress(const void* input, int length, void* output, int maxout) {
|
||||
const uint8_t* ip = (const uint8_t*)input;
|
||||
const uint8_t* ip_limit = ip + length;
|
||||
const uint8_t* ip_bound = ip_limit - 2;
|
||||
uint8_t* op = (uint8_t*)output;
|
||||
uint8_t* op_limit = op + maxout;
|
||||
uint32_t ctrl = (*ip++) & 31;
|
||||
|
||||
while (1) {
|
||||
if (ctrl >= 32) {
|
||||
uint32_t len = (ctrl >> 5) - 1;
|
||||
uint32_t ofs = (ctrl & 31) << 8;
|
||||
const uint8_t* ref = op - ofs - 1;
|
||||
if (len == 7 - 1) {
|
||||
FASTLZ_BOUND_CHECK(ip <= ip_bound);
|
||||
len += *ip++;
|
||||
}
|
||||
ref -= *ip++;
|
||||
len += 3;
|
||||
FASTLZ_BOUND_CHECK(op + len <= op_limit);
|
||||
FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output);
|
||||
fastlz_memmove(op, ref, len);
|
||||
op += len;
|
||||
} else {
|
||||
ctrl++;
|
||||
FASTLZ_BOUND_CHECK(op + ctrl <= op_limit);
|
||||
FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit);
|
||||
fastlz_memcpy(op, ip, ctrl);
|
||||
ip += ctrl;
|
||||
op += ctrl;
|
||||
}
|
||||
|
||||
if (FASTLZ_UNLIKELY(ip > ip_bound)) break;
|
||||
ctrl = *ip++;
|
||||
}
|
||||
|
||||
return op - (uint8_t*)output;
|
||||
}
|
||||
|
||||
static uint8_t* flz2_match(uint32_t len, uint32_t distance, uint8_t* op) {
|
||||
--distance;
|
||||
if (distance < MAX_L2_DISTANCE) {
|
||||
if (len < 7) {
|
||||
*op++ = (len << 5) + (distance >> 8);
|
||||
*op++ = (distance & 255);
|
||||
} else {
|
||||
*op++ = (7 << 5) + (distance >> 8);
|
||||
for (len -= 7; len >= 255; len -= 255) *op++ = 255;
|
||||
*op++ = len;
|
||||
*op++ = (distance & 255);
|
||||
}
|
||||
} else {
|
||||
/* far away, but not yet in the another galaxy... */
|
||||
if (len < 7) {
|
||||
distance -= MAX_L2_DISTANCE;
|
||||
*op++ = (len << 5) + 31;
|
||||
*op++ = 255;
|
||||
*op++ = distance >> 8;
|
||||
*op++ = distance & 255;
|
||||
} else {
|
||||
distance -= MAX_L2_DISTANCE;
|
||||
*op++ = (7 << 5) + 31;
|
||||
for (len -= 7; len >= 255; len -= 255) *op++ = 255;
|
||||
*op++ = len;
|
||||
*op++ = 255;
|
||||
*op++ = distance >> 8;
|
||||
*op++ = distance & 255;
|
||||
}
|
||||
}
|
||||
return op;
|
||||
}
|
||||
|
||||
static int fastlz2_compress(const void* input, int length, void* output) {
|
||||
const uint8_t* ip = (const uint8_t*)input;
|
||||
const uint8_t* ip_start = ip;
|
||||
const uint8_t* ip_bound = ip + length - 4; /* because readU32 */
|
||||
const uint8_t* ip_limit = ip + length - 12 - 1;
|
||||
uint8_t* op = (uint8_t*)output;
|
||||
|
||||
uint32_t htab[HASH_SIZE];
|
||||
uint32_t seq, hash;
|
||||
|
||||
/* initializes hash table */
|
||||
for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0;
|
||||
|
||||
/* we start with literal copy */
|
||||
const uint8_t* anchor = ip;
|
||||
ip += 2;
|
||||
|
||||
/* main loop */
|
||||
while (FASTLZ_LIKELY(ip < ip_limit)) {
|
||||
const uint8_t* ref;
|
||||
uint32_t distance, cmp;
|
||||
|
||||
/* find potential match */
|
||||
do {
|
||||
seq = flz_readu32(ip) & 0xffffff;
|
||||
hash = flz_hash(seq);
|
||||
ref = ip_start + htab[hash];
|
||||
htab[hash] = ip - ip_start;
|
||||
distance = ip - ref;
|
||||
cmp = FASTLZ_LIKELY(distance < MAX_FARDISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000;
|
||||
if (FASTLZ_UNLIKELY(ip >= ip_limit)) break;
|
||||
++ip;
|
||||
} while (seq != cmp);
|
||||
|
||||
if (FASTLZ_UNLIKELY(ip >= ip_limit)) break;
|
||||
|
||||
--ip;
|
||||
|
||||
/* far, needs at least 5-byte match */
|
||||
if (distance >= MAX_L2_DISTANCE) {
|
||||
if (ref[3] != ip[3] || ref[4] != ip[4]) {
|
||||
++ip;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (FASTLZ_LIKELY(ip > anchor)) {
|
||||
op = flz_literals(ip - anchor, anchor, op);
|
||||
}
|
||||
|
||||
uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound);
|
||||
op = flz2_match(len, distance, op);
|
||||
|
||||
/* update the hash at match boundary */
|
||||
ip += len;
|
||||
seq = flz_readu32(ip);
|
||||
hash = flz_hash(seq & 0xffffff);
|
||||
htab[hash] = ip++ - ip_start;
|
||||
seq >>= 8;
|
||||
hash = flz_hash(seq);
|
||||
htab[hash] = ip++ - ip_start;
|
||||
|
||||
anchor = ip;
|
||||
}
|
||||
|
||||
uint32_t copy = (uint8_t*)input + length - anchor;
|
||||
op = flz_literals(copy, anchor, op);
|
||||
|
||||
/* marker for fastlz2 */
|
||||
*(uint8_t*)output |= (1 << 5);
|
||||
|
||||
return op - (uint8_t*)output;
|
||||
}
|
||||
|
||||
static int fastlz2_decompress(const void* input, int length, void* output, int maxout) {
|
||||
const uint8_t* ip = (const uint8_t*)input;
|
||||
const uint8_t* ip_limit = ip + length;
|
||||
const uint8_t* ip_bound = ip_limit - 2;
|
||||
uint8_t* op = (uint8_t*)output;
|
||||
uint8_t* op_limit = op + maxout;
|
||||
uint32_t ctrl = (*ip++) & 31;
|
||||
|
||||
while (1) {
|
||||
if (ctrl >= 32) {
|
||||
uint32_t len = (ctrl >> 5) - 1;
|
||||
uint32_t ofs = (ctrl & 31) << 8;
|
||||
const uint8_t* ref = op - ofs - 1;
|
||||
|
||||
uint8_t code;
|
||||
if (len == 7 - 1) do {
|
||||
FASTLZ_BOUND_CHECK(ip <= ip_bound);
|
||||
code = *ip++;
|
||||
len += code;
|
||||
} while (code == 255);
|
||||
code = *ip++;
|
||||
ref -= code;
|
||||
len += 3;
|
||||
|
||||
/* match from 16-bit distance */
|
||||
if (FASTLZ_UNLIKELY(code == 255))
|
||||
if (FASTLZ_LIKELY(ofs == (31 << 8))) {
|
||||
FASTLZ_BOUND_CHECK(ip < ip_bound);
|
||||
ofs = (*ip++) << 8;
|
||||
ofs += *ip++;
|
||||
ref = op - ofs - MAX_L2_DISTANCE - 1;
|
||||
}
|
||||
|
||||
FASTLZ_BOUND_CHECK(op + len <= op_limit);
|
||||
FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output);
|
||||
fastlz_memmove(op, ref, len);
|
||||
op += len;
|
||||
} else {
|
||||
ctrl++;
|
||||
FASTLZ_BOUND_CHECK(op + ctrl <= op_limit);
|
||||
FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit);
|
||||
fastlz_memcpy(op, ip, ctrl);
|
||||
ip += ctrl;
|
||||
op += ctrl;
|
||||
}
|
||||
|
||||
if (FASTLZ_UNLIKELY(ip >= ip_limit)) break;
|
||||
ctrl = *ip++;
|
||||
}
|
||||
|
||||
return op - (uint8_t*)output;
|
||||
}
|
||||
|
||||
int fastlz_compress(const void* input, int length, void* output) {
|
||||
/* for short block, choose fastlz1 */
|
||||
if (length < 65536) return fastlz1_compress(input, length, output);
|
||||
|
||||
/* else... */
|
||||
return fastlz2_compress(input, length, output);
|
||||
}
|
||||
|
||||
int fastlz_decompress(const void* input, int length, void* output, int maxout) {
|
||||
/* magic identifier for compression level */
|
||||
int level = ((*(const uint8_t*)input) >> 5) + 1;
|
||||
|
||||
if (level == 1) return fastlz1_decompress(input, length, output, maxout);
|
||||
if (level == 2) return fastlz2_decompress(input, length, output, maxout);
|
||||
|
||||
/* unknown level, trigger error */
|
||||
return 0;
|
||||
}
|
||||
|
||||
int fastlz_compress_level(int level, const void* input, int length, void* output) {
|
||||
if (level == 1) return fastlz1_compress(input, length, output);
|
||||
if (level == 2) return fastlz2_compress(input, length, output);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
97
FastLZ/fastlz.h
Normal file
97
FastLZ/fastlz.h
Normal file
@ -0,0 +1,97 @@
|
||||
/*
|
||||
FastLZ - Byte-aligned LZ77 compression library
|
||||
Copyright (C) 2005-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef FASTLZ_H
|
||||
#define FASTLZ_H
|
||||
|
||||
#define FASTLZ_VERSION 0x000500
|
||||
|
||||
#define FASTLZ_VERSION_MAJOR 0
|
||||
#define FASTLZ_VERSION_MINOR 5
|
||||
#define FASTLZ_VERSION_REVISION 0
|
||||
|
||||
#define FASTLZ_VERSION_STRING "0.5.0"
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
Compress a block of data in the input buffer and returns the size of
|
||||
compressed block. The size of input buffer is specified by length. The
|
||||
minimum input buffer size is 16.
|
||||
|
||||
The output buffer must be at least 5% larger than the input buffer
|
||||
and can not be smaller than 66 bytes.
|
||||
|
||||
If the input is not compressible, the return value might be larger than
|
||||
length (input buffer size).
|
||||
|
||||
The input buffer and the output buffer can not overlap.
|
||||
|
||||
Compression level can be specified in parameter level. At the moment,
|
||||
only level 1 and level 2 are supported.
|
||||
Level 1 is the fastest compression and generally useful for short data.
|
||||
Level 2 is slightly slower but it gives better compression ratio.
|
||||
|
||||
Note that the compressed data, regardless of the level, can always be
|
||||
decompressed using the function fastlz_decompress below.
|
||||
*/
|
||||
|
||||
int fastlz_compress_level(int level, const void* input, int length, void* output);
|
||||
|
||||
/**
|
||||
Decompress a block of compressed data and returns the size of the
|
||||
decompressed block. If error occurs, e.g. the compressed data is
|
||||
corrupted or the output buffer is not large enough, then 0 (zero)
|
||||
will be returned instead.
|
||||
|
||||
The input buffer and the output buffer can not overlap.
|
||||
|
||||
Decompression is memory safe and guaranteed not to write the output buffer
|
||||
more than what is specified in maxout.
|
||||
|
||||
Note that the decompression will always work, regardless of the
|
||||
compression level specified in fastlz_compress_level above (when
|
||||
producing the compressed block).
|
||||
*/
|
||||
|
||||
int fastlz_decompress(const void* input, int length, void* output, int maxout);
|
||||
|
||||
/**
|
||||
DEPRECATED.
|
||||
|
||||
This is similar to fastlz_compress_level above, but with the level
|
||||
automatically chosen.
|
||||
|
||||
This function is deprecated and it will be completely removed in some future
|
||||
version.
|
||||
*/
|
||||
|
||||
int fastlz_compress(const void* input, int length, void* output);
|
||||
|
||||
#if defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FASTLZ_H */
|
||||
13
FastLZ/tests/Makefile
Normal file
13
FastLZ/tests/Makefile
Normal file
@ -0,0 +1,13 @@
|
||||
CFLAGS?=-Wall -std=c90
|
||||
TEST_ROUNDTRIP?=./test_roundtrip
|
||||
|
||||
all: roundtrip
|
||||
|
||||
roundtrip: test_roundtrip
|
||||
$(TEST_ROUNDTRIP)
|
||||
|
||||
test_roundtrip: test_roundtrip.c ../fastlz.c refimpl.c
|
||||
$(CC) -o $(TEST_ROUNDTRIP) $(CFLAGS) -I.. test_roundtrip.c ../fastlz.c refimpl.c
|
||||
|
||||
clean :
|
||||
$(RM) $(TEST_ROUNDTRIP) *.o
|
||||
14
FastLZ/tests/Makefile.win
Executable file
14
FastLZ/tests/Makefile.win
Executable file
@ -0,0 +1,14 @@
|
||||
CC=cl.exe
|
||||
CFLAGS=/Wall /Za
|
||||
RM=del
|
||||
|
||||
TEST_ROUNDTRIP=test_roundtrip.exe
|
||||
|
||||
all: roundtrip
|
||||
|
||||
roundtrip: test_roundtrip.c ../fastlz.c refimpl.c
|
||||
$(CC) -o $(TEST_ROUNDTRIP) $(CFLAGS) -I.. test_roundtrip.c ../fastlz.c refimpl.c
|
||||
$(TEST_ROUNDTRIP)
|
||||
|
||||
clean :
|
||||
$(RM) $(TEST_ROUNDTRIP) *.obj
|
||||
103
FastLZ/tests/refimpl.c
Normal file
103
FastLZ/tests/refimpl.c
Normal file
@ -0,0 +1,103 @@
|
||||
#include <stdint.h>
|
||||
|
||||
/*
|
||||
* Workaround for DJGPP to find uint8_t, uint16_t, etc.
|
||||
*/
|
||||
#if defined(__MSDOS__) && defined(__GNUC__)
|
||||
#include <stdint-gcc.h>
|
||||
#endif
|
||||
|
||||
void REF_Level1_decompress(const uint8_t* input, int length, uint8_t* output) {
|
||||
int src = 0;
|
||||
int dest = 0;
|
||||
while (src < length) {
|
||||
int type = input[src] >> 5;
|
||||
if (type == 0) {
|
||||
/* literal run */
|
||||
int run = 1 + input[src];
|
||||
src = src + 1;
|
||||
while (run > 0) {
|
||||
output[dest] = input[src];
|
||||
src = src + 1;
|
||||
dest = dest + 1;
|
||||
run = run - 1;
|
||||
}
|
||||
} else if (type < 7) {
|
||||
/* short match */
|
||||
int ofs = 256 * (input[src] & 31) + input[src + 1];
|
||||
int len = 2 + (input[src] >> 5);
|
||||
src = src + 2;
|
||||
int ref = dest - ofs - 1;
|
||||
while (len > 0) {
|
||||
output[dest] = output[ref];
|
||||
ref = ref + 1;
|
||||
dest = dest + 1;
|
||||
len = len - 1;
|
||||
}
|
||||
} else {
|
||||
/* long match */
|
||||
int ofs = 256 * (input[src] & 31) + input[src + 2];
|
||||
int len = 9 + input[src + 1];
|
||||
src = src + 3;
|
||||
int ref = dest - ofs - 1;
|
||||
while (len > 0) {
|
||||
output[dest] = output[ref];
|
||||
ref = ref + 1;
|
||||
dest = dest + 1;
|
||||
len = len - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void REF_Level2_decompress(const uint8_t* input, int length, uint8_t* output) {
|
||||
int src = 0;
|
||||
int dest = 0;
|
||||
while (src < length) {
|
||||
int type = input[src] >> 5;
|
||||
if (type == 0) {
|
||||
/* literal run */
|
||||
int run = 1 + input[src];
|
||||
src = src + 1;
|
||||
while (run > 0) {
|
||||
output[dest] = input[src];
|
||||
src = src + 1;
|
||||
dest = dest + 1;
|
||||
run = run - 1;
|
||||
}
|
||||
} else {
|
||||
int next = 2;
|
||||
int len = 2 + (input[src] >> 5);
|
||||
if (len == 9) {
|
||||
/* long match */
|
||||
next = next + 1;
|
||||
len = len + input[src + 1];
|
||||
if (len == 9 + 255) {
|
||||
/* Gamma code for match length */
|
||||
int nn = input[src + 1];
|
||||
while (nn == 255) {
|
||||
nn = input[src + next - 1];
|
||||
next = next + 1;
|
||||
len += nn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int ofs = 256 * (input[src] & 31) + input[src + next - 1];
|
||||
if (ofs == 8191) {
|
||||
/* match from 16-bit distance */
|
||||
ofs += 256 * input[src + next] + input[src + next + 1];
|
||||
next = next + 2;
|
||||
}
|
||||
src = src + next;
|
||||
|
||||
int ref = dest - ofs - 1;
|
||||
while (len > 0) {
|
||||
output[dest] = output[ref];
|
||||
ref = ref + 1;
|
||||
dest = dest + 1;
|
||||
len = len - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
451
FastLZ/tests/test_roundtrip.c
Normal file
451
FastLZ/tests/test_roundtrip.c
Normal file
@ -0,0 +1,451 @@
|
||||
/*
|
||||
FastLZ - Byte-aligned LZ77 compression library
|
||||
Copyright (C) 2005-2020 Ariya Hidayat <ariya.hidayat@gmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "fastlz.h"
|
||||
|
||||
/*
|
||||
* Workaround for DJGPP to find uint8_t, uint16_t, etc.
|
||||
*/
|
||||
#if defined(__MSDOS__) && defined(__GNUC__)
|
||||
#include <stdint-gcc.h>
|
||||
#endif
|
||||
|
||||
#define LOG
|
||||
#undef LOG
|
||||
|
||||
int compare(const char* name, const uint8_t* a, const uint8_t* b, int size) {
|
||||
int bad = 0;
|
||||
int i;
|
||||
for (i = 0; i < size; ++i) {
|
||||
if (a[i] != b[i]) {
|
||||
bad = 1;
|
||||
printf("Error on %s!\n", name);
|
||||
printf("Different at index %d: expecting %02x,actual %02x\n", i, a[i], b[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return bad;
|
||||
}
|
||||
|
||||
#if !defined(__MSDOS__)
|
||||
#define MAX_FILE_SIZE (100 * 1024 * 1024)
|
||||
#else
|
||||
#define MAX_FILE_SIZE (32 * 1024 * 1024)
|
||||
#endif
|
||||
|
||||
/* prototype, implemented in refimpl.c */
|
||||
void REF_Level1_decompress(const uint8_t* input, int length, uint8_t* output);
|
||||
void REF_Level2_decompress(const uint8_t* input, int length, uint8_t* output);
|
||||
|
||||
/*
|
||||
Same as test_roundtrip_level1 EXCEPT that the decompression is carried out
|
||||
using the highly-simplified, unoptimized vanilla reference decompressor.
|
||||
*/
|
||||
|
||||
void test_ref_decompressor_level1(const char* name, const char* file_name) {
|
||||
#ifdef LOG
|
||||
printf("Processing %s...\n", name);
|
||||
#endif
|
||||
FILE* f = fopen(file_name, "rb");
|
||||
if (!f) {
|
||||
printf("Error: can not open %s!\n", file_name);
|
||||
exit(1);
|
||||
}
|
||||
fseek(f, 0L, SEEK_END);
|
||||
long file_size = ftell(f);
|
||||
rewind(f);
|
||||
|
||||
#ifdef LOG
|
||||
printf("Size is %ld bytes.\n", file_size);
|
||||
#endif
|
||||
if (file_size > MAX_FILE_SIZE) {
|
||||
fclose(f);
|
||||
printf("%25s %10ld [skipped, file too big]\n", name, file_size);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t* file_buffer = malloc(file_size);
|
||||
long read = fread(file_buffer, 1, file_size, f);
|
||||
fclose(f);
|
||||
if (read != file_size) {
|
||||
free(file_buffer);
|
||||
printf("Error: only read %ld bytes!\n", read);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef LOG
|
||||
printf("Compressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* compressed_buffer = malloc(1.05 * file_size);
|
||||
int compressed_size = fastlz_compress_level(1, file_buffer, file_size, compressed_buffer);
|
||||
double ratio = (100.0 * compressed_size) / file_size;
|
||||
#ifdef LOG
|
||||
printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio);
|
||||
#endif
|
||||
|
||||
#ifdef LOG
|
||||
printf("Decompressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* uncompressed_buffer = malloc(file_size);
|
||||
if (uncompressed_buffer == NULL) {
|
||||
printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio);
|
||||
return;
|
||||
}
|
||||
memset(uncompressed_buffer, '-', file_size);
|
||||
REF_Level1_decompress(compressed_buffer, compressed_size, uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("Comparing. Please wait...\n");
|
||||
#endif
|
||||
int result = compare(file_name, file_buffer, uncompressed_buffer, file_size);
|
||||
if (result == 1) {
|
||||
free(uncompressed_buffer);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
free(file_buffer);
|
||||
free(compressed_buffer);
|
||||
free(uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("OK.\n");
|
||||
#else
|
||||
printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
Same as test_roundtrip_level2 EXCEPT that the decompression is carried out
|
||||
using the highly-simplified, unoptimized vanilla reference decompressor.
|
||||
*/
|
||||
|
||||
void test_ref_decompressor_level2(const char* name, const char* file_name) {
|
||||
#ifdef LOG
|
||||
printf("Processing %s...\n", name);
|
||||
#endif
|
||||
FILE* f = fopen(file_name, "rb");
|
||||
if (!f) {
|
||||
printf("Error: can not open %s!\n", file_name);
|
||||
exit(1);
|
||||
}
|
||||
fseek(f, 0L, SEEK_END);
|
||||
long file_size = ftell(f);
|
||||
rewind(f);
|
||||
|
||||
#ifdef LOG
|
||||
printf("Size is %ld bytes.\n", file_size);
|
||||
#endif
|
||||
if (file_size > MAX_FILE_SIZE) {
|
||||
fclose(f);
|
||||
printf("%25s %10ld [skipped, file too big]\n", name, file_size);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t* file_buffer = malloc(file_size);
|
||||
long read = fread(file_buffer, 1, file_size, f);
|
||||
fclose(f);
|
||||
if (read != file_size) {
|
||||
free(file_buffer);
|
||||
printf("Error: only read %ld bytes!\n", read);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef LOG
|
||||
printf("Compressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* compressed_buffer = malloc(1.05 * file_size);
|
||||
int compressed_size = fastlz_compress_level(2, file_buffer, file_size, compressed_buffer);
|
||||
double ratio = (100.0 * compressed_size) / file_size;
|
||||
#ifdef LOG
|
||||
printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio);
|
||||
#endif
|
||||
|
||||
#ifdef LOG
|
||||
printf("Decompressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* uncompressed_buffer = malloc(file_size);
|
||||
if (uncompressed_buffer == NULL) {
|
||||
printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio);
|
||||
return;
|
||||
}
|
||||
memset(uncompressed_buffer, '-', file_size);
|
||||
|
||||
/* intentionally mask out the block tag */
|
||||
compressed_buffer[0] = compressed_buffer[0] & 31;
|
||||
|
||||
REF_Level2_decompress(compressed_buffer, compressed_size, uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("Comparing. Please wait...\n");
|
||||
#endif
|
||||
int result = compare(file_name, file_buffer, uncompressed_buffer, file_size);
|
||||
if (result == 1) {
|
||||
free(uncompressed_buffer);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
free(file_buffer);
|
||||
free(compressed_buffer);
|
||||
free(uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("OK.\n");
|
||||
#else
|
||||
printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
Read the content of the file.
|
||||
Compress it first using the Level 1 compressor.
|
||||
Decompress the output with Level 1 decompressor.
|
||||
Compare the result with the original file content.
|
||||
*/
|
||||
void test_roundtrip_level1(const char* name, const char* file_name) {
|
||||
#ifdef LOG
|
||||
printf("Processing %s...\n", name);
|
||||
#endif
|
||||
FILE* f = fopen(file_name, "rb");
|
||||
if (!f) {
|
||||
printf("Error: can not open %s!\n", file_name);
|
||||
exit(1);
|
||||
}
|
||||
fseek(f, 0L, SEEK_END);
|
||||
long file_size = ftell(f);
|
||||
rewind(f);
|
||||
|
||||
#ifdef LOG
|
||||
printf("Size is %ld bytes.\n", file_size);
|
||||
#endif
|
||||
if (file_size > MAX_FILE_SIZE) {
|
||||
fclose(f);
|
||||
printf("%25s %10ld [skipped, file too big]\n", name, file_size);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t* file_buffer = malloc(file_size);
|
||||
long read = fread(file_buffer, 1, file_size, f);
|
||||
fclose(f);
|
||||
if (read != file_size) {
|
||||
free(file_buffer);
|
||||
printf("Error: only read %ld bytes!\n", read);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef LOG
|
||||
printf("Compressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* compressed_buffer = malloc(1.05 * file_size);
|
||||
int compressed_size = fastlz_compress_level(1, file_buffer, file_size, compressed_buffer);
|
||||
double ratio = (100.0 * compressed_size) / file_size;
|
||||
#ifdef LOG
|
||||
printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio);
|
||||
#endif
|
||||
|
||||
#ifdef LOG
|
||||
printf("Decompressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* uncompressed_buffer = malloc(file_size);
|
||||
if (uncompressed_buffer == NULL) {
|
||||
printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio);
|
||||
return;
|
||||
}
|
||||
memset(uncompressed_buffer, '-', file_size);
|
||||
fastlz_decompress(compressed_buffer, compressed_size, uncompressed_buffer, file_size);
|
||||
#ifdef LOG
|
||||
printf("Comparing. Please wait...\n");
|
||||
#endif
|
||||
int result = compare(file_name, file_buffer, uncompressed_buffer, file_size);
|
||||
if (result == 1) {
|
||||
free(uncompressed_buffer);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
free(file_buffer);
|
||||
free(compressed_buffer);
|
||||
free(uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("OK.\n");
|
||||
#else
|
||||
printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
Read the content of the file.
|
||||
Compress it first using the Level 2 compressor.
|
||||
Decompress the output with Level 2 decompressor.
|
||||
Compare the result with the original file content.
|
||||
*/
|
||||
void test_roundtrip_level2(const char* name, const char* file_name) {
|
||||
#ifdef LOG
|
||||
printf("Processing %s...\n", name);
|
||||
#endif
|
||||
FILE* f = fopen(file_name, "rb");
|
||||
if (!f) {
|
||||
printf("Error: can not open %s!\n", file_name);
|
||||
exit(1);
|
||||
}
|
||||
fseek(f, 0L, SEEK_END);
|
||||
long file_size = ftell(f);
|
||||
rewind(f);
|
||||
|
||||
#ifdef LOG
|
||||
printf("Size is %ld bytes.\n", file_size);
|
||||
#endif
|
||||
if (file_size > MAX_FILE_SIZE) {
|
||||
fclose(f);
|
||||
printf("%25s %10ld [skipped, file too big]\n", name, file_size);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t* file_buffer = malloc(file_size);
|
||||
long read = fread(file_buffer, 1, file_size, f);
|
||||
fclose(f);
|
||||
if (read != file_size) {
|
||||
free(file_buffer);
|
||||
printf("Error: only read %ld bytes!\n", read);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef LOG
|
||||
printf("Compressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* compressed_buffer = malloc(1.05 * file_size);
|
||||
int compressed_size = fastlz_compress_level(2, file_buffer, file_size, compressed_buffer);
|
||||
double ratio = (100.0 * compressed_size) / file_size;
|
||||
#ifdef LOG
|
||||
printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio);
|
||||
#endif
|
||||
|
||||
#ifdef LOG
|
||||
printf("Decompressing. Please wait...\n");
|
||||
#endif
|
||||
uint8_t* uncompressed_buffer = malloc(file_size);
|
||||
if (uncompressed_buffer == NULL) {
|
||||
free(file_buffer);
|
||||
free(compressed_buffer);
|
||||
printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress OOM\n", name, file_size, compressed_size, ratio);
|
||||
exit(1);
|
||||
return;
|
||||
}
|
||||
memset(uncompressed_buffer, '-', file_size);
|
||||
fastlz_decompress(compressed_buffer, compressed_size, uncompressed_buffer, file_size);
|
||||
#ifdef LOG
|
||||
printf("Comparing. Please wait...\n");
|
||||
#endif
|
||||
int result = compare(file_name, file_buffer, uncompressed_buffer, file_size);
|
||||
if (result == 1) {
|
||||
free(uncompressed_buffer);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
free(file_buffer);
|
||||
free(compressed_buffer);
|
||||
free(uncompressed_buffer);
|
||||
#ifdef LOG
|
||||
printf("OK.\n");
|
||||
#else
|
||||
printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const char* default_prefix = "../compression-corpus/";
|
||||
const char* names[] = {"canterbury/alice29.txt",
|
||||
"canterbury/asyoulik.txt",
|
||||
"canterbury/cp.html",
|
||||
"canterbury/fields.c",
|
||||
"canterbury/grammar.lsp",
|
||||
"canterbury/kennedy.xls",
|
||||
"canterbury/lcet10.txt",
|
||||
"canterbury/plrabn12.txt",
|
||||
"canterbury/ptt5",
|
||||
"canterbury/sum",
|
||||
"canterbury/xargs.1",
|
||||
"silesia/dickens",
|
||||
"silesia/mozilla",
|
||||
"silesia/mr",
|
||||
"silesia/nci",
|
||||
"silesia/ooffice",
|
||||
"silesia/osdb",
|
||||
"silesia/reymont",
|
||||
"silesia/samba",
|
||||
"silesia/sao",
|
||||
"silesia/webster",
|
||||
"silesia/x-ray",
|
||||
"silesia/xml",
|
||||
"enwik/enwik8.txt"};
|
||||
|
||||
const char* prefix = (argc == 2) ? argv[1] : default_prefix;
|
||||
|
||||
const int count = sizeof(names) / sizeof(names[0]);
|
||||
int i;
|
||||
|
||||
printf("Test reference decompressor for Level 1\n\n");
|
||||
for (i = 0; i < count; ++i) {
|
||||
const char* name = names[i];
|
||||
char* filename = malloc(strlen(prefix) + strlen(name) + 1);
|
||||
strcpy(filename, prefix);
|
||||
strcat(filename, name);
|
||||
test_ref_decompressor_level1(name, filename);
|
||||
free(filename);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
printf("Test reference decompressor for Level 2\n\n");
|
||||
for (i = 0; i < count; ++i) {
|
||||
const char* name = names[i];
|
||||
char* filename = malloc(strlen(prefix) + strlen(name) + 1);
|
||||
strcpy(filename, prefix);
|
||||
strcat(filename, name);
|
||||
test_ref_decompressor_level2(name, filename);
|
||||
free(filename);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
printf("Test round-trip for Level 1\n\n");
|
||||
for (i = 0; i < count; ++i) {
|
||||
const char* name = names[i];
|
||||
char* filename = malloc(strlen(prefix) + strlen(name) + 1);
|
||||
strcpy(filename, prefix);
|
||||
strcat(filename, name);
|
||||
test_roundtrip_level1(name, filename);
|
||||
free(filename);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
printf("Test round-trip for Level 2\n\n");
|
||||
for (i = 0; i < count; ++i) {
|
||||
const char* name = names[i];
|
||||
char* filename = malloc(strlen(prefix) + strlen(name) + 1);
|
||||
strcpy(filename, prefix);
|
||||
strcat(filename, name);
|
||||
test_roundtrip_level2(name, filename);
|
||||
free(filename);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
4
FastLZ/tools/format-code.sh
Executable file
4
FastLZ/tools/format-code.sh
Executable file
@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
cwd=$(pwd)
|
||||
clang-format-6.0 -i --style='{BasedOnStyle: "google", ColumnLimit: 120}' $cwd/*.h $cwd/*.c $cwd/tests/*.c $cwd/examples/*.c
|
||||
7
Makefile
7
Makefile
@ -29,6 +29,11 @@ prepare:
|
||||
$(PWD)/scripts/build-gcc.sh; \
|
||||
cd ..; \
|
||||
fi
|
||||
if [ ! -e FastLZ/examples/6pack ] || [ ! -e FastLZ/examples/6unpack ]; then \
|
||||
cd FastLZ/examples; \
|
||||
make; \
|
||||
cd ..; \
|
||||
fi
|
||||
|
||||
cleanall:
|
||||
make clean
|
||||
@ -38,7 +43,7 @@ clean:
|
||||
make -C kernel ARCH=$(ARCH) ROOT=$(PWD) clean
|
||||
make -C user ARCH=$(ARCH) ROOT=$(PWD) clean
|
||||
make -C ulib ARCH=$(ARCH) ROOT=$(PWD) clean
|
||||
rm -f mop2.iso base.img disk.hdd
|
||||
rm -f mop2.iso base.img disk.hdd base.img.lz77
|
||||
|
||||
base:
|
||||
./scripts/mkbaseimg.sh
|
||||
|
||||
115
kernel/FastLZ/6unpack_mem.c
Normal file
115
kernel/FastLZ/6unpack_mem.c
Normal file
@ -0,0 +1,115 @@
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include "hal/hal.h"
|
||||
#include "fastlz.h"
|
||||
|
||||
#define SIXPACK_OK 0
|
||||
#define SIXPACK_ERR_MAGIC -1
|
||||
#define SIXPACK_ERR_CHECKSUM -2
|
||||
#define SIXPACK_ERR_DECOMPRESS -3
|
||||
#define SIXPACK_ERR_BAD_FORMAT -4
|
||||
#define SIXPACK_ERR_NO_SPACE -5
|
||||
|
||||
#define ADLER32_BASE 65521
|
||||
#define BLOCK_SIZE 65536
|
||||
|
||||
static const uint8_t sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10};
|
||||
|
||||
static uint32_t update_adler32(uint32_t checksum, const void *buf, int len) {
|
||||
const uint8_t *ptr = (const uint8_t*)buf;
|
||||
uint32_t s1 = checksum & 0xffff;
|
||||
uint32_t s2 = (checksum >> 16) & 0xffff;
|
||||
|
||||
while (len > 0) {
|
||||
unsigned k = len < 5552 ? len : 5552;
|
||||
len -= k;
|
||||
while (k--) {
|
||||
s1 += *ptr++;
|
||||
if (s1 >= ADLER32_BASE) s1 -= ADLER32_BASE;
|
||||
s2 += s1;
|
||||
if (s2 >= ADLER32_BASE) s2 -= ADLER32_BASE;
|
||||
}
|
||||
}
|
||||
return (s2 << 16) + s1;
|
||||
}
|
||||
|
||||
static uint32_t readU16(const uint8_t *p) {
|
||||
return (uint32_t)p[0] | ((uint32_t)p[1] << 8);
|
||||
}
|
||||
static uint32_t readU32(const uint8_t *p) {
|
||||
return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
|
||||
}
|
||||
|
||||
static void read_chunk_header(const uint8_t *p, int *id, int *opts,
|
||||
uint32_t *size, uint32_t *checksum, uint32_t *extra) {
|
||||
*id = (int)readU16(p);
|
||||
*opts = (int)readU16(p + 2);
|
||||
*size = readU32(p + 4);
|
||||
*checksum = readU32(p + 8);
|
||||
*extra = readU32(p + 12);
|
||||
}
|
||||
|
||||
int sixpack_decompress_mem(const uint8_t *input, size_t in_size,
|
||||
uint8_t *output, size_t out_cap)
|
||||
{
|
||||
if (!input || in_size < 8)
|
||||
return SIXPACK_ERR_BAD_FORMAT;
|
||||
|
||||
/* Check magic */
|
||||
for (int i = 0; i < 8; i++)
|
||||
if (input[i] != sixpack_magic[i])
|
||||
return SIXPACK_ERR_MAGIC;
|
||||
|
||||
size_t pos = 8; /* skip magic */
|
||||
size_t written = 0;
|
||||
|
||||
while (pos + 16 <= in_size) {
|
||||
int chunk_id, opts;
|
||||
uint32_t size, checksum, extra;
|
||||
|
||||
read_chunk_header(input + pos, &chunk_id, &opts, &size, &checksum, &extra);
|
||||
pos += 16;
|
||||
|
||||
if (pos + size > in_size)
|
||||
return SIXPACK_ERR_BAD_FORMAT;
|
||||
|
||||
const uint8_t *chunk_data = input + pos;
|
||||
pos += size;
|
||||
|
||||
/* File header chunk */
|
||||
if (chunk_id == 1) {
|
||||
uint32_t decomp_size = readU32(chunk_data);
|
||||
(void)decomp_size; /* not strictly needed here */
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Data chunk */
|
||||
if (chunk_id == 17) {
|
||||
uint32_t cksum = update_adler32(1L, chunk_data, size);
|
||||
if (cksum != checksum)
|
||||
return SIXPACK_ERR_CHECKSUM;
|
||||
|
||||
if (opts == 0) {
|
||||
/* Stored (uncompressed) */
|
||||
if (written + size > out_cap)
|
||||
return SIXPACK_ERR_NO_SPACE;
|
||||
hal_memcpy(output + written, chunk_data, size);
|
||||
written += size;
|
||||
} else if (opts == 1) {
|
||||
/* FastLZ compressed */
|
||||
if (written + extra > out_cap)
|
||||
return SIXPACK_ERR_NO_SPACE;
|
||||
|
||||
int dec = fastlz_decompress(chunk_data, size, output + written, extra);
|
||||
if (dec != (int)extra)
|
||||
return SIXPACK_ERR_DECOMPRESS;
|
||||
|
||||
written += extra;
|
||||
} else {
|
||||
return SIXPACK_ERR_BAD_FORMAT;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (int)written;
|
||||
}
|
||||
7
kernel/FastLZ/6unpack_mem.h
Normal file
7
kernel/FastLZ/6unpack_mem.h
Normal file
@ -0,0 +1,7 @@
|
||||
#ifndef SIXUNPACK_MEM_H_
|
||||
#define SIXUNPACK_MEM_H_
|
||||
|
||||
int sixpack_decompress_mem(const uint8_t *input, size_t in_size,
|
||||
uint8_t *output, size_t out_cap);
|
||||
|
||||
#endif // SIXUNPACK_MEM_H_
|
||||
1
kernel/FastLZ/fastlz.c
Symbolic link
1
kernel/FastLZ/fastlz.c
Symbolic link
@ -0,0 +1 @@
|
||||
../../FastLZ/fastlz.c
|
||||
1
kernel/FastLZ/fastlz.h
Symbolic link
1
kernel/FastLZ/fastlz.h
Symbolic link
@ -0,0 +1 @@
|
||||
../../FastLZ/fastlz.h
|
||||
@ -20,6 +20,7 @@ CFLAGS += -I. \
|
||||
-DLFS_NO_DEBUG \
|
||||
-DLFS_NO_WARN \
|
||||
-DLFS_NO_ERROR \
|
||||
-DFASTLZ_USE_MEMMOVE=0 \
|
||||
|
||||
include arch/$(ARCH)/$(ARCH).mk
|
||||
include extconf/extra.mk
|
||||
@ -58,6 +59,7 @@ SRCFILES += $(call GRABSRC, \
|
||||
randcrypto \
|
||||
time \
|
||||
diskpart \
|
||||
FastLZ \
|
||||
)
|
||||
|
||||
CFILES := $(call GET_CFILES, $(SRCFILES))
|
||||
|
||||
@ -6,32 +6,40 @@
|
||||
#include "kprintf.h"
|
||||
#include "util/util.h"
|
||||
#include "hal/hal.h"
|
||||
#include "dlmalloc/malloc.h"
|
||||
#include "FastLZ/fastlz.h"
|
||||
#include "FastLZ/6unpack_mem.h"
|
||||
|
||||
struct limine_file *baseimg = NULL;
|
||||
#define BASEIMG_DECOMPRESSED (1024*1024*4)
|
||||
|
||||
size_t BASEIMG_DECOMP_SIZE;
|
||||
uint8_t *BASEIMG_DECOMP_ADDR;
|
||||
|
||||
uint64_t baseimg_getaddr(void) {
|
||||
return (uint64_t)baseimg->address;
|
||||
return (uint64_t)BASEIMG_DECOMP_ADDR;
|
||||
}
|
||||
|
||||
uint64_t baseimg_getsize(void) {
|
||||
return baseimg->size;
|
||||
return (uint64_t)BASEIMG_DECOMP_SIZE;
|
||||
}
|
||||
|
||||
void baseimg_init(void) {
|
||||
struct limine_file *baseimg = NULL;
|
||||
|
||||
LOG("baseimg", "looking for base image...\n");
|
||||
for (size_t i = 0; i < BOOT_INFO.modules->module_count; i++) {
|
||||
struct limine_file *module = BOOT_INFO.modules->modules[i];
|
||||
if (hal_strcmp(util_get_filename(module->path), "base.img") == 0) {
|
||||
if (hal_strcmp(util_get_filename(module->path), "base.img.6pack") == 0) {
|
||||
baseimg = module;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (baseimg == NULL) {
|
||||
ERR("baseimg", "base.img not found\n");
|
||||
ERR("baseimg", "base.img.6pack not found\n");
|
||||
hal_hang();
|
||||
} else {
|
||||
LOG("baseimg", "base.img found\n");
|
||||
LOG("baseimg", "base.img.6pack found\n");
|
||||
LOG("baseimg", "addr = %p, size = %lu\n", baseimg->address, baseimg->size);
|
||||
for (size_t i = 0; i < 30; i++) {
|
||||
kprintf("%02X ", ((uint8_t *)(baseimg->address))[i]);
|
||||
@ -40,4 +48,14 @@ void baseimg_init(void) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BASEIMG_DECOMP_ADDR = dlmalloc(BASEIMG_DECOMPRESSED);
|
||||
int res = sixpack_decompress_mem(baseimg->address, baseimg->size,
|
||||
BASEIMG_DECOMP_ADDR, BASEIMG_DECOMPRESSED);
|
||||
if (res < 0) {
|
||||
ERR("baseimg", "could not uncompress base.img.6pack\n");
|
||||
hal_hang();
|
||||
}
|
||||
BASEIMG_DECOMP_SIZE = res;
|
||||
kprintf("%p %zu\n", BASEIMG_DECOMP_ADDR, BASEIMG_DECOMP_SIZE);
|
||||
}
|
||||
|
||||
@ -4,4 +4,4 @@ verbose: yes
|
||||
/mop2
|
||||
protocol: limine
|
||||
path: boot():/boot/mop2
|
||||
module_path: boot():/base.img
|
||||
module_path: boot():/base.img.6pack
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
rm -f base.img
|
||||
rm -f base.img base.img.6pack
|
||||
cat user/FILES.txt | while read line; do cp -v $line base/bin; done
|
||||
mklittlefs -c base -b 512 -s $((1<<20)) base.img
|
||||
./FastLZ/examples/6pack -2 base.img base.img.6pack
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
rm -rf iso_root
|
||||
mkdir -p iso_root/boot
|
||||
cp -v kernel/mop2 iso_root/boot
|
||||
cp -v base.img iso_root
|
||||
cp -v base.img.6pack iso_root
|
||||
mkdir -p iso_root/boot/limine
|
||||
cp -v limine.conf limine/limine-bios.sys limine/limine-bios-cd.bin \
|
||||
limine/limine-uefi-cd.bin iso_root/boot/limine
|
||||
|
||||
Reference in New Issue
Block a user