Miroirs/Conty
1
0
Fork 0
mirror of https://github.com/Kron4ek/Conty synced 2024-11-16 19:50:06 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement the self-update function for dwarfs

Browse source 
And some other improvements.
This commit is contained in:
Kron4ek 2021-08-10 23:57:36 +05:00
parent 9e366b00fe
commit 268ff808ba
4 changed files with 119 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
conty-start.sh
View file

@ -43,11 +43,11 @@ mount_point="${working_dir}"/mnt
# a problem with mounting the image due to an incorrectly calculated offset.
# The size of this script
scriptsize=23232
scriptsize=24850
# The size of the utils.tar.gz archive
# utils.tar.gz contains bwrap, squashfuse and dwarfs binaries
utilssize=7418944
utilssize=9144313
# Offset where the image is stored
offset=$((scriptsize+utilssize))
@ -58,6 +58,11 @@ dwarfs_image=0
dwarfs_cache_size="128M"
dwarfs_num_workers="2"
# These arguments are used to rebuild the image when using the self-update function
squashfs_comp_arguments="-b 256K -comp zstd -Xcompression-level 14"
dwarfs_comp_arguments="-l7 -C zstd:level=19 --metadata-compression null \
-S 22 -B 3"
if [ "$1" = "--help" ] || [ "$1" = "-h" ] || ([ -z "$1" ] && [ ! -L "${script_literal}" ]); then
echo "Usage: ./conty.sh command command_arguments"
echo
@ -150,7 +155,7 @@ elif [ "$1" = "-e" ]; then
exit
elif [ "$1" = "-o" ]; then
echo $offset
echo ${offset}
exit
fi
@ -183,10 +188,16 @@ launch_wrapper () {
fi
}
# Disable the regular mount command when using a dwarfs-compressed image
# because Linux kernel doesn't support dwarfs directly, only via FUSE
if [ "${dwarfs_image}" = 1 ]; then
unset SUDO_MOUNT
fi
# Check if FUSE is installed when SUDO_MOUNT is not enabled
if [ "${SUDO_MOUNT}" != 1 ] || [ "${dwarfs_image}" = 1 ]; then
if [ "${SUDO_MOUNT}" != 1 ]; then
if ! command -v fusermount3 1>/dev/null && ! command -v fusermount 1>/dev/null; then
echo "Please install fuse2 or fuse3 and run the script again!"
echo "Please install fuse2 or fuse3 and run the script again."
exit 1
fi
@ -200,7 +211,7 @@ fi
if [ "${dwarfs_image}" = 1 ]; then
if getconf _PHYS_PAGES &>/dev/null && getconf PAGE_SIZE &>/dev/null; then
memory_size="$(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))"
if [ "${memory_size}" -ge 23000 ]; then
dwarfs_cache_size="2048M"
elif [ "${memory_size}" -ge 15000 ]; then
@ -218,7 +229,7 @@ if [ "${dwarfs_image}" = 1 ]; then
if getconf _NPROCESSORS_ONLN &>/dev/null; then
dwarfs_num_workers="$(getconf _NPROCESSORS_ONLN)"
if [ "${dwarfs_num_workers}" -ge 16 ]; then
dwarfs_num_workers=16
fi
@ -243,7 +254,12 @@ if [ "${USE_SYS_UTILS}" != 1 ]; then
exit 1
fi
fi
if ! command -v tar 1>/dev/null || ! command -v gzip 1>/dev/null; then
echo "Please install tar and gzip and run the script again."
exit 1
fi
if [ "${dwarfs_image}" = 1 ]; then
mount_tool="${working_dir}"/utils/dwarfs${fuse_version}
else
@ -253,11 +269,11 @@ if [ "${USE_SYS_UTILS}" != 1 ]; then
bwrap="${working_dir}"/utils/bwrap
if [ ! -f "${mount_tool}" ] || [ ! -f "${bwrap}" ]; then
tail -c +$((scriptsize+1)) "${script}" | head -c $utilssize | tar -C "${working_dir}" -zxf -
tail -c +$((scriptsize+1)) "${script}" | head -c ${utilssize} | tar -C "${working_dir}" -zxf -
if [ ! -f "${mount_tool}" ] || [ ! -f "${bwrap}" ]; then
clear
echo "The utilities were not extracted!"
echo "The integrated utils were not extracted!"
echo "Perhaps something is wrong with the integrated utils.tar.gz."
exit 1
@ -268,23 +284,27 @@ if [ "${USE_SYS_UTILS}" != 1 ]; then
fi
else
if ! command -v bwrap 1>/dev/null; then
echo "USE_SYS_UTILS is enabled, but bwrap is not installed!"
echo "USE_SYS_UTILS is enabled, but bubblewrap is not installed!"
echo "Please install it and run the script again."
exit 1
fi
bwrap=bwrap
if [ "${dwarfs_image}" = 1 ]; then
if ! command -v dwarfs 1>/dev/null; then
if ! command -v dwarfs 1>/dev/null && ! command -v dwarfs2 1>/dev/null; then
echo "USE_SYS_UTILS is enabled, but dwarfs is not installed!"
echo "Please install it and run the script again."
exit 1
fi
mount_tool=dwarfs
if command -v dwarfs2 1>/dev/null; then
mount_tool=dwarfs2
else
mount_tool=dwarfs
fi
else
if ! command -v squashfuse 1>/dev/null && [ "${SUDO_MOUNT}" != 1 ]; then
echo "USE_SYS_UTILS is enabled, but squshfuse is not installed!"
@ -294,14 +314,14 @@ else
exit 1
fi
mount_tool=squashfuse
fi
show_msg "Using system-wide ${mount_tool} and bwrap"
fi
if [ "${SUDO_MOUNT}" = 1 ] && [ "${dwarfs_image}" != 1 ]; then
if [ "${SUDO_MOUNT}" = 1 ]; then
show_msg "Using regular mount command (sudo mount) instead of squashfuse"
mount_tool=mount
@ -309,14 +329,9 @@ if [ "${SUDO_MOUNT}" = 1 ] && [ "${dwarfs_image}" != 1 ]; then
fi
if [ "$1" = "-u" ] || [ "$1" = "-U" ]; then
if [ "${dwarfs_image}" = 1 ]; then
echo "This feature is currently unimplemented for dwarfs-compressed images."
exit 1
fi
OLD_PWD="${PWD}"
# Check if current directory is writable
# Check if the current directory is writable
# And if it's not, use ~/.local/share/Conty as a working directory
if ! touch test_rw 2>/dev/null; then
update_temp_dir="${HOME}"/.local/share/Conty/conty_update_temp
@ -332,6 +347,34 @@ if [ "$1" = "-u" ] || [ "$1" = "-U" ]; then
mkdir -p "${update_temp_dir}"
cd "${update_temp_dir}" || exit 1
tail -c +$((scriptsize+1)) "${script}" | head -c ${utilssize} | tar -C "${update_temp_dir}" -zxf -
if [ "${dwarfs_image}" = 1 ]; then
chmod +x utils/dwarfsextract 2>/dev/null
chmod +x utils/mkdwarfs 2>/dev/null
if [ ! -x "utils/dwarfsextract" ] || [ ! -x "utils/mkdwarfs" ]; then
missing_utils="dwarfsextract and/or mkdwarfs"
fi
else
chmod +x utils/unsquashfs 2>/dev/null
chmod +x utils/mksquashfs 2>/dev/null
if [ ! -x "utils/unsquashfs" ] || [ ! -x "utils/mksquashfs" ]; then
missing_utils="unsquashfs and/or mksquashfs"
fi
fi
if [ -n "${missing_utils}" ]; then
echo "The integrated utils don't contain ${missing_utils}."
echo "Or your file system is mounted with noexec."
exit 1
fi
tools_wrapper () {
"${update_temp_dir}"/utils/ld-linux-x86-64.so.2 --library-path "${update_temp_dir}"/utils "$@"
}
# Since Conty is used here to update itself, it's necessary to disable
# some environment variables for this to work properly
unset NVIDIA_FIX
@ -342,12 +385,20 @@ if [ "$1" = "-u" ] || [ "$1" = "-U" ]; then
# Enable SANDBOX
export SANDBOX=1
# Extract the squashfs image
export QUIET_MODE=1
# Extract the image
clear
echo "Extracting the squashfs image"
bash "${script}" --bind "${update_temp_dir}" "${update_temp_dir}" \
--bind "${script}" /tmp/conty.sh \
unsquashfs -o $offset -user-xattrs -d sqfs /tmp/conty.sh
echo "Extracting the image"
if [ "${dwarfs_image}" = 1 ]; then
mkdir sqfs
tools_wrapper "${update_temp_dir}"/utils/dwarfsextract \
-i "${script}" -o sqfs -O ${offset} --cache-size "${dwarfs_cache_size}" \
--num-workers "${dwarfs_num_workers}"
else
tools_wrapper "${update_temp_dir}"/utils/unsquashfs \
-o ${offset} -user-xattrs -d sqfs "${script}"
fi
# Download or extract the utils.tar.gz and the init script depending
# on what command line argument is used (-u or -U)
@ -417,11 +468,16 @@ EOF
--proc /proc --bind "${update_temp_dir}" "${update_temp_dir}" \
bash container-update.sh
# Create a squashfs image
# Create an image
clear
echo "Creating a squashfs image"
bash "${script}" --bind "${update_temp_dir}" "${update_temp_dir}" \
mksquashfs sqfs image -b 256K -comp zstd -Xcompression-level 14
echo "Creating an image"
if [ "${dwarfs_image}" = 1 ]; then
tools_wrapper "${update_temp_dir}"/utils/mkdwarfs \
-i sqfs -o image ${dwarfs_comp_arguments}
else
tools_wrapper "${update_temp_dir}"/utils/mksquashfs \
sqfs image ${squashfs_comp_arguments}
fi
# Combine into a single executable
clear
@ -655,14 +711,14 @@ if [ "$(ls "${working_dir}"/running_* 2>/dev/null)" ] && [ ! "$(ls "${mount_poin
rm -f "${working_dir}"/running_*
fi
# Mount the squashfs image
# Mount the image
mkdir -p "${mount_point}"
if [ "$(ls "${mount_point}" 2>/dev/null)" ] || \
( [ "${dwarfs_image}" != 1 ] && launch_wrapper "${mount_tool}" -o offset="${offset}",ro "${script}" "${mount_point}" ) || \
launch_wrapper "${mount_tool}" "${script}" "${mount_point}" -o offset="${offset}" -o debuglevel=error -o workers="${dwarfs_num_workers}" \
-o mlock=try -o no_cache_image -o cache_files -o cachesize="${dwarfs_cache_size}"; then
if [ "$1" = "-m" ]; then
if [ ! -f "${working_dir}"/running_mount ]; then
echo 1 > "${working_dir}"/running_mount
@ -671,12 +727,12 @@ if [ "$(ls "${mount_point}" 2>/dev/null)" ] || \
rm -f "${working_dir}"/running_mount
echo "The image has been unmounted"
fi
exit
fi
echo 1 > "${working_dir}"/running_"${script_id}"
show_msg "Running Conty"
if [ "${NVIDIA_FIX}" = 1 ]; then

6
create-conty.sh
View file

@ -13,7 +13,7 @@ squashfs_compressor_arguments="-b 256K -comp ${squashfs_compressor} -Xhc"
# Use dwarfs instead of squashfs
dwarfs="false"
dwarfs_compressor_arguments="-l7 -C zstd:level=19 --metadata-compression null \
-S 22 -B 3 --bloom-filter-size 5"
-S 22 -B 3"
# Set to true to use an existing image if it exists
# Otherwise the script will always create a new image
@ -70,14 +70,14 @@ if [ ! -f "${image_path}" ] || [ "${use_existing_image}" != "true" ]; then
echo "Please install dwarfs and run the script again"
exit 1
fi
mkdwarfs -i "${bootstrap}" -o "${image_path}" ${dwarfs_compressor_arguments}
else
if ! command -v mksquashfs 1>/dev/null; then
echo "Please install squashfs-tools and run the script again"
exit 1
fi
mksquashfs "${bootstrap}" "${image_path}" ${squashfs_compressor_arguments}
fi
fi

25
create-utils.sh
View file

@ -1,9 +1,11 @@
#!/usr/bin/env bash
# Dependencies: gawk grep lz4 zstd wget gcc make autoconf libtool pkgconf libcap fuse2 (or fuse3)
# Dependencies: gawk grep lz4 zstd wget gcc make autoconf libtool pkgconf
# libcap fuse2 (or fuse3) lzo xz zlib findutils
#
# Dwarfs dependencies: fuse2 (or fuse3) openssl jemalloc xxhash boost lz4 xz zstd libarchive
# libunwind google-glod gtest fmt gflags double-conversion cmake ruby-ronn libevent libdwarf
# Dwarfs dependencies: fuse2 (or fuse3) openssl jemalloc xxhash boost lz4
# xz zstd libarchive libunwind google-glod gtest fmt gflags double-conversion
# cmake ruby-ronn libevent libdwarf
#
# Dwarfs compilation is optional and disabled by default.
@ -15,6 +17,7 @@ squashfuse_version="0.1.104"
bwrap_version="0.4.1"
lz4_version="1.9.3"
zstd_version="1.5.0"
squashfs_tools_version="4.5"
dwarfs_version="0.5.6"
export CC=gcc
@ -30,11 +33,13 @@ wget -q --show-progress -O lz4.tar.gz https://github.com/lz4/lz4/archive/refs/ta
wget -q --show-progress -O zstd.tar.gz https://github.com/facebook/zstd/archive/refs/tags/v${zstd_version}.tar.gz
wget -q --show-progress -O squashfuse.tar.gz https://github.com/vasi/squashfuse/archive/refs/tags/${squashfuse_version}.tar.gz
wget -q --show-progress -O bwrap.tar.gz https://github.com/containers/bubblewrap/archive/refs/tags/v${bwrap_version}.tar.gz
wget -q --show-progress -O sqfstools.tar.gz https://github.com/plougher/squashfs-tools/archive/refs/tags/${squashfs_tools_version}.tar.gz
tar xf lz4.tar.gz
tar xf zstd.tar.gz
tar xf squashfuse.tar.gz
tar xf bwrap.tar.gz
tar xf sqfstools.tar.gz
cd bubblewrap-${bwrap_version}
./autogen.sh
@ -52,11 +57,18 @@ cd ../squashfuse-${squashfuse_version}
./configure
make -j$(nproc) DESTDIR="${script_dir}"/build-utils/bin install
cd ../squashfs-tools-${squashfs_tools_version}/squashfs-tools
make -j$(nproc) GZIP_SUPPORT=1 XZ_SUPPORT=1 LZO_SUPPORT=1 LZMA_XZ_SUPPORT=1 \
LZ4_SUPPORT=1 ZSTD_SUPPORT=1 XATTR_SUPPORT=1
make INSTALL_DIR="${script_dir}"/build-utils/bin/usr/local/bin install
cd "${script_dir}"/build-utils
mkdir utils
mv bin/usr/local/bin/bwrap utils
mv bin/usr/local/bin/squashfuse utils
mv bin/usr/local/bin/squashfuse_ll utils
mv bin/usr/local/bin/mksquashfs utils
mv bin/usr/local/bin/unsquashfs utils
mv bin/usr/local/lib/liblz4.so.${lz4_version} utils/liblz4.so.1
mv bin/usr/local/lib/libzstd.so.${zstd_version} utils/libzstd.so.1
mv bin/usr/local/lib/libfuseprivate.so.0.0.0 utils/libfuseprivate.so.0
@ -70,7 +82,7 @@ fi
if [ "${build_dwarfs}" = "true" ]; then
wget -q --show-progress -O dwarfs.tar.xz https://github.com/mhx/dwarfs/releases/download/v${dwarfs_version}/dwarfs-${dwarfs_version}.tar.xz
tar xf dwarfs.tar.xz
mkdir build
cmake -B build -S dwarfs-${dwarfs_version} -DCMAKE_BUILD_TYPE=Release \
-DPREFER_SYSTEM_ZSTD=ON -DPREFER_SYSTEM_XXHASH=ON \
@ -78,9 +90,11 @@ if [ "${build_dwarfs}" = "true" ]; then
make -C build -j$(nproc)
make -C build DESTDIR="${script_dir}"/build-utils/bin install
mv bin/usr/local/sbin/dwarfs2 utils/dwarfs
mv bin/usr/local/sbin/dwarfs utils/dwarfs3
mv bin/usr/local/bin/mkdwarfs utils
mv bin/usr/local/bin/dwarfsextract utils
fi
libs_list="$(ldd utils/* | grep "=> /" | awk '{print $3}' | xargs)"
@ -95,6 +109,7 @@ find utils -type f -exec strip --strip-unneeded {} \; 2>/dev/null
cat <<EOF > utils/info
squashfuse ${squashfuse_version}
squashfs-tools ${squashfs_tools_version}
bubblewrap ${bwrap_version}
lz4 ${lz4_version}
zstd ${zstd_version}

4
utils.tar.gz
View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:44d768081ead4fdb547648844bf8334b7533ba07828ccf2f63a9a0d3f08058db
size 7418944
oid sha256:f02bdddb8c050751aa359e3bdabac08e1e2cc6ef767f0572fb84ccaec3a6012a
size 9144313