ethonion10/v86
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

copy-to-sha256 and fs2json: Support for tar files, refactor debian-full script to not require root

Browse source
This commit is contained in:
Fabian 2018-10-08 16:49:48 -03:00
parent af0f153640
commit 68c59d10a5
5 changed files with 155 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docker/debian-full/Readme.md
View file

@ -1,8 +1,7 @@
You can build a Linux image for use with v86:
1. Run `./build-container.sh` to build the Docker container (requires dockerd)
2. Run `./create-9p-from-container.sh` to extract the files for v86 from the Docker container (requires root)
3. Run `./build-state.js` to build a state image in order to skip the boot process
1. Run `./build-container.sh` to build the Docker container and v86 images (requires dockerd)
2. Run `./build-state.js` to build a state image in order to skip the boot process
Go to `debug.html?profile=debian` to start the generated container.

23
docker/debian-full/build-container.sh
View file

@ -1,6 +1,21 @@
#!/usr/bin/env bash
set -e
set -veu
docker build . --rm --tag i386/debian-full
docker rm debian-full || true
docker create -t -i --name debian-full i386/debian-full bash
OUT_ROOTFS_TAR=$(dirname "$0")/../../images/debian-9p-rootfs.tar
OUT_ROOTFS_FLAT=$(dirname "$0")/../../images/debian-9p-rootfs-flat
OUT_FSJSON=$(dirname "$0")/../../images/debian-base-fs.json
CONTAINER_NAME=debian-full
IMAGE_NAME=i386/debian-full
docker build . --rm --tag "$IMAGE_NAME"
docker rm "$CONTAINER_NAME" || true
docker create -t -i --name "$CONTAINER_NAME" "$IMAGE_NAME" bash
docker export "$CONTAINER_NAME" > "$OUT_ROOTFS_TAR"
$(dirname "$0")/../../tools/fs2json.py --out "$OUT_FSJSON" "$OUT_ROOTFS_TAR"
# Note: Not deleting old files here
$(dirname "$0")/../../tools/copy-to-sha256.py "$OUT_ROOTFS_TAR" "$OUT_ROOTFS_FLAT"
echo "$OUT_ROOTFS_TAR", "$OUT_ROOTFS_FLAT" and "$OUT_FSJSON" created.

27
docker/debian-full/create-9p-from-container.sh
View file

@ -1,27 +0,0 @@
#!/usr/bin/env bash
set -veu
if [ $(id -u) != "0" ]
then
echo "Please run as root"
exit 1
fi
OUT_ROOTFS=$(dirname "$0")/../../images/debian-9p-rootfs
OUT_ROOTFS_FLAT=$(dirname "$0")/../../images/debian-9p-rootfs-flat
OUT_FSJSON=$(dirname "$0")/../../images/debian-base-fs.json
CONTAINER_NAME=debian-full
rm -rf "$OUT_ROOTFS/" && mkdir "$OUT_ROOTFS/"
docker export "$CONTAINER_NAME" | tar -xvC "$OUT_ROOTFS/"
$(dirname "$0")/../../tools/fs2json.py --out "$OUT_FSJSON" "$OUT_ROOTFS/"
chmod 644 "$OUT_FSJSON"
# Note: Not deleting old files here
$(dirname "$0")/../../tools/copy-to-sha256.py "$OUT_ROOTFS" "$OUT_ROOTFS_FLAT"
#find $OUT_ROOTFS/ -type d -exec chmod 755 {} ";"
#find $OUT_ROOTFS/ -type f -exec chmod 644 {} ";"
echo "$OUT_ROOTFS" and "$OUT_ROOTFS_FLAT" created.

40
tools/copy-to-sha256.py
View file

@ -6,14 +6,18 @@ import stat
import argparse
import hashlib
import shutil
import tarfile
def hash_file(filename):
h = hashlib.sha256()
with open(filename, "rb", buffering=0) as f:
for b in iter(lambda: f.read(128 * 1024), b""):
h.update(b)
return h.hexdigest()
return hash_fileobj(f)
def hash_fileobj(f):
h = hashlib.sha256()
for b in iter(lambda: f.read(128*1024), b""):
h.update(b)
return h.hexdigest()
def main():
logging.basicConfig(format="%(message)s")
@ -30,6 +34,17 @@ def main():
from_path = os.path.normpath(args.from_path)
to_path = os.path.normpath(args.to_path)
try:
tar = tarfile.open(from_path, "r")
except IsADirectoryError:
tar = None
if tar:
handle_tar(logger, tar, to_path)
else:
handle_dir(logger, path, to_path)
def handle_dir(logger, from_path, to_path):
def onerror(oserror):
logger.warning(oserror)
@ -56,5 +71,22 @@ def main():
logger.info("cp {} {}".format(absname, to_abs))
shutil.copyfile(absname, to_abs)
def handle_tar(logger, tar, to_path):
for member in tar.getmembers():
if member.isfile() or member.islnk():
f = tar.extractfile(member)
sha256 = hash_fileobj(f)
to_abs = os.path.join(to_path, sha256)
if os.path.exists(to_abs):
logger.info("Exists, skipped {}".format(to_abs))
else:
logger.info("Extracted {}".format(to_abs))
to_file = open(to_abs, "wb")
f.seek(0)
shutil.copyfileobj(f, to_file)
if __name__ == "__main__":
main()

114
tools/fs2json.py
View file

@ -1,5 +1,10 @@
#!/usr/bin/env python
# Note:
# - Hardlinks are copied
# - The size of symlinks and directories is meaningless, it depends on whatever
# the filesystem/tar file reports
import argparse
import json
import os
@ -8,6 +13,7 @@ import sys
import itertools
import logging
import hashlib
import tarfile
VERSION = 3
@ -26,12 +32,14 @@ IDX_SHA256 = 6
def hash_file(filename):
h = hashlib.sha256()
with open(filename, "rb", buffering=0) as f:
for b in iter(lambda : f.read(128*1024), b""):
h.update(b)
return h.hexdigest()
return hash_fileobj(f)
def hash_fileobj(f):
h = hashlib.sha256()
for b in iter(lambda: f.read(128*1024), b""):
h.update(b)
return h.hexdigest()
def main():
logging.basicConfig(format="%(message)s")
@ -53,14 +61,46 @@ def main():
help="File to write to (defaults to stdout)",
default=sys.stdout)
args.add_argument("path",
metavar="path",
help="Base path to include in JSON")
metavar="path-or-tar",
help="Base path or tar file to include in JSON")
args = args.parse_args()
path = os.path.normpath(args.path)
try:
tar = tarfile.open(path, "r")
except IsADirectoryError:
tar = None
if tar:
(root, total_size) = handle_tar(logger, tar)
else:
(root, total_size) = handle_dir(logger, path, args.exclude)
if False:
# normalize the order of children, useful to debug differences between
# the tar and filesystem reader
def sort_children(children):
for c in children:
if isinstance(c[IDX_TARGET], list):
sort_children(c[IDX_TARGET])
children.sort()
sort_children(root)
result = {
"fsroot": root,
"version": VERSION,
"size": total_size,
}
logger.info("Creating json ...")
json.dump(result, args.out, check_circular=False, separators=(',', ':'))
def handle_dir(logger, path, exclude):
path = path + "/"
exclude = args.exclude or []
exclude = exclude or []
exclude = [os.path.join("/", os.path.normpath(p)) for p in exclude]
exclude = set(exclude)
@ -72,11 +112,7 @@ def main():
prevpath = []
mainroot = []
result = {
"fsroot": mainroot,
"version": VERSION,
"size": 0,
}
total_size = 0
rootstack = [mainroot]
def make_node(st, name):
@ -90,7 +126,8 @@ def main():
obj[IDX_UID] = st.st_uid
obj[IDX_GID] = st.st_gid
result["size"] += st.st_size
nonlocal total_size
total_size += st.st_size
# Missing:
# int(st.st_atime),
@ -116,7 +153,7 @@ def main():
break
depth += 1
for name in prevpath[depth:]:
for _name in prevpath[depth:]:
rootstack.pop()
oldroot = rootstack[-1]
@ -162,9 +199,54 @@ def main():
prevpath = pathparts
logger.info("Creating json ...")
return (mainroot, total_size)
def handle_tar(logger, tar):
mainroot = []
total_size = 0
for member in tar.getmembers():
parts = member.name.split("/")
name = parts.pop()
dir = mainroot
for p in parts:
for c in dir:
if c[IDX_NAME] == p:
dir = c[IDX_TARGET]
obj = [None] * 7
obj[IDX_NAME] = name
obj[IDX_SIZE] = member.size
obj[IDX_MTIME] = member.mtime
obj[IDX_MODE] = member.mode
obj[IDX_UID] = member.uid
obj[IDX_GID] = member.gid
if member.isfile() or member.islnk():
f = tar.extractfile(member)
obj[IDX_SHA256] = hash_fileobj(f)
if member.islnk():
# fix size for hard links
f.seek(0, os.SEEK_END)
obj[IDX_SIZE] = int(f.tell())
elif member.isdir():
obj[IDX_TARGET] = []
elif member.issym():
obj[IDX_TARGET] = member.linkname
else:
logger.error("Unsupported type: {} ({})".format(member.type, name))
total_size += obj[IDX_SIZE]
while obj[-1] is None:
obj.pop()
dir.append(obj)
return mainroot, total_size
json.dump(result, args.out, check_circular=False, separators=(',', ':'))
if __name__ == "__main__":
main()