fix git ignore

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2
 
 jobs:
-  golang-1.11-pipeline:
+  golang-1.11:
     working_directory: /home/circleci/app
     docker:
       - image: circleci/golang:1.11
@@ -21,7 +21,7 @@ jobs:
           name: run static analysis & tests
           command: make ci
 
-  golang-1.12-pipeline:
+  golang-1.12:
     working_directory: /home/circleci/app
     docker:
       - image: circleci/golang:1.12
@@ -41,7 +41,7 @@ jobs:
           name: run static analysis & tests
           command: make ci
 
-  golang-1.13-pipeline:
+  golang-1.13:
     working_directory: /home/circleci/app
     docker:
       - image: circleci/golang:1.13
@@ -65,6 +65,6 @@ workflows:
   version: 2
   commit:
     jobs:
-      - golang-1.11-pipeline
-      - golang-1.12-pipeline
-      - golang-1.13-pipeline
+      - golang-1.11
+      - golang-1.12
+      - golang-1.13

.gitignore

@@ -1,4 +1,5 @@
 /.idea
+/bin 
 
 # Binaries for programs and plugins
 *.exe
@@ -21,6 +22,3 @@
 /dist
 .cover
 coverage.txt
-
-# ignore the binary
-dive

dive/filetree/cache.go

@@ -0,0 +1,79 @@
+package filetree
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+type TreeCacheKey struct {
+	bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int
+}
+
+type TreeCache struct {
+	refTrees []*FileTree
+	cache    map[TreeCacheKey]*FileTree
+}
+
+func (cache *TreeCache) Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int) *FileTree {
+	key := TreeCacheKey{bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop}
+	if value, exists := cache.cache[key]; exists {
+		return value
+	}
+
+	value := cache.buildTree(key)
+	cache.cache[key] = value
+	return value
+}
+
+func (cache *TreeCache) buildTree(key TreeCacheKey) *FileTree {
+	newTree := StackTreeRange(cache.refTrees, key.bottomTreeStart, key.bottomTreeStop)
+	for idx := key.topTreeStart; idx <= key.topTreeStop; idx++ {
+		err := newTree.CompareAndMark(cache.refTrees[idx])
+		if err != nil {
+			logrus.Errorf("unable to build tree: %+v", err)
+		}
+	}
+	return newTree
+}
+
+func (cache *TreeCache) Build() {
+	var bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int
+
+	// case 1: layer compare (top tree SIZE is fixed (BUT floats forward), Bottom tree SIZE changes)
+	for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ {
+		bottomTreeStart = 0
+		topTreeStop = selectIdx
+
+		if selectIdx == 0 {
+			bottomTreeStop = selectIdx
+			topTreeStart = selectIdx
+		} else {
+			bottomTreeStop = selectIdx - 1
+			topTreeStart = selectIdx
+		}
+
+		cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop)
+	}
+
+	// case 2: aggregated compare (bottom tree is ENTIRELY fixed, top tree SIZE changes)
+	for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ {
+		bottomTreeStart = 0
+		topTreeStop = selectIdx
+		if selectIdx == 0 {
+			bottomTreeStop = selectIdx
+			topTreeStart = selectIdx
+		} else {
+			bottomTreeStop = 0
+			topTreeStart = 1
+		}
+
+		cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop)
+	}
+}
+
+func NewFileTreeCache(refTrees []*FileTree) TreeCache {
+
+	return TreeCache{
+		refTrees: refTrees,
+		cache:    make(map[TreeCacheKey]*FileTree),
+	}
+}

dive/filetree/diff.go

@@ -0,0 +1,40 @@
+package filetree
+
+import (
+	"fmt"
+)
+
+const (
+	Unmodified DiffType = iota
+	Modified
+	Added
+	Removed
+)
+
+// DiffType defines the comparison result between two FileNodes
+type DiffType int
+
+// String of a DiffType
+func (diff DiffType) String() string {
+	switch diff {
+	case Unmodified:
+		return "Unmodified"
+	case Modified:
+		return "Modified"
+	case Added:
+		return "Added"
+	case Removed:
+		return "Removed"
+	default:
+		return fmt.Sprintf("%d", int(diff))
+	}
+}
+
+// merge two DiffTypes into a single result. Essentially, return the given value unless they two values differ,
+// in which case we can only determine that there is "a change".
+func (diff DiffType) merge(other DiffType) DiffType {
+	if diff == other {
+		return diff
+	}
+	return Modified
+}

dive/filetree/efficiency.go

@@ -0,0 +1,122 @@
+package filetree
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/sirupsen/logrus"
+)
+
+// EfficiencyData represents the storage and reference statistics for a given file tree path.
+type EfficiencyData struct {
+	Path              string
+	Nodes             []*FileNode
+	CumulativeSize    int64
+	minDiscoveredSize int64
+}
+
+// EfficiencySlice represents an ordered set of EfficiencyData data structures.
+type EfficiencySlice []*EfficiencyData
+
+// Len is required for sorting.
+func (efs EfficiencySlice) Len() int {
+	return len(efs)
+}
+
+// Swap operation is required for sorting.
+func (efs EfficiencySlice) Swap(i, j int) {
+	efs[i], efs[j] = efs[j], efs[i]
+}
+
+// Less comparison is required for sorting.
+func (efs EfficiencySlice) Less(i, j int) bool {
+	return efs[i].CumulativeSize < efs[j].CumulativeSize
+}
+
+// Efficiency returns the score and file set of the given set of FileTrees (layers). This is loosely based on:
+// 1. Files that are duplicated across layers discounts your score, weighted by file size
+// 2. Files that are removed discounts your score, weighted by the original file size
+func Efficiency(trees []*FileTree) (float64, EfficiencySlice) {
+	efficiencyMap := make(map[string]*EfficiencyData)
+	inefficientMatches := make(EfficiencySlice, 0)
+	currentTree := 0
+
+	visitor := func(node *FileNode) error {
+		path := node.Path()
+		if _, ok := efficiencyMap[path]; !ok {
+			efficiencyMap[path] = &EfficiencyData{
+				Path:              path,
+				Nodes:             make([]*FileNode, 0),
+				minDiscoveredSize: -1,
+			}
+		}
+		data := efficiencyMap[path]
+
+		// this node may have had children that were deleted, however, we won't explicitly list out every child, only
+		// the top-most parent with the cumulative size. These operations will need to be done on the full (stacked)
+		// tree.
+		// Note: whiteout files may also represent directories, so we need to find out if this was previously a file or dir.
+		var sizeBytes int64
+
+		if node.IsWhiteout() {
+			sizer := func(curNode *FileNode) error {
+				sizeBytes += curNode.Data.FileInfo.Size
+				return nil
+			}
+			stackedTree := StackTreeRange(trees, 0, currentTree-1)
+			previousTreeNode, err := stackedTree.GetNode(node.Path())
+			if err != nil {
+				logrus.Debug(fmt.Sprintf("CurrentTree: %d : %s", currentTree, err))
+			} else if previousTreeNode.Data.FileInfo.IsDir {
+				err = previousTreeNode.VisitDepthChildFirst(sizer, nil)
+				if err != nil {
+					logrus.Errorf("unable to propagate whiteout dir: %+v", err)
+				}
+			}
+
+		} else {
+			sizeBytes = node.Data.FileInfo.Size
+		}
+
+		data.CumulativeSize += sizeBytes
+		if data.minDiscoveredSize < 0 || sizeBytes < data.minDiscoveredSize {
+			data.minDiscoveredSize = sizeBytes
+		}
+		data.Nodes = append(data.Nodes, node)
+
+		if len(data.Nodes) == 2 {
+			inefficientMatches = append(inefficientMatches, data)
+		}
+
+		return nil
+	}
+	visitEvaluator := func(node *FileNode) bool {
+		return node.IsLeaf()
+	}
+	for idx, tree := range trees {
+		currentTree = idx
+		err := tree.VisitDepthChildFirst(visitor, visitEvaluator)
+		if err != nil {
+			logrus.Errorf("unable to propagate ref tree: %+v", err)
+		}
+	}
+
+	// calculate the score
+	var minimumPathSizes int64
+	var discoveredPathSizes int64
+
+	for _, value := range efficiencyMap {
+		minimumPathSizes += value.minDiscoveredSize
+		discoveredPathSizes += value.CumulativeSize
+	}
+	var score float64
+	if discoveredPathSizes == 0 {
+		score = 1.0
+	} else {
+		score = float64(minimumPathSizes) / float64(discoveredPathSizes)
+	}
+
+	sort.Sort(inefficientMatches)
+
+	return score, inefficientMatches
+}

dive/filetree/efficiency_test.go

@@ -0,0 +1,80 @@
+package filetree
+
+import (
+	"testing"
+)
+
+func checkError(t *testing.T, err error, message string) {
+	if err != nil {
+		t.Errorf(message+": %+v", err)
+	}
+}
+
+func TestEfficency(t *testing.T) {
+	trees := make([]*FileTree, 3)
+	for idx := range trees {
+		trees[idx] = NewFileTree()
+	}
+
+	_, _, err := trees[0].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 2000})
+	checkError(t, err, "could not setup test")
+
+	_, _, err = trees[0].AddPath("/etc/nginx/public", FileInfo{Size: 3000})
+	checkError(t, err, "could not setup test")
+
+	_, _, err = trees[1].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 5000})
+	checkError(t, err, "could not setup test")
+	_, _, err = trees[1].AddPath("/etc/athing", FileInfo{Size: 10000})
+	checkError(t, err, "could not setup test")
+
+	_, _, err = trees[2].AddPath("/etc/.wh.nginx", *BlankFileChangeInfo("/etc/.wh.nginx"))
+	checkError(t, err, "could not setup test")
+
+	var expectedScore = 0.75
+	var expectedMatches = EfficiencySlice{
+		&EfficiencyData{Path: "/etc/nginx/nginx.conf", CumulativeSize: 7000},
+	}
+	actualScore, actualMatches := Efficiency(trees)
+
+	if expectedScore != actualScore {
+		t.Errorf("Expected score of %v but go %v", expectedScore, actualScore)
+	}
+
+	if len(actualMatches) != len(expectedMatches) {
+		for _, match := range actualMatches {
+			t.Logf("   match: %+v", match)
+		}
+		t.Fatalf("Expected to find %d inefficient paths, but found %d", len(expectedMatches), len(actualMatches))
+	}
+
+	if expectedMatches[0].Path != actualMatches[0].Path {
+		t.Errorf("Expected path of %s but go %s", expectedMatches[0].Path, actualMatches[0].Path)
+	}
+
+	if expectedMatches[0].CumulativeSize != actualMatches[0].CumulativeSize {
+		t.Errorf("Expected cumulative size of %v but go %v", expectedMatches[0].CumulativeSize, actualMatches[0].CumulativeSize)
+	}
+}
+
+func TestEfficency_ScratchImage(t *testing.T) {
+	trees := make([]*FileTree, 3)
+	for idx := range trees {
+		trees[idx] = NewFileTree()
+	}
+
+	_, _, err := trees[0].AddPath("/nothing", FileInfo{Size: 0})
+	checkError(t, err, "could not setup test")
+
+	var expectedScore = 1.0
+	var expectedMatches = EfficiencySlice{}
+	actualScore, actualMatches := Efficiency(trees)
+
+	if expectedScore != actualScore {
+		t.Errorf("Expected score of %v but go %v", expectedScore, actualScore)
+	}
+
+	if len(actualMatches) > 0 {
+		t.Fatalf("Expected to find %d inefficient paths, but found %d", len(expectedMatches), len(actualMatches))
+	}
+
+}

dive/filetree/file_info.go

@@ -0,0 +1,106 @@
+package filetree
+
+import (
+	"archive/tar"
+	"github.com/cespare/xxhash"
+	"github.com/sirupsen/logrus"
+	"io"
+	"os"
+)
+
+// FileInfo contains tar metadata for a specific FileNode
+type FileInfo struct {
+	Path     string
+	TypeFlag byte
+	Linkname string
+	hash     uint64
+	Size     int64
+	Mode     os.FileMode
+	Uid      int
+	Gid      int
+	IsDir    bool
+}
+
+// NewFileInfo extracts the metadata from a tar header and file contents and generates a new FileInfo object.
+func NewFileInfo(reader *tar.Reader, header *tar.Header, path string) FileInfo {
+	if header.Typeflag == tar.TypeDir {
+		return FileInfo{
+			Path:     path,
+			TypeFlag: header.Typeflag,
+			Linkname: header.Linkname,
+			hash:     0,
+			Size:     header.FileInfo().Size(),
+			Mode:     header.FileInfo().Mode(),
+			Uid:      header.Uid,
+			Gid:      header.Gid,
+			IsDir:    header.FileInfo().IsDir(),
+		}
+	}
+
+	hash := getHashFromReader(reader)
+
+	return FileInfo{
+		Path:     path,
+		TypeFlag: header.Typeflag,
+		Linkname: header.Linkname,
+		hash:     hash,
+		Size:     header.FileInfo().Size(),
+		Mode:     header.FileInfo().Mode(),
+		Uid:      header.Uid,
+		Gid:      header.Gid,
+		IsDir:    header.FileInfo().IsDir(),
+	}
+}
+
+// Copy duplicates a FileInfo
+func (data *FileInfo) Copy() *FileInfo {
+	if data == nil {
+		return nil
+	}
+	return &FileInfo{
+		Path:     data.Path,
+		TypeFlag: data.TypeFlag,
+		Linkname: data.Linkname,
+		hash:     data.hash,
+		Size:     data.Size,
+		Mode:     data.Mode,
+		Uid:      data.Uid,
+		Gid:      data.Gid,
+		IsDir:    data.IsDir,
+	}
+}
+
+// Compare determines the DiffType between two FileInfos based on the type and contents of each given FileInfo
+func (data *FileInfo) Compare(other FileInfo) DiffType {
+	if data.TypeFlag == other.TypeFlag {
+		if data.hash == other.hash &&
+			data.Mode == other.Mode &&
+			data.Uid == other.Uid &&
+			data.Gid == other.Gid {
+			return Unmodified
+		}
+	}
+	return Modified
+}
+
+func getHashFromReader(reader io.Reader) uint64 {
+	h := xxhash.New()
+
+	buf := make([]byte, 1024)
+	for {
+		n, err := reader.Read(buf)
+		if err != nil && err != io.EOF {
+			logrus.Panic(err)
+		}
+		if n == 0 {
+			break
+		}
+
+		_, err = h.Write(buf[:n])
+		if err != nil {
+			logrus.Panic(err)
+		}
+	}
+
+	return h.Sum64()
+}

dive/filetree/file_node.go

@@ -0,0 +1,325 @@
+package filetree
+
+import (
+	"archive/tar"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+	"github.com/phayes/permbits"
+)
+
+const (
+	AttributeFormat = "%s%s %11s %10s "
+)
+
+var diffTypeColor = map[DiffType]*color.Color{
+	Added:      color.New(color.FgGreen),
+	Removed:    color.New(color.FgRed),
+	Modified:   color.New(color.FgYellow),
+	Unmodified: color.New(color.Reset),
+}
+
+// FileNode represents a single file, its relation to files beneath it, the tree it exists in, and the metadata of the given file.
+type FileNode struct {
+	Tree     *FileTree
+	Parent   *FileNode
+	Name     string
+	Data     NodeData
+	Children map[string]*FileNode
+	path     string
+}
+
+// NewNode creates a new FileNode relative to the given parent node with a payload.
+func NewNode(parent *FileNode, name string, data FileInfo) (node *FileNode) {
+	node = new(FileNode)
+	node.Name = name
+	node.Data = *NewNodeData()
+	node.Data.FileInfo = *data.Copy()
+
+	node.Children = make(map[string]*FileNode)
+	node.Parent = parent
+	if parent != nil {
+		node.Tree = parent.Tree
+	}
+
+	return node
+}
+
+// renderTreeLine returns a string representing this FileNode in the context of a greater ASCII tree.
+func (node *FileNode) renderTreeLine(spaces []bool, last bool, collapsed bool) string {
+	var otherBranches string
+	for _, space := range spaces {
+		if space {
+			otherBranches += noBranchSpace
+		} else {
+			otherBranches += branchSpace
+		}
+	}
+
+	thisBranch := middleItem
+	if last {
+		thisBranch = lastItem
+	}
+
+	collapsedIndicator := uncollapsedItem
+	if collapsed {
+		collapsedIndicator = collapsedItem
+	}
+
+	return otherBranches + thisBranch + collapsedIndicator + node.String() + newLine
+}
+
+// Copy duplicates the existing node relative to a new parent node.
+func (node *FileNode) Copy(parent *FileNode) *FileNode {
+	newNode := NewNode(parent, node.Name, node.Data.FileInfo)
+	newNode.Data.ViewInfo = node.Data.ViewInfo
+	newNode.Data.DiffType = node.Data.DiffType
+	for name, child := range node.Children {
+		newNode.Children[name] = child.Copy(newNode)
+		child.Parent = newNode
+	}
+	return newNode
+}
+
+// AddChild creates a new node relative to the current FileNode.
+func (node *FileNode) AddChild(name string, data FileInfo) (child *FileNode) {
+	// never allow processing of purely whiteout flag files (for now)
+	if strings.HasPrefix(name, doubleWhiteoutPrefix) {
+		return nil
+	}
+
+	child = NewNode(node, name, data)
+	if node.Children[name] != nil {
+		// tree node already exists, replace the payload, keep the children
+		node.Children[name].Data.FileInfo = *data.Copy()
+	} else {
+		node.Children[name] = child
+		node.Tree.Size++
+	}
+
+	return child
+}
+
+// Remove deletes the current FileNode from it's parent FileNode's relations.
+func (node *FileNode) Remove() error {
+	if node == node.Tree.Root {
+		return fmt.Errorf("cannot remove the tree root")
+	}
+	for _, child := range node.Children {
+		err := child.Remove()
+		if err != nil {
+			return err
+		}
+	}
+	delete(node.Parent.Children, node.Name)
+	node.Tree.Size--
+	return nil
+}
+
+// String shows the filename formatted into the proper color (by DiffType), additionally indicating if it is a symlink.
+func (node *FileNode) String() string {
+	var display string
+	if node == nil {
+		return ""
+	}
+
+	display = node.Name
+	if node.Data.FileInfo.TypeFlag == tar.TypeSymlink || node.Data.FileInfo.TypeFlag == tar.TypeLink {
+		display += " → " + node.Data.FileInfo.Linkname
+	}
+	return diffTypeColor[node.Data.DiffType].Sprint(display)
+}
+
+// MetadatString returns the FileNode metadata in a columnar string.
+func (node *FileNode) MetadataString() string {
+	if node == nil {
+		return ""
+	}
+
+	fileMode := permbits.FileMode(node.Data.FileInfo.Mode).String()
+	dir := "-"
+	if node.Data.FileInfo.IsDir {
+		dir = "d"
+	}
+	user := node.Data.FileInfo.Uid
+	group := node.Data.FileInfo.Gid
+	userGroup := fmt.Sprintf("%d:%d", user, group)
+
+	var sizeBytes int64
+
+	if node.IsLeaf() {
+		sizeBytes = node.Data.FileInfo.Size
+	} else {
+		sizer := func(curNode *FileNode) error {
+			// don't include file sizes of children that have been removed (unless the node in question is a removed dir,
+			// then show the accumulated size of removed files)
+			if curNode.Data.DiffType != Removed || node.Data.DiffType == Removed {
+				sizeBytes += curNode.Data.FileInfo.Size
+			}
+			return nil
+		}
+
+		err := node.VisitDepthChildFirst(sizer, nil)
+		if err != nil {
+			logrus.Errorf("unable to propagate node for metadata: %+v", err)
+		}
+	}
+
+	size := humanize.Bytes(uint64(sizeBytes))
+
+	return diffTypeColor[node.Data.DiffType].Sprint(fmt.Sprintf(AttributeFormat, dir, fileMode, userGroup, size))
+}
+
+// VisitDepthChildFirst iterates a tree depth-first (starting at this FileNode), evaluating the deepest depths first (visit on bubble up)
+func (node *FileNode) VisitDepthChildFirst(visitor Visitor, evaluator VisitEvaluator) error {
+	var keys []string
+	for key := range node.Children {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, name := range keys {
+		child := node.Children[name]
+		err := child.VisitDepthChildFirst(visitor, evaluator)
+		if err != nil {
+			return err
+		}
+	}
+	// never visit the root node
+	if node == node.Tree.Root {
+		return nil
+	} else if evaluator != nil && evaluator(node) || evaluator == nil {
+		return visitor(node)
+	}
+
+	return nil
+}
+
+// VisitDepthParentFirst iterates a tree depth-first (starting at this FileNode), evaluating the shallowest depths first (visit while sinking down)
+func (node *FileNode) VisitDepthParentFirst(visitor Visitor, evaluator VisitEvaluator) error {
+	var err error
+
+	doVisit := evaluator != nil && evaluator(node) || evaluator == nil
+
+	if !doVisit {
+		return nil
+	}
+
+	// never visit the root node
+	if node != node.Tree.Root {
+		err = visitor(node)
+		if err != nil {
+			return err
+		}
+	}
+
+	var keys []string
+	for key := range node.Children {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, name := range keys {
+		child := node.Children[name]
+		err = child.VisitDepthParentFirst(visitor, evaluator)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// IsWhiteout returns an indication if this file may be a overlay-whiteout file.
+func (node *FileNode) IsWhiteout() bool {
+	return strings.HasPrefix(node.Name, whiteoutPrefix)
+}
+
+// IsLeaf returns true is the current node has no child nodes.
+func (node *FileNode) IsLeaf() bool {
+	return len(node.Children) == 0
+}
+
+// Path returns a slash-delimited string from the root of the greater tree to the current node (e.g. /a/path/to/here)
+func (node *FileNode) Path() string {
+	if node.path == "" {
+		var path []string
+		curNode := node
+		for {
+			if curNode.Parent == nil {
+				break
+			}
+
+			name := curNode.Name
+			if curNode == node {
+				// white out prefixes are fictitious on leaf nodes
+				name = strings.TrimPrefix(name, whiteoutPrefix)
+			}
+
+			path = append([]string{name}, path...)
+			curNode = curNode.Parent
+		}
+		node.path = "/" + strings.Join(path, "/")
+	}
+	return strings.Replace(node.path, "//", "/", -1)
+}
+
+// deriveDiffType determines a DiffType to the current FileNode. Note: the DiffType of a node is always the DiffType of
+// its attributes and its contents. The contents are the bytes of the file of the children of a directory.
+func (node *FileNode) deriveDiffType(diffType DiffType) error {
+	if node.IsLeaf() {
+		return node.AssignDiffType(diffType)
+	}
+
+	myDiffType := diffType
+	for _, v := range node.Children {
+		myDiffType = myDiffType.merge(v.Data.DiffType)
+	}
+
+	return node.AssignDiffType(myDiffType)
+}
+
+// AssignDiffType will assign the given DiffType to this node, possibly affecting child nodes.
+func (node *FileNode) AssignDiffType(diffType DiffType) error {
+	var err error
+
+	node.Data.DiffType = diffType
+
+	if diffType == Removed {
+		// if we've removed this node, then all children have been removed as well
+		for _, child := range node.Children {
+			err = child.AssignDiffType(diffType)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// compare the current node against the given node, returning a definitive DiffType.
+func (node *FileNode) compare(other *FileNode) DiffType {
+	if node == nil && other == nil {
+		return Unmodified
+	}
+
+	if node == nil && other != nil {
+		return Added
+	}
+
+	if node != nil && other == nil {
+		return Removed
+	}
+
+	if other.IsWhiteout() {
+		return Removed
+	}
+	if node.Name != other.Name {
+		panic("comparing mismatched nodes")
+	}
+
+	return node.Data.FileInfo.Compare(other.Data.FileInfo)
+}

dive/filetree/file_node_test.go

@@ -0,0 +1,168 @@
+package filetree
+
+import (
+	"testing"
+)
+
+func TestAddChild(t *testing.T) {
+	var expected, actual int
+	tree := NewFileTree()
+
+	payload := FileInfo{
+		Path: "stufffffs",
+	}
+
+	one := tree.Root.AddChild("first node!", payload)
+
+	two := tree.Root.AddChild("nil node!", FileInfo{})
+
+	tree.Root.AddChild("third node!", FileInfo{})
+	two.AddChild("forth, one level down...", FileInfo{})
+	two.AddChild("fifth, one level down...", FileInfo{})
+	two.AddChild("fifth, one level down...", FileInfo{})
+
+	expected, actual = 5, tree.Size
+	if expected != actual {
+		t.Errorf("Expected a tree size of %d got %d.", expected, actual)
+	}
+
+	expected, actual = 2, len(two.Children)
+	if expected != actual {
+		t.Errorf("Expected 'twos' number of children to be %d got %d.", expected, actual)
+	}
+
+	expected, actual = 3, len(tree.Root.Children)
+	if expected != actual {
+		t.Errorf("Expected 'twos' number of children to be %d got %d.", expected, actual)
+	}
+
+	expectedFC := FileInfo{
+		Path: "stufffffs",
+	}
+	actualFC := one.Data.FileInfo
+	if expectedFC.Path != actualFC.Path {
+		t.Errorf("Expected 'ones' payload to be %+v got %+v.", expectedFC, actualFC)
+	}
+
+}
+
+func TestRemoveChild(t *testing.T) {
+	var expected, actual int
+
+	tree := NewFileTree()
+	tree.Root.AddChild("first", FileInfo{})
+	two := tree.Root.AddChild("nil", FileInfo{})
+	tree.Root.AddChild("third", FileInfo{})
+	forth := two.AddChild("forth", FileInfo{})
+	two.AddChild("fifth", FileInfo{})
+
+	err := forth.Remove()
+	checkError(t, err, "unable to setup test")
+
+	expected, actual = 4, tree.Size
+	if expected != actual {
+		t.Errorf("Expected a tree size of %d got %d.", expected, actual)
+	}
+
+	if tree.Root.Children["forth"] != nil {
+		t.Errorf("Expected 'forth' node to be deleted.")
+	}
+
+	err = two.Remove()
+	checkError(t, err, "unable to setup test")
+
+	expected, actual = 2, tree.Size
+	if expected != actual {
+		t.Errorf("Expected a tree size of %d got %d.", expected, actual)
+	}
+
+	if tree.Root.Children["nil"] != nil {
+		t.Errorf("Expected 'nil' node to be deleted.")
+	}
+
+}
+
+func TestPath(t *testing.T) {
+	expected := "/etc/nginx/nginx.conf"
+	tree := NewFileTree()
+	node, _, _ := tree.AddPath(expected, FileInfo{})
+
+	actual := node.Path()
+	if expected != actual {
+		t.Errorf("Expected path '%s' got '%s'", expected, actual)
+	}
+}
+
+func TestIsWhiteout(t *testing.T) {
+	tree1 := NewFileTree()
+	p1, _, _ := tree1.AddPath("/etc/nginx/public1", FileInfo{})
+	p2, _, _ := tree1.AddPath("/etc/nginx/.wh.public2", FileInfo{})
+	p3, _, _ := tree1.AddPath("/etc/nginx/public3/.wh..wh..opq", FileInfo{})
+
+	if p1.IsWhiteout() != false {
+		t.Errorf("Expected path '%s' to **not** be a whiteout file", p1.Name)
+	}
+
+	if p2.IsWhiteout() != true {
+		t.Errorf("Expected path '%s' to be a whiteout file", p2.Name)
+	}
+
+	if p3 != nil {
+		t.Errorf("Expected to not be able to add path '%s'", p2.Name)
+	}
+}
+
+func TestDiffTypeFromAddedChildren(t *testing.T) {
+	tree := NewFileTree()
+	node, _, _ := tree.AddPath("/usr", *BlankFileChangeInfo("/usr"))
+	node.Data.DiffType = Unmodified
+
+	node, _, _ = tree.AddPath("/usr/bin", *BlankFileChangeInfo("/usr/bin"))
+	node.Data.DiffType = Added
+
+	node, _, _ = tree.AddPath("/usr/bin2", *BlankFileChangeInfo("/usr/bin2"))
+	node.Data.DiffType = Removed
+
+	err := tree.Root.Children["usr"].deriveDiffType(Unmodified)
+	checkError(t, err, "unable to setup test")
+
+	if tree.Root.Children["usr"].Data.DiffType != Modified {
+		t.Errorf("Expected Modified but got %v", tree.Root.Children["usr"].Data.DiffType)
+	}
+}
+func TestDiffTypeFromRemovedChildren(t *testing.T) {
+	tree := NewFileTree()
+	_, _, _ = tree.AddPath("/usr", *BlankFileChangeInfo("/usr"))
+
+	info1 := BlankFileChangeInfo("/usr/.wh.bin")
+	node, _, _ := tree.AddPath("/usr/.wh.bin", *info1)
+	node.Data.DiffType = Removed
+
+	info2 := BlankFileChangeInfo("/usr/.wh.bin2")
+	node, _, _ = tree.AddPath("/usr/.wh.bin2", *info2)
+	node.Data.DiffType = Removed
+
+	err := tree.Root.Children["usr"].deriveDiffType(Unmodified)
+	checkError(t, err, "unable to setup test")
+
+	if tree.Root.Children["usr"].Data.DiffType != Modified {
+		t.Errorf("Expected Modified but got %v", tree.Root.Children["usr"].Data.DiffType)
+	}
+
+}
+
+func TestDirSize(t *testing.T) {
+	tree1 := NewFileTree()
+	_, _, err := tree1.AddPath("/etc/nginx/public1", FileInfo{Size: 100})
+	checkError(t, err, "unable to setup test")
+	_, _, err = tree1.AddPath("/etc/nginx/thing1", FileInfo{Size: 200})
+	checkError(t, err, "unable to setup test")
+	_, _, err = tree1.AddPath("/etc/nginx/public3/thing2", FileInfo{Size: 300})
+	checkError(t, err, "unable to setup test")
+
+	node, _ := tree1.GetNode("/etc/nginx")
+	expected, actual := "----------         0:0      600 B ", node.MetadataString()
+	if expected != actual {
+		t.Errorf("Expected metadata '%s' got '%s'", expected, actual)
+	}
+}

dive/filetree/file_tree.go

@@ -0,0 +1,380 @@
+package filetree
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	newLine              = "\n"
+	noBranchSpace        = "    "
+	branchSpace          = "│   "
+	middleItem           = "├─"
+	lastItem             = "└─"
+	whiteoutPrefix       = ".wh."
+	doubleWhiteoutPrefix = ".wh..wh.."
+	uncollapsedItem      = "─ "
+	collapsedItem        = "⊕ "
+)
+
+// FileTree represents a set of files, directories, and their relations.
+type FileTree struct {
+	Root     *FileNode
+	Size     int
+	FileSize uint64
+	Name     string
+	Id       uuid.UUID
+}
+
+// NewFileTree creates an empty FileTree
+func NewFileTree() (tree *FileTree) {
+	tree = new(FileTree)
+	tree.Size = 0
+	tree.Root = new(FileNode)
+	tree.Root.Tree = tree
+	tree.Root.Children = make(map[string]*FileNode)
+	tree.Id = uuid.New()
+	return tree
+}
+
+// renderParams is a representation of a FileNode in the context of the greater tree. All
+// data stored is necessary for rendering a single line in a tree format.
+type renderParams struct {
+	node          *FileNode
+	spaces        []bool
+	childSpaces   []bool
+	showCollapsed bool
+	isLast        bool
+}
+
+// renderStringTreeBetween returns a string representing the given tree between the given rows. Since each node
+// is rendered on its own line, the returned string shows the visible nodes not affected by a collapsed parent.
+func (tree *FileTree) renderStringTreeBetween(startRow, stopRow int, showAttributes bool) string {
+	// generate a list of nodes to render
+	var params = make([]renderParams, 0)
+	var result string
+
+	// visit from the front of the list
+	var paramsToVisit = []renderParams{{node: tree.Root, spaces: []bool{}, showCollapsed: false, isLast: false}}
+	for currentRow := 0; len(paramsToVisit) > 0 && currentRow <= stopRow; currentRow++ {
+		// pop the first node
+		var currentParams renderParams
+		currentParams, paramsToVisit = paramsToVisit[0], paramsToVisit[1:]
+
+		// take note of the next nodes to visit later
+		var keys []string
+		for key := range currentParams.node.Children {
+			keys = append(keys, key)
+		}
+		// we should always visit nodes in order
+		sort.Strings(keys)
+
+		var childParams = make([]renderParams, 0)
+		for idx, name := range keys {
+			child := currentParams.node.Children[name]
+			// don't visit this node...
+			if child.Data.ViewInfo.Hidden || currentParams.node.Data.ViewInfo.Collapsed {
+				continue
+			}
+
+			// visit this node...
+			isLast := idx == (len(currentParams.node.Children) - 1)
+			showCollapsed := child.Data.ViewInfo.Collapsed && len(child.Children) > 0
+
+			// completely copy the reference slice
+			childSpaces := make([]bool, len(currentParams.childSpaces))
+			copy(childSpaces, currentParams.childSpaces)
+
+			if len(child.Children) > 0 && !child.Data.ViewInfo.Collapsed {
+				childSpaces = append(childSpaces, isLast)
+			}
+
+			childParams = append(childParams, renderParams{
+				node:          child,
+				spaces:        currentParams.childSpaces,
+				childSpaces:   childSpaces,
+				showCollapsed: showCollapsed,
+				isLast:        isLast,
+			})
+		}
+		// keep the child nodes to visit later
+		paramsToVisit = append(childParams, paramsToVisit...)
+
+		// never process the root node
+		if currentParams.node == tree.Root {
+			currentRow--
+			continue
+		}
+
+		// process the current node
+		if currentRow >= startRow && currentRow <= stopRow {
+			params = append(params, currentParams)
+		}
+	}
+
+	// render the result
+	for idx := range params {
+		currentParams := params[idx]
+
+		if showAttributes {
+			result += currentParams.node.MetadataString() + " "
+		}
+		result += currentParams.node.renderTreeLine(currentParams.spaces, currentParams.isLast, currentParams.showCollapsed)
+	}
+
+	return result
+}
+
+func (tree *FileTree) VisibleSize() int {
+	var size int
+
+	visitor := func(node *FileNode) error {
+		size++
+		return nil
+	}
+	visitEvaluator := func(node *FileNode) bool {
+		if node.Data.FileInfo.IsDir {
+			// we won't visit a collapsed dir, but we need to count it
+			if node.Data.ViewInfo.Collapsed {
+				size++
+			}
+			return !node.Data.ViewInfo.Collapsed && !node.Data.ViewInfo.Hidden
+		}
+		return !node.Data.ViewInfo.Hidden
+	}
+	err := tree.VisitDepthParentFirst(visitor, visitEvaluator)
+	if err != nil {
+		logrus.Errorf("unable to determine visible tree size: %+v", err)
+	}
+
+	// don't include root
+	size--
+
+	return size
+}
+
+// String returns the entire tree in an ASCII representation.
+func (tree *FileTree) String(showAttributes bool) string {
+	return tree.renderStringTreeBetween(0, tree.Size, showAttributes)
+}
+
+// StringBetween returns a partial tree in an ASCII representation.
+func (tree *FileTree) StringBetween(start, stop int, showAttributes bool) string {
+	return tree.renderStringTreeBetween(start, stop, showAttributes)
+}
+
+// Copy returns a copy of the given FileTree
+func (tree *FileTree) Copy() *FileTree {
+	newTree := NewFileTree()
+	newTree.Size = tree.Size
+	newTree.FileSize = tree.FileSize
+	newTree.Root = tree.Root.Copy(newTree.Root)
+
+	// update the tree pointers
+	err := newTree.VisitDepthChildFirst(func(node *FileNode) error {
+		node.Tree = newTree
+		return nil
+	}, nil)
+
+	if err != nil {
+		logrus.Errorf("unable to propagate tree on copy(): %+v", err)
+	}
+
+	return newTree
+}
+
+// Visitor is a function that processes, observes, or otherwise transforms the given node
+type Visitor func(*FileNode) error
+
+// VisitEvaluator is a function that indicates whether the given node should be visited by a Visitor.
+type VisitEvaluator func(*FileNode) bool
+
+// VisitDepthChildFirst iterates the given tree depth-first, evaluating the deepest depths first (visit on bubble up)
+func (tree *FileTree) VisitDepthChildFirst(visitor Visitor, evaluator VisitEvaluator) error {
+	return tree.Root.VisitDepthChildFirst(visitor, evaluator)
+}
+
+// VisitDepthParentFirst iterates the given tree depth-first, evaluating the shallowest depths first (visit while sinking down)
+func (tree *FileTree) VisitDepthParentFirst(visitor Visitor, evaluator VisitEvaluator) error {
+	return tree.Root.VisitDepthParentFirst(visitor, evaluator)
+}
+
+// Stack takes two trees and combines them together. This is done by "stacking" the given tree on top of the owning tree.
+func (tree *FileTree) Stack(upper *FileTree) error {
+	graft := func(node *FileNode) error {
+		if node.IsWhiteout() {
+			err := tree.RemovePath(node.Path())
+			if err != nil {
+				return fmt.Errorf("cannot remove node %s: %v", node.Path(), err.Error())
+			}
+		} else {
+			newNode, _, err := tree.AddPath(node.Path(), node.Data.FileInfo)
+			if err != nil {
+				return fmt.Errorf("cannot add node %s: %v", newNode.Path(), err.Error())
+			}
+		}
+		return nil
+	}
+	return upper.VisitDepthChildFirst(graft, nil)
+}
+
+// GetNode fetches a single node when given a slash-delimited string from root ('/') to the desired node (e.g. '/a/node/path')
+func (tree *FileTree) GetNode(path string) (*FileNode, error) {
+	nodeNames := strings.Split(strings.Trim(path, "/"), "/")
+	node := tree.Root
+	for _, name := range nodeNames {
+		if name == "" {
+			continue
+		}
+		if node.Children[name] == nil {
+			return nil, fmt.Errorf("path does not exist: %s", path)
+		}
+		node = node.Children[name]
+	}
+	return node, nil
+}
+
+// AddPath adds a new node to the tree with the given payload
+func (tree *FileTree) AddPath(path string, data FileInfo) (*FileNode, []*FileNode, error) {
+	nodeNames := strings.Split(strings.Trim(path, "/"), "/")
+	node := tree.Root
+	addedNodes := make([]*FileNode, 0)
+	for idx, name := range nodeNames {
+		if name == "" {
+			continue
+		}
+		// find or create node
+		if node.Children[name] != nil {
+			node = node.Children[name]
+		} else {
+			// don't add paths that should be deleted
+			if strings.HasPrefix(name, doubleWhiteoutPrefix) {
+				return nil, addedNodes, nil
+			}
+
+			// don't attach the payload. The payload is destined for the
+			// Path's end node, not any intermediary node.
+			node = node.AddChild(name, FileInfo{})
+			addedNodes = append(addedNodes, node)
+
+			if node == nil {
+				// the child could not be added
+				return node, addedNodes, fmt.Errorf(fmt.Sprintf("could not add child node: '%s' (path:'%s')", name, path))
+			}
+		}
+
+		// attach payload to the last specified node
+		if idx == len(nodeNames)-1 {
+			node.Data.FileInfo = data
+		}
+
+	}
+	return node, addedNodes, nil
+}
+
+// RemovePath removes a node from the tree given its path.
+func (tree *FileTree) RemovePath(path string) error {
+	node, err := tree.GetNode(path)
+	if err != nil {
+		return err
+	}
+	return node.Remove()
+}
+
+type compareMark struct {
+	lowerNode *FileNode
+	upperNode *FileNode
+	tentative DiffType
+	final     DiffType
+}
+
+// CompareAndMark marks the FileNodes in the owning (lower) tree with DiffType annotations when compared to the given (upper) tree.
+func (tree *FileTree) CompareAndMark(upper *FileTree) error {
+	// always compare relative to the original, unaltered tree.
+	originalTree := tree
+
+	modifications := make([]compareMark, 0)
+
+	graft := func(upperNode *FileNode) error {
+		if upperNode.IsWhiteout() {
+			err := tree.markRemoved(upperNode.Path())
+			if err != nil {
+				return fmt.Errorf("cannot remove upperNode %s: %v", upperNode.Path(), err.Error())
+			}
+			return nil
+		}
+
+		// note: since we are not comparing against the original tree (copying the tree is expensive) we may mark the parent
+		// of an added node incorrectly as modified. This will be corrected later.
+		originalLowerNode, _ := originalTree.GetNode(upperNode.Path())
+
+		if originalLowerNode == nil {
+			_, newNodes, err := tree.AddPath(upperNode.Path(), upperNode.Data.FileInfo)
+			if err != nil {
+				return fmt.Errorf("cannot add new upperNode %s: %v", upperNode.Path(), err.Error())
+			}
+			for idx := len(newNodes) - 1; idx >= 0; idx-- {
+				newNode := newNodes[idx]
+				modifications = append(modifications, compareMark{lowerNode: newNode, upperNode: upperNode, tentative: -1, final: Added})
+			}
+			return nil
+		}
+
+		// the file exists in the lower layer
+		lowerNode, _ := tree.GetNode(upperNode.Path())
+		diffType := lowerNode.compare(upperNode)
+		modifications = append(modifications, compareMark{lowerNode: lowerNode, upperNode: upperNode, tentative: diffType, final: -1})
+
+		return nil
+	}
+	// we must visit from the leaves upwards to ensure that diff types can be derived from and assigned to children
+	err := upper.VisitDepthChildFirst(graft, nil)
+	if err != nil {
+		return err
+	}
+
+	// take note of the comparison results on each note in the owning tree.
+	for _, pair := range modifications {
+		if pair.final > 0 {
+			err = pair.lowerNode.AssignDiffType(pair.final)
+			if err != nil {
+				return err
+			}
+		} else if pair.lowerNode.Data.DiffType == Unmodified {
+			err = pair.lowerNode.deriveDiffType(pair.tentative)
+			if err != nil {
+				return err
+			}
+		}
+
+		// persist the upper's payload on the owning tree
+		pair.lowerNode.Data.FileInfo = *pair.upperNode.Data.FileInfo.Copy()
+	}
+	return nil
+}
+
+// markRemoved annotates the FileNode at the given path as Removed.
+func (tree *FileTree) markRemoved(path string) error {
+	node, err := tree.GetNode(path)
+	if err != nil {
+		return err
+	}
+	return node.AssignDiffType(Removed)
+}
+
+// StackTreeRange combines an array of trees into a single tree
+func StackTreeRange(trees []*FileTree, start, stop int) *FileTree {
+
+	tree := trees[0].Copy()
+	for idx := start; idx <= stop; idx++ {
+		err := tree.Stack(trees[idx])
+		if err != nil {
+			logrus.Errorf("could not stack tree range: %v", err)
+		}
+	}
+	return tree
+}

dive/filetree/file_tree_test.go

@@ -0,0 +1,791 @@
+package filetree
+
+import (
+	"fmt"
+	"testing"
+)
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+func AssertDiffType(node *FileNode, expectedDiffType DiffType) error {
+	if node.Data.DiffType != expectedDiffType {
+		return fmt.Errorf("Expecting node at %s to have DiffType %v, but had %v", node.Path(), expectedDiffType, node.Data.DiffType)
+	}
+	return nil
+}
+
+func TestStringCollapsed(t *testing.T) {
+	tree := NewFileTree()
+	tree.Root.AddChild("1 node!", FileInfo{})
+	two := tree.Root.AddChild("2 node!", FileInfo{})
+	subTwo := two.AddChild("2 child!", FileInfo{})
+	subTwo.AddChild("2 grandchild!", FileInfo{})
+	subTwo.Data.ViewInfo.Collapsed = true
+	three := tree.Root.AddChild("3 node!", FileInfo{})
+	subThree := three.AddChild("3 child!", FileInfo{})
+	three.AddChild("3 nested child 1!", FileInfo{})
+	threeGc1 := subThree.AddChild("3 grandchild 1!", FileInfo{})
+	threeGc1.AddChild("3 greatgrandchild 1!", FileInfo{})
+	subThree.AddChild("3 grandchild 2!", FileInfo{})
+	four := tree.Root.AddChild("4 node!", FileInfo{})
+	four.Data.ViewInfo.Collapsed = true
+	tree.Root.AddChild("5 node!", FileInfo{})
+	four.AddChild("6, one level down...", FileInfo{})
+
+	expected :=
+		`├── 1 node!
+├── 2 node!
+│   └─⊕ 2 child!
+├── 3 node!
+│   ├── 3 child!
+│   │   ├── 3 grandchild 1!
+│   │   │   └── 3 greatgrandchild 1!
+│   │   └── 3 grandchild 2!
+│   └── 3 nested child 1!
+├─⊕ 4 node!
+└── 5 node!
+`
+	actual := tree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestString(t *testing.T) {
+	tree := NewFileTree()
+	tree.Root.AddChild("1 node!", FileInfo{})
+	tree.Root.AddChild("2 node!", FileInfo{})
+	tree.Root.AddChild("3 node!", FileInfo{})
+	four := tree.Root.AddChild("4 node!", FileInfo{})
+	tree.Root.AddChild("5 node!", FileInfo{})
+	four.AddChild("6, one level down...", FileInfo{})
+
+	expected :=
+		`├── 1 node!
+├── 2 node!
+├── 3 node!
+├── 4 node!
+│   └── 6, one level down...
+└── 5 node!
+`
+	actual := tree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestStringBetween(t *testing.T) {
+	tree := NewFileTree()
+	_, _, err := tree.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/systemd", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	expected :=
+		`│       └── public
+├── tmp
+│   └── nonsense
+`
+	actual := tree.StringBetween(3, 5, false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestAddPath(t *testing.T) {
+	tree := NewFileTree()
+	_, _, err := tree.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/systemd", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	expected :=
+		`├── etc
+│   └── nginx
+│       ├── nginx.conf
+│       └── public
+├── tmp
+│   └── nonsense
+└── var
+    └── run
+        ├── bashful
+        └── systemd
+`
+	actual := tree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestAddWhiteoutPath(t *testing.T) {
+	tree := NewFileTree()
+	node, _, err := tree.AddPath("usr/local/lib/python3.7/site-packages/pip/.wh..wh..opq", FileInfo{})
+	if err != nil {
+		t.Errorf("expected no error but got: %v", err)
+	}
+	if node != nil {
+		t.Errorf("expected node to be nil, but got: %v", node)
+	}
+	expected :=
+		`└── usr
+    └── local
+        └── lib
+            └── python3.7
+                └── site-packages
+                    └── pip
+`
+	actual := tree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+}
+
+func TestRemovePath(t *testing.T) {
+	tree := NewFileTree()
+	_, _, err := tree.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/systemd", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	err = tree.RemovePath("/var/run/bashful")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	err = tree.RemovePath("/tmp")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	expected :=
+		`├── etc
+│   └── nginx
+│       ├── nginx.conf
+│       └── public
+└── var
+    └── run
+        └── systemd
+`
+	actual := tree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestStack(t *testing.T) {
+	payloadKey := "/var/run/systemd"
+	payloadValue := FileInfo{
+		Path: "yup",
+	}
+
+	tree1 := NewFileTree()
+
+	_, _, err := tree1.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree1.AddPath(payloadKey, FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree1.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree1.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree1.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	tree2 := NewFileTree()
+	// add new files
+	_, _, err = tree2.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	// modify current files
+	_, _, err = tree2.AddPath(payloadKey, payloadValue)
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	// whiteout the following files
+	_, _, err = tree2.AddPath("/var/run/.wh.bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree2.AddPath("/.wh.tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	// ignore opaque whiteout files entirely
+	node, _, err := tree2.AddPath("/.wh..wh..opq", FileInfo{})
+	if err != nil {
+		t.Errorf("expected no error on whiteout file add, but got %v", err)
+	}
+	if node != nil {
+		t.Errorf("expected no node on whiteout file add, but got %v", node)
+	}
+
+	err = tree1.Stack(tree2)
+
+	if err != nil {
+		t.Errorf("Could not stack refTrees: %v", err)
+	}
+
+	expected :=
+		`├── etc
+│   └── nginx
+│       ├── nginx.conf
+│       └── public
+└── var
+    └── run
+        └── systemd
+`
+
+	node, err = tree1.GetNode(payloadKey)
+	if err != nil {
+		t.Errorf("Expected '%s' to still exist, but it doesn't", payloadKey)
+	}
+
+	if node == nil || node.Data.FileInfo.Path != payloadValue.Path {
+		t.Errorf("Expected '%s' value to be %+v but got %+v", payloadKey, payloadValue, node.Data.FileInfo)
+	}
+
+	actual := tree1.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestCopy(t *testing.T) {
+	tree := NewFileTree()
+	_, _, err := tree.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/systemd", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	err = tree.RemovePath("/var/run/bashful")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	err = tree.RemovePath("/tmp")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	expected :=
+		`├── etc
+│   └── nginx
+│       ├── nginx.conf
+│       └── public
+└── var
+    └── run
+        └── systemd
+`
+
+	NewFileTree := tree.Copy()
+	actual := NewFileTree.String(false)
+
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}
+
+func TestCompareWithNoChanges(t *testing.T) {
+	lowerTree := NewFileTree()
+	upperTree := NewFileTree()
+	paths := [...]string{"/etc", "/etc/sudoers", "/etc/hosts", "/usr/bin", "/usr/bin/bash", "/usr"}
+
+	for _, value := range paths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		}
+		_, _, err := lowerTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+		_, _, err = upperTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+	err := lowerTree.CompareAndMark(upperTree)
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	asserter := func(n *FileNode) error {
+		if n.Path() == "/" {
+			return nil
+		}
+		if (n.Data.DiffType) != Unmodified {
+			t.Errorf("Expecting node at %s to have DiffType unchanged, but had %v", n.Path(), n.Data.DiffType)
+		}
+		return nil
+	}
+	err = lowerTree.VisitDepthChildFirst(asserter, nil)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestCompareWithAdds(t *testing.T) {
+	lowerTree := NewFileTree()
+	upperTree := NewFileTree()
+	lowerPaths := [...]string{"/etc", "/etc/sudoers", "/usr", "/etc/hosts", "/usr/bin"}
+	upperPaths := [...]string{"/etc", "/etc/sudoers", "/usr", "/etc/hosts", "/usr/bin", "/usr/bin/bash", "/a/new/path"}
+
+	for _, value := range lowerPaths {
+		_, _, err := lowerTree.AddPath(value, FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		})
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	for _, value := range upperPaths {
+		_, _, err := upperTree.AddPath(value, FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		})
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	failedAssertions := []error{}
+	err := lowerTree.CompareAndMark(upperTree)
+	if err != nil {
+		t.Errorf("Expected tree compare to have no errors, got: %v", err)
+	}
+	asserter := func(n *FileNode) error {
+
+		p := n.Path()
+		if p == "/" {
+			return nil
+		} else if stringInSlice(p, []string{"/usr/bin/bash", "/a", "/a/new", "/a/new/path"}) {
+			if err := AssertDiffType(n, Added); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		} else if stringInSlice(p, []string{"/usr/bin", "/usr"}) {
+			if err := AssertDiffType(n, Modified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		} else {
+			if err := AssertDiffType(n, Unmodified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		}
+		return nil
+	}
+	err = lowerTree.VisitDepthChildFirst(asserter, nil)
+	if err != nil {
+		t.Errorf("Expected no errors when visiting nodes, got: %+v", err)
+	}
+
+	if len(failedAssertions) > 0 {
+		str := "\n"
+		for _, value := range failedAssertions {
+			str += fmt.Sprintf("  - %s\n", value.Error())
+		}
+		t.Errorf("Expected no errors when evaluating nodes, got: %s", str)
+	}
+}
+
+func TestCompareWithChanges(t *testing.T) {
+	lowerTree := NewFileTree()
+	upperTree := NewFileTree()
+	changedPaths := []string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin"}
+
+	for _, value := range changedPaths {
+		_, _, err := lowerTree.AddPath(value, FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		})
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+		_, _, err = upperTree.AddPath(value, FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     456,
+		})
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	chmodPath := "/etc/non-data-change"
+
+	_, _, err := lowerTree.AddPath(chmodPath, FileInfo{
+		Path:     chmodPath,
+		TypeFlag: 1,
+		hash:     123,
+		Mode:     0,
+	})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	_, _, err = upperTree.AddPath(chmodPath, FileInfo{
+		Path:     chmodPath,
+		TypeFlag: 1,
+		hash:     123,
+		Mode:     1,
+	})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	changedPaths = append(changedPaths, chmodPath)
+
+	chownPath := "/etc/non-data-change-2"
+
+	_, _, err = lowerTree.AddPath(chmodPath, FileInfo{
+		Path:     chownPath,
+		TypeFlag: 1,
+		hash:     123,
+		Mode:     1,
+		Gid:      0,
+		Uid:      0,
+	})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	_, _, err = upperTree.AddPath(chmodPath, FileInfo{
+		Path:     chownPath,
+		TypeFlag: 1,
+		hash:     123,
+		Mode:     1,
+		Gid:      12,
+		Uid:      12,
+	})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	changedPaths = append(changedPaths, chownPath)
+
+	err = lowerTree.CompareAndMark(upperTree)
+	if err != nil {
+		t.Errorf("unable to compare and mark: %+v", err)
+	}
+
+	failedAssertions := []error{}
+	asserter := func(n *FileNode) error {
+		p := n.Path()
+		if p == "/" {
+			return nil
+		} else if stringInSlice(p, changedPaths) {
+			if err := AssertDiffType(n, Modified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		} else {
+			if err := AssertDiffType(n, Unmodified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		}
+		return nil
+	}
+	err = lowerTree.VisitDepthChildFirst(asserter, nil)
+	if err != nil {
+		t.Errorf("Expected no errors when visiting nodes, got: %+v", err)
+	}
+
+	if len(failedAssertions) > 0 {
+		str := "\n"
+		for _, value := range failedAssertions {
+			str += fmt.Sprintf("  - %s\n", value.Error())
+		}
+		t.Errorf("Expected no errors when evaluating nodes, got: %s", str)
+	}
+}
+
+func TestCompareWithRemoves(t *testing.T) {
+	lowerTree := NewFileTree()
+	upperTree := NewFileTree()
+	lowerPaths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin", "/root", "/root/example", "/root/example/some1", "/root/example/some2"}
+	upperPaths := [...]string{"/.wh.etc", "/usr", "/usr/.wh.bin", "/root/.wh.example"}
+
+	for _, value := range lowerPaths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		}
+		_, _, err := lowerTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	for _, value := range upperPaths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		}
+		_, _, err := upperTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	err := lowerTree.CompareAndMark(upperTree)
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	failedAssertions := []error{}
+	asserter := func(n *FileNode) error {
+		p := n.Path()
+		if p == "/" {
+			return nil
+		} else if stringInSlice(p, []string{"/etc", "/usr/bin", "/etc/hosts", "/etc/sudoers", "/root/example/some1", "/root/example/some2", "/root/example"}) {
+			if err := AssertDiffType(n, Removed); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		} else if stringInSlice(p, []string{"/usr", "/root"}) {
+			if err := AssertDiffType(n, Modified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		} else {
+			if err := AssertDiffType(n, Unmodified); err != nil {
+				failedAssertions = append(failedAssertions, err)
+			}
+		}
+		return nil
+	}
+	err = lowerTree.VisitDepthChildFirst(asserter, nil)
+	if err != nil {
+		t.Errorf("Expected no errors when visiting nodes, got: %+v", err)
+	}
+
+	if len(failedAssertions) > 0 {
+		str := "\n"
+		for _, value := range failedAssertions {
+			str += fmt.Sprintf("  - %s\n", value.Error())
+		}
+		t.Errorf("Expected no errors when evaluating nodes, got: %s", str)
+	}
+}
+
+func TestStackRange(t *testing.T) {
+	tree := NewFileTree()
+	_, _, err := tree.AddPath("/etc/nginx/nginx.conf", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/etc/nginx/public", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/systemd", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/var/run/bashful", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	_, _, err = tree.AddPath("/tmp/nonsense", FileInfo{})
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	err = tree.RemovePath("/var/run/bashful")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+	err = tree.RemovePath("/tmp")
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	lowerTree := NewFileTree()
+	upperTree := NewFileTree()
+	lowerPaths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin"}
+	upperPaths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin"}
+
+	for _, value := range lowerPaths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		}
+		_, _, err = lowerTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+
+	for _, value := range upperPaths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     456,
+		}
+		_, _, err = upperTree.AddPath(value, fakeData)
+		if err != nil {
+			t.Errorf("could not setup test: %v", err)
+		}
+	}
+	trees := []*FileTree{lowerTree, upperTree, tree}
+	StackTreeRange(trees, 0, 2)
+}
+
+func TestRemoveOnIterate(t *testing.T) {
+
+	tree := NewFileTree()
+	paths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin", "/usr/something"}
+
+	for _, value := range paths {
+		fakeData := FileInfo{
+			Path:     value,
+			TypeFlag: 1,
+			hash:     123,
+		}
+		node, _, err := tree.AddPath(value, fakeData)
+		if err == nil && stringInSlice(node.Path(), []string{"/etc"}) {
+			node.Data.ViewInfo.Hidden = true
+		}
+	}
+
+	err := tree.VisitDepthChildFirst(func(node *FileNode) error {
+		if node.Data.ViewInfo.Hidden {
+			err := tree.RemovePath(node.Path())
+			if err != nil {
+				t.Errorf("could not setup test: %v", err)
+			}
+		}
+		return nil
+	}, nil)
+	if err != nil {
+		t.Errorf("could not setup test: %v", err)
+	}
+
+	expected :=
+		`└── usr
+    ├── bin
+    └── something
+`
+	actual := tree.String(false)
+	if expected != actual {
+		t.Errorf("Expected tree string:\n--->%s<---\nGot:\n--->%s<---", expected, actual)
+	}
+
+}

dive/filetree/node_data.go

@@ -0,0 +1,28 @@
+package filetree
+
+var GlobalFileTreeCollapse bool
+
+// NodeData is the payload for a FileNode
+type NodeData struct {
+	ViewInfo ViewInfo
+	FileInfo FileInfo
+	DiffType DiffType
+}
+
+// NewNodeData creates an empty NodeData struct for a FileNode
+func NewNodeData() *NodeData {
+	return &NodeData{
+		ViewInfo: *NewViewInfo(),
+		FileInfo: FileInfo{},
+		DiffType: Unmodified,
+	}
+}
+
+// Copy duplicates a NodeData
+func (data *NodeData) Copy() *NodeData {
+	return &NodeData{
+		ViewInfo: *data.ViewInfo.Copy(),
+		FileInfo: *data.FileInfo.Copy(),
+		DiffType: data.DiffType,
+	}
+}

dive/filetree/node_data_test.go

@@ -0,0 +1,41 @@
+package filetree
+
+import (
+	"testing"
+)
+
+func TestAssignDiffType(t *testing.T) {
+	tree := NewFileTree()
+	node, _, err := tree.AddPath("/usr", *BlankFileChangeInfo("/usr"))
+	if err != nil {
+		t.Errorf("Expected no error from fetching path. got: %v", err)
+	}
+	node.Data.DiffType = Modified
+	if tree.Root.Children["usr"].Data.DiffType != Modified {
+		t.Fail()
+	}
+}
+
+func TestMergeDiffTypes(t *testing.T) {
+	a := Unmodified
+	b := Unmodified
+	merged := a.merge(b)
+	if merged != Unmodified {
+		t.Errorf("Expected Unchaged (0) but got %v", merged)
+	}
+	a = Modified
+	b = Unmodified
+	merged = a.merge(b)
+	if merged != Modified {
+		t.Errorf("Expected Unchaged (0) but got %v", merged)
+	}
+}
+
+func BlankFileChangeInfo(path string) (f *FileInfo) {
+	result := FileInfo{
+		Path:     path,
+		TypeFlag: 1,
+		hash:     123,
+	}
+	return &result
+}

dive/filetree/view_info.go

@@ -0,0 +1,22 @@
+package filetree
+
+// ViewInfo contains UI specific detail for a specific FileNode
+type ViewInfo struct {
+	Collapsed bool
+	Hidden    bool
+}
+
+// NewViewInfo creates a default ViewInfo
+func NewViewInfo() (view *ViewInfo) {
+	return &ViewInfo{
+		Collapsed: GlobalFileTreeCollapse,
+		Hidden:    false,
+	}
+}
+
+// Copy duplicates a ViewInfo
+func (view *ViewInfo) Copy() (newView *ViewInfo) {
+	newView = NewViewInfo()
+	*newView = *view
+	return newView
+}

dive/get_analyzer.go

@@ -0,0 +1,12 @@
+package dive
+
+import (
+	"github.com/wagoodman/dive/dive/image"
+	"github.com/wagoodman/dive/dive/image/docker"
+)
+
+func GetAnalyzer(imageID string) image.Analyzer {
+	// u, _ := url.Parse(imageID)
+	// fmt.Printf("\n\nurl: %+v\n", u.Scheme)
+	return docker.NewImageAnalyzer(imageID)
+}

dive/image/analyzer.go

@@ -0,0 +1,23 @@
+package image
+
+import (
+	"github.com/wagoodman/dive/dive/filetree"
+	"io"
+)
+
+type Analyzer interface {
+	Fetch() (io.ReadCloser, error)
+	Parse(io.ReadCloser) error
+	Analyze() (*AnalysisResult, error)
+}
+
+type AnalysisResult struct {
+	Layers            []Layer
+	RefTrees          []*filetree.FileTree
+	Efficiency        float64
+	SizeBytes         uint64
+	UserSizeByes      uint64  // this is all bytes except for the base image
+	WastedUserPercent float64 // = wasted-bytes/user-size-bytes
+	WastedBytes       uint64
+	Inefficiencies    filetree.EfficiencySlice
+}

dive/image/docker/analyzer.go

@@ -0,0 +1,272 @@
+package docker
+
+import (
+	"archive/tar"
+	"fmt"
+	"github.com/wagoodman/dive/dive/image"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/docker/cli/cli/connhelper"
+	"github.com/docker/docker/client"
+	"github.com/wagoodman/dive/dive/filetree"
+	"github.com/wagoodman/dive/utils"
+	"golang.org/x/net/context"
+)
+
+var dockerVersion string
+
+type imageAnalyzer struct {
+	id        string
+	client    *client.Client
+	jsonFiles map[string][]byte
+	trees     []*filetree.FileTree
+	layerMap  map[string]*filetree.FileTree
+	layers    []*dockerLayer
+}
+
+func NewImageAnalyzer(imageId string) *imageAnalyzer {
+	return &imageAnalyzer{
+		// store discovered json files in a map so we can read the image in one pass
+		jsonFiles: make(map[string][]byte),
+		layerMap:  make(map[string]*filetree.FileTree),
+		id:        imageId,
+	}
+}
+
+func (img *imageAnalyzer) Fetch() (io.ReadCloser, error) {
+	var err error
+
+	// pull the img if it does not exist
+	ctx := context.Background()
+
+	host := os.Getenv("DOCKER_HOST")
+	var clientOpts []func(*client.Client) error
+
+	switch strings.Split(host, ":")[0] {
+	case "ssh":
+		helper, err := connhelper.GetConnectionHelper(host)
+		if err != nil {
+			fmt.Println("docker host", err)
+		}
+		clientOpts = append(clientOpts, func(c *client.Client) error {
+			httpClient := &http.Client{
+				Transport: &http.Transport{
+					DialContext: helper.Dialer,
+				},
+			}
+			return client.WithHTTPClient(httpClient)(c)
+		})
+		clientOpts = append(clientOpts, client.WithHost(helper.Host))
+		clientOpts = append(clientOpts, client.WithDialContext(helper.Dialer))
+
+	default:
+
+		if os.Getenv("DOCKER_TLS_VERIFY") != "" && os.Getenv("DOCKER_CERT_PATH") == "" {
+			os.Setenv("DOCKER_CERT_PATH", "~/.docker")
+		}
+
+		clientOpts = append(clientOpts, client.FromEnv)
+	}
+
+	clientOpts = append(clientOpts, client.WithVersion(dockerVersion))
+	img.client, err = client.NewClientWithOpts(clientOpts...)
+	if err != nil {
+		return nil, err
+	}
+	_, _, err = img.client.ImageInspectWithRaw(ctx, img.id)
+	if err != nil {
+
+		if !utils.IsDockerClientAvailable() {
+			return nil, fmt.Errorf("cannot find docker client executable")
+		}
+
+		// don't use the API, the CLI has more informative output
+		fmt.Println("Image not available locally. Trying to pull '" + img.id + "'...")
+		err = utils.RunDockerCmd("pull", img.id)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	readCloser, err := img.client.ImageSave(ctx, []string{img.id})
+	if err != nil {
+		return nil, err
+	}
+
+	return readCloser, nil
+}
+
+func (img *imageAnalyzer) Parse(tarFile io.ReadCloser) error {
+	tarReader := tar.NewReader(tarFile)
+
+	var currentLayer uint
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			fmt.Println(err)
+			utils.Exit(1)
+		}
+
+		name := header.Name
+
+		// some layer tars can be relative layer symlinks to other layer tars
+		if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeReg {
+
+			if strings.HasSuffix(name, "layer.tar") {
+				currentLayer++
+				if err != nil {
+					return err
+				}
+				layerReader := tar.NewReader(tarReader)
+				err := img.processLayerTar(name, currentLayer, layerReader)
+				if err != nil {
+					return err
+				}
+			} else if strings.HasSuffix(name, ".json") {
+				fileBuffer, err := ioutil.ReadAll(tarReader)
+				if err != nil {
+					return err
+				}
+				img.jsonFiles[name] = fileBuffer
+			}
+		}
+	}
+
+	return nil
+}
+
+func (img *imageAnalyzer) Analyze() (*image.AnalysisResult, error) {
+	img.trees = make([]*filetree.FileTree, 0)
+
+	manifest := newDockerImageManifest(img.jsonFiles["manifest.json"])
+	config := newDockerImageConfig(img.jsonFiles[manifest.ConfigPath])
+
+	// build the content tree
+	for _, treeName := range manifest.LayerTarPaths {
+		img.trees = append(img.trees, img.layerMap[treeName])
+	}
+
+	// build the layers array
+	img.layers = make([]*dockerLayer, len(img.trees))
+
+	// note that the img config stores images in reverse chronological order, so iterate backwards through layers
+	// as you iterate chronologically through history (ignoring history items that have no layer contents)
+	// Note: history is not required metadata in a docker img!
+	tarPathIdx := 0
+	histIdx := 0
+	for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- {
+
+		tree := img.trees[(len(img.trees)-1)-layerIdx]
+
+		// ignore empty layers, we are only observing layers with content
+		historyObj := imageHistoryEntry{
+			CreatedBy: "(missing)",
+		}
+		for nextHistIdx := histIdx; nextHistIdx < len(config.History); nextHistIdx++ {
+			if !config.History[nextHistIdx].EmptyLayer {
+				histIdx = nextHistIdx
+				break
+			}
+		}
+		if histIdx < len(config.History) && !config.History[histIdx].EmptyLayer {
+			historyObj = config.History[histIdx]
+			histIdx++
+		}
+
+		img.layers[layerIdx] = &dockerLayer{
+			history: historyObj,
+			index:   tarPathIdx,
+			tree:    img.trees[layerIdx],
+			tarPath: manifest.LayerTarPaths[tarPathIdx],
+		}
+		img.layers[layerIdx].history.Size = tree.FileSize
+
+		tarPathIdx++
+	}
+
+	efficiency, inefficiencies := filetree.Efficiency(img.trees)
+
+	var sizeBytes, userSizeBytes uint64
+	layers := make([]image.Layer, len(img.layers))
+	for i, v := range img.layers {
+		layers[i] = v
+		sizeBytes += v.Size()
+		if i != 0 {
+			userSizeBytes += v.Size()
+		}
+	}
+
+	var wastedBytes uint64
+	for idx := 0; idx < len(inefficiencies); idx++ {
+		fileData := inefficiencies[len(inefficiencies)-1-idx]
+		wastedBytes += uint64(fileData.CumulativeSize)
+	}
+
+	return &image.AnalysisResult{
+		Layers:            layers,
+		RefTrees:          img.trees,
+		Efficiency:        efficiency,
+		UserSizeByes:      userSizeBytes,
+		SizeBytes:         sizeBytes,
+		WastedBytes:       wastedBytes,
+		WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes),
+		Inefficiencies:    inefficiencies,
+	}, nil
+}
+
+func (img *imageAnalyzer) processLayerTar(name string, layerIdx uint, reader *tar.Reader) error {
+	tree := filetree.NewFileTree()
+	tree.Name = name
+
+	fileInfos, err := img.getFileList(reader)
+	if err != nil {
+		return err
+	}
+
+	for _, element := range fileInfos {
+		tree.FileSize += uint64(element.Size)
+
+		_, _, err := tree.AddPath(element.Path, element)
+		if err != nil {
+			return err
+		}
+	}
+
+	img.layerMap[tree.Name] = tree
+	return nil
+}
+
+func (img *imageAnalyzer) getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
+	var files []filetree.FileInfo
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			fmt.Println(err)
+			utils.Exit(1)
+		}
+
+		name := header.Name
+
+		switch header.Typeflag {
+		case tar.TypeXGlobalHeader:
+			return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name)
+		case tar.TypeXHeader:
+			return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name)
+		default:
+			files = append(files, filetree.NewFileInfo(tarReader, header, name))
+		}
+	}
+	return files, nil
+}

dive/image/docker/image_config.go

@@ -0,0 +1,36 @@
+package docker
+
+import (
+	"encoding/json"
+	"github.com/sirupsen/logrus"
+)
+
+type imageConfig struct {
+	History []imageHistoryEntry `json:"history"`
+	RootFs  rootFs              `json:"rootfs"`
+}
+
+type rootFs struct {
+	Type    string   `json:"type"`
+	DiffIds []string `json:"diff_ids"`
+}
+
+func newDockerImageConfig(configBytes []byte) imageConfig {
+	var imageConfig imageConfig
+	err := json.Unmarshal(configBytes, &imageConfig)
+	if err != nil {
+		logrus.Panic(err)
+	}
+
+	layerIdx := 0
+	for idx := range imageConfig.History {
+		if imageConfig.History[idx].EmptyLayer {
+			imageConfig.History[idx].ID = "<missing>"
+		} else {
+			imageConfig.History[idx].ID = imageConfig.RootFs.DiffIds[layerIdx]
+			layerIdx++
+		}
+	}
+
+	return imageConfig
+}

dive/image/docker/image_manifest.go

@@ -0,0 +1,21 @@
+package docker
+
+import (
+	"encoding/json"
+	"github.com/sirupsen/logrus"
+)
+
+type imageManifest struct {
+	ConfigPath    string   `json:"Config"`
+	RepoTags      []string `json:"RepoTags"`
+	LayerTarPaths []string `json:"Layers"`
+}
+
+func newDockerImageManifest(manifestBytes []byte) imageManifest {
+	var manifest []imageManifest
+	err := json.Unmarshal(manifestBytes, &manifest)
+	if err != nil {
+		logrus.Panic(err)
+	}
+	return manifest[0]
+}

dive/image/docker/layer.go

@@ -0,0 +1,95 @@
+package docker
+
+import (
+	"fmt"
+	"github.com/wagoodman/dive/dive/image"
+	"strings"
+
+	"github.com/dustin/go-humanize"
+	"github.com/wagoodman/dive/dive/filetree"
+)
+
+// Layer represents a Docker image layer and metadata
+type dockerLayer struct {
+	tarPath string
+	history imageHistoryEntry
+	index   int
+	tree    *filetree.FileTree
+}
+
+type imageHistoryEntry struct {
+	ID         string
+	Size       uint64
+	Created    string `json:"created"`
+	Author     string `json:"author"`
+	CreatedBy  string `json:"created_by"`
+	EmptyLayer bool   `json:"empty_layer"`
+}
+
+// ShortId returns the truncated id of the current layer.
+func (layer *dockerLayer) TarId() string {
+	return strings.TrimSuffix(layer.tarPath, "/layer.tar")
+}
+
+// ShortId returns the truncated id of the current layer.
+func (layer *dockerLayer) Id() string {
+	return layer.history.ID
+}
+
+// index returns the relative position of the layer within the image.
+func (layer *dockerLayer) Index() int {
+	return layer.index
+}
+
+// Size returns the number of bytes that this image is.
+func (layer *dockerLayer) Size() uint64 {
+	return layer.history.Size
+}
+
+// Tree returns the file tree representing the current layer.
+func (layer *dockerLayer) Tree() *filetree.FileTree {
+	return layer.tree
+}
+
+// ShortId returns the truncated id of the current layer.
+func (layer *dockerLayer) Command() string {
+	return strings.TrimPrefix(layer.history.CreatedBy, "/bin/sh -c ")
+}
+
+// ShortId returns the truncated id of the current layer.
+func (layer *dockerLayer) ShortId() string {
+	rangeBound := 15
+	id := layer.Id()
+	if length := len(id); length < 15 {
+		rangeBound = length
+	}
+	id = id[0:rangeBound]
+
+	// show the tagged image as the last layer
+	// if len(layer.History.Tags) > 0 {
+	// 	id = "[" + strings.Join(layer.History.Tags, ",") + "]"
+	// }
+
+	return id
+}
+
+func (layer *dockerLayer) StringFormat() string {
+	return image.LayerFormat
+}
+
+// String represents a layer in a columnar format.
+func (layer *dockerLayer) String() string {
+
+	if layer.index == 0 {
+		return fmt.Sprintf(image.LayerFormat,
+			// layer.ShortId(),
+			// fmt.Sprintf("%d",layer.Index()),
+			humanize.Bytes(layer.Size()),
+			"FROM "+layer.ShortId())
+	}
+	return fmt.Sprintf(image.LayerFormat,
+		// layer.ShortId(),
+		// fmt.Sprintf("%d",layer.Index()),
+		humanize.Bytes(layer.Size()),
+		layer.Command())
+}

dive/image/docker/testing.go

@@ -0,0 +1,20 @@
+package docker
+
+import (
+	"github.com/wagoodman/dive/dive/image"
+	"os"
+)
+
+func TestLoadDockerImageTar(tarPath string) (*image.AnalysisResult, error) {
+	f, err := os.Open(tarPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	analyzer := NewImageAnalyzer("dive-test:latest")
+	err = analyzer.Parse(f)
+	if err != nil {
+		return nil, err
+	}
+	return analyzer.Analyze()
+}

dive/image/layer.go

@@ -0,0 +1,19 @@
+package image
+
+import (
+	"github.com/wagoodman/dive/dive/filetree"
+)
+
+const (
+	LayerFormat = "%7s  %s"
+)
+
+type Layer interface {
+	Id() string
+	ShortId() string
+	Index() int
+	Command() string
+	Size() uint64
+	Tree() *filetree.FileTree
+	String() string
+}