gitea/routers/repo/lfs.go
zeripath 5e6a008fba
Add basic repository lfs management (#7199)
This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions.

* Add basic repository lfs management
* add auto-associate function
* Add functionality to find commits with this lfs file
* Add link to find commits on the lfs file view
* Adjust commit view to state the likely branch causing the commit
* Only read Oid from database
2019-10-28 18:31:55 +00:00

552 lines
15 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"bufio"
"bytes"
"fmt"
gotemplate "html/template"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/pipeline"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/mcuadros/go-version"
"github.com/unknwon/com"
gogit "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
)
const (
tplSettingsLFS base.TplName = "repo/settings/lfs"
tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
)
// LFSFiles shows a repository's LFS files
func LFSFiles(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFiles", nil)
return
}
page := ctx.QueryInt("page")
if page <= 1 {
page = 1
}
total, err := ctx.Repo.Repository.CountLFSMetaObjects()
if err != nil {
ctx.ServerError("LFSFiles", err)
return
}
pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
ctx.Data["PageIsSettingsLFS"] = true
lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
if err != nil {
ctx.ServerError("LFSFiles", err)
return
}
ctx.Data["LFSFiles"] = lfsMetaObjects
ctx.Data["Page"] = pager
ctx.HTML(200, tplSettingsLFS)
}
// LFSFileGet serves a single LFS file
func LFSFileGet(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
oid := ctx.Params("oid")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
if err != nil {
if err == models.ErrLFSObjectNotExist {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.ServerError("LFSFileGet", err)
return
}
ctx.Data["LFSFile"] = meta
dataRc, err := lfs.ReadMetaObject(meta)
if err != nil {
ctx.ServerError("LFSFileGet", err)
return
}
defer dataRc.Close()
buf := make([]byte, 1024)
n, err := dataRc.Read(buf)
if err != nil {
ctx.ServerError("Data", err)
return
}
buf = buf[:n]
isTextFile := base.IsTextFile(buf)
ctx.Data["IsTextFile"] = isTextFile
fileSize := meta.Size
ctx.Data["FileSize"] = meta.Size
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
switch {
case isTextFile:
if fileSize >= setting.UI.MaxDisplayFileSize {
ctx.Data["IsFileTooLarge"] = true
break
}
d, _ := ioutil.ReadAll(dataRc)
buf = charset.ToUTF8WithFallback(append(buf, d...))
// Building code view blocks with line number on server side.
var fileContent string
if content, err := charset.ToUTF8WithErr(buf); err != nil {
log.Error("ToUTF8WithErr: %v", err)
fileContent = string(buf)
} else {
fileContent = content
}
var output bytes.Buffer
lines := strings.Split(fileContent, "\n")
//Remove blank line at the end of file
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
for index, line := range lines {
line = gotemplate.HTMLEscapeString(line)
if index != len(lines)-1 {
line += "\n"
}
output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
}
ctx.Data["FileContent"] = gotemplate.HTML(output.String())
output.Reset()
for i := 0; i < len(lines); i++ {
output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
}
ctx.Data["LineNums"] = gotemplate.HTML(output.String())
case base.IsPDFFile(buf):
ctx.Data["IsPDFFile"] = true
case base.IsVideoFile(buf):
ctx.Data["IsVideoFile"] = true
case base.IsAudioFile(buf):
ctx.Data["IsAudioFile"] = true
case base.IsImageFile(buf):
ctx.Data["IsImageFile"] = true
}
ctx.HTML(200, tplSettingsLFSFile)
}
// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
func LFSDelete(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSDelete", nil)
return
}
oid := ctx.Params("oid")
count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
if err != nil {
ctx.ServerError("LFSDelete", err)
return
}
// FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
// Please note a similar condition happens in models/repo.go DeleteRepository
if count == 0 {
oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:])
err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath))
if err != nil {
ctx.ServerError("LFSDelete", err)
return
}
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}
type lfsResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []plumbing.Hash
BranchName string
FullCommitName string
}
type lfsResultSlice []*lfsResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
func LFSFileFind(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFind", nil)
return
}
oid := ctx.Query("oid")
size := ctx.QueryInt64("size")
if len(oid) == 0 || size == 0 {
ctx.NotFound("LFSFind", nil)
return
}
sha := ctx.Query("sha")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
var hash plumbing.Hash
if len(sha) == 0 {
meta := models.LFSMetaObject{Oid: oid, Size: size}
pointer := meta.Pointer()
hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
sha = hash.String()
} else {
hash = plumbing.NewHash(sha)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
ctx.Data["Oid"] = oid
ctx.Data["Size"] = size
ctx.Data["SHA"] = sha
resultsMap := map[string]*lfsResult{}
results := make([]*lfsResult, 0)
basePath := ctx.Repo.Repository.RepoPath()
gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
Order: gogit.LogOrderCommitterTime,
All: true,
})
if err != nil {
log.Error("Failed to get GoGit CommitsIter: %v", err)
ctx.ServerError("LFSFind: Iterate Commits", err)
return
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
tree, err := gitCommit.Tree()
if err != nil {
return err
}
treeWalker := object.NewTreeWalker(tree, true, nil)
defer treeWalker.Close()
for {
name, entry, err := treeWalker.Next()
if err == io.EOF {
break
}
if entry.Hash == hash {
result := lfsResult{
Name: name,
SHA: gitCommit.Hash.String(),
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
When: gitCommit.Author.When,
ParentHashes: gitCommit.ParentHashes,
}
resultsMap[gitCommit.Hash.String()+":"+name] = &result
}
}
return nil
})
if err != nil && err != io.EOF {
log.Error("Failure in CommitIter.ForEach: %v", err)
ctx.ServerError("LFSFind: IterateCommits ForEach", err)
return
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
ctx.ServerError("LFSPointerFiles", err)
}
default:
}
ctx.Data["Results"] = results
ctx.HTML(200, tplSettingsLFSFileFind)
}
// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
func LFSPointerFiles(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.Data["PageIsSettingsLFS"] = true
binVersion, err := git.BinVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
basePath := ctx.Repo.Repository.RepoPath()
pointerChan := make(chan pointerResult)
catFileCheckReader, catFileCheckWriter := io.Pipe()
shasToBatchReader, shasToBatchWriter := io.Pipe()
catFileBatchReader, catFileBatchWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(5)
var numPointers, numAssociated, numNoExist, numAssociatable int
go func() {
defer wg.Done()
pointers := make([]pointerResult, 0, 50)
for pointer := range pointerChan {
pointers = append(pointers, pointer)
if pointer.InRepo {
numAssociated++
}
if !pointer.Exists {
numNoExist++
}
if !pointer.InRepo && pointer.Accessible {
numAssociatable++
}
}
numPointers = len(pointers)
ctx.Data["Pointers"] = pointers
ctx.Data["NumPointers"] = numPointers
ctx.Data["NumAssociated"] = numAssociated
ctx.Data["NumAssociatable"] = numAssociatable
ctx.Data["NumNoExist"] = numNoExist
ctx.Data["NumNotAssociated"] = numPointers - numAssociated
}()
go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
if !version.Compare(binVersion, "2.6.0", ">=") {
revListReader, revListWriter := io.Pipe()
shasToCheckReader, shasToCheckWriter := io.Pipe()
wg.Add(2)
go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
} else {
go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
}
wg.Wait()
select {
case err, has := <-errChan:
if has {
ctx.ServerError("LFSPointerFiles", err)
}
default:
}
ctx.HTML(200, tplSettingsLFSPointers)
}
type pointerResult struct {
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
}
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
defer wg.Done()
defer catFileBatchReader.Close()
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
bufferedReader := bufio.NewReader(catFileBatchReader)
buf := make([]byte, 1025)
for {
// File descriptor line: sha
sha, err := bufferedReader.ReadString(' ')
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
// Throw away the blob
if _, err := bufferedReader.ReadString(' '); err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
sizeStr, err := bufferedReader.ReadString('\n')
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
pointerBuf := buf[:size+1]
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
pointerBuf = pointerBuf[:size]
// Now we need to check if the pointerBuf is an LFS pointer
pointer := lfs.IsPointerFile(&pointerBuf)
if pointer == nil {
continue
}
result := pointerResult{
SHA: strings.TrimSpace(sha),
Oid: pointer.Oid,
Size: pointer.Size,
}
// Then we need to check that this pointer is in the db
if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
if err != models.ErrLFSObjectNotExist {
_ = catFileBatchReader.CloseWithError(err)
break
}
} else {
result.InRepo = true
}
result.Exists = contentStore.Exists(pointer)
if result.Exists {
if !result.InRepo {
// Can we fix?
// OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
} else {
result.Accessible = true
}
}
pointerChan <- result
}
close(pointerChan)
}
// LFSAutoAssociate auto associates accessible lfs files
func LFSAutoAssociate(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSAutoAssociate", nil)
return
}
oids := ctx.QueryStrings("oid")
metas := make([]*models.LFSMetaObject, len(oids))
for i, oid := range oids {
idx := strings.IndexRune(oid, ' ')
if idx < 0 || idx+1 > len(oid) {
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
return
}
var err error
metas[i] = &models.LFSMetaObject{}
metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64()
if err != nil {
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
return
}
metas[i].Oid = oid[:idx]
//metas[i].RepositoryID = ctx.Repo.Repository.ID
}
if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
ctx.ServerError("LFSAutoAssociate", err)
return
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}