gitea/routers/repo/migrate.go
KN4CK3R c03e488e14
Add LFS Migration and Mirror (#14726)
* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 18:25:57 -04:00

255 lines
8.7 KiB
Go

// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"net/http"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/task"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/services/forms"
)
const (
tplMigrate base.TplName = "repo/migrate/migrate"
)
// Migrate render migration of repository page
func Migrate(ctx *context.Context) {
if setting.Repository.DisableMigrations {
ctx.Error(http.StatusForbidden, "Migrate: the site administrator has disabled migrations")
return
}
serviceType := structs.GitServiceType(ctx.QueryInt("service_type"))
setMigrationContextData(ctx, serviceType)
if serviceType == 0 {
ctx.Data["Org"] = ctx.Query("org")
ctx.Data["Mirror"] = ctx.Query("mirror")
ctx.HTML(http.StatusOK, tplMigrate)
return
}
ctx.Data["private"] = getRepoPrivate(ctx)
ctx.Data["mirror"] = ctx.Query("mirror") == "1"
ctx.Data["lfs"] = ctx.Query("lfs") == "1"
ctx.Data["wiki"] = ctx.Query("wiki") == "1"
ctx.Data["milestones"] = ctx.Query("milestones") == "1"
ctx.Data["labels"] = ctx.Query("labels") == "1"
ctx.Data["issues"] = ctx.Query("issues") == "1"
ctx.Data["pull_requests"] = ctx.Query("pull_requests") == "1"
ctx.Data["releases"] = ctx.Query("releases") == "1"
ctxUser := checkContextUser(ctx, ctx.QueryInt64("org"))
if ctx.Written() {
return
}
ctx.Data["ContextUser"] = ctxUser
ctx.HTML(http.StatusOK, base.TplName("repo/migrate/"+serviceType.Name()))
}
func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *forms.MigrateRepoForm) {
if setting.Repository.DisableMigrations {
ctx.Error(http.StatusForbidden, "MigrateError: the site administrator has disabled migrations")
return
}
switch {
case migrations.IsRateLimitError(err):
ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
case migrations.IsTwoFactorAuthError(err):
ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form)
case models.IsErrReachLimitOfRepo(err):
ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
case models.IsErrRepoAlreadyExist(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form)
case models.IsErrRepoFilesAlreadyExist(err):
ctx.Data["Err_RepoName"] = true
switch {
case ctx.IsUserSiteAdmin() || (setting.Repository.AllowAdoptionOfUnadoptedRepositories && setting.Repository.AllowDeleteOfUnadoptedRepositories):
ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.adopt_or_delete"), tpl, form)
case setting.Repository.AllowAdoptionOfUnadoptedRepositories:
ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.adopt"), tpl, form)
case setting.Repository.AllowDeleteOfUnadoptedRepositories:
ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.delete"), tpl, form)
default:
ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist"), tpl, form)
}
case models.IsErrNameReserved(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form)
case models.IsErrNamePatternNotAllowed(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
default:
remoteAddr, _ := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
err = util.URLSanitizedError(err, remoteAddr)
if strings.Contains(err.Error(), "Authentication failed") ||
strings.Contains(err.Error(), "Bad credentials") ||
strings.Contains(err.Error(), "could not read Username") {
ctx.Data["Err_Auth"] = true
ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form)
} else if strings.Contains(err.Error(), "fatal:") {
ctx.Data["Err_CloneAddr"] = true
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form)
} else {
ctx.ServerError(name, err)
}
}
}
func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplName, form *forms.MigrateRepoForm) {
if models.IsErrInvalidCloneAddr(err) {
addrErr := err.(*models.ErrInvalidCloneAddr)
switch {
case addrErr.IsProtocolInvalid:
ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, form)
case addrErr.IsURLError:
ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
case addrErr.IsPermissionDenied:
if addrErr.LocalPath {
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
} else if len(addrErr.PrivateNet) == 0 {
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
} else {
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form)
}
case addrErr.IsInvalidPath:
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
default:
log.Error("Error whilst updating url: %v", err)
ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
}
} else {
log.Error("Error whilst updating url: %v", err)
ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
}
}
// MigratePost response for migrating from external git repository
func MigratePost(ctx *context.Context) {
form := web.GetForm(ctx).(*forms.MigrateRepoForm)
if setting.Repository.DisableMigrations {
ctx.Error(http.StatusForbidden, "MigratePost: the site administrator has disabled migrations")
return
}
serviceType := structs.GitServiceType(form.Service)
setMigrationContextData(ctx, serviceType)
ctxUser := checkContextUser(ctx, form.UID)
if ctx.Written() {
return
}
ctx.Data["ContextUser"] = ctxUser
tpl := base.TplName("repo/migrate/" + serviceType.Name())
if ctx.HasError() {
ctx.HTML(http.StatusOK, tpl)
return
}
remoteAddr, err := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
if err == nil {
err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
}
if err != nil {
ctx.Data["Err_CloneAddr"] = true
handleMigrateRemoteAddrError(ctx, err, tpl, form)
return
}
form.LFS = form.LFS && setting.LFS.StartServer
if form.LFS && len(form.LFSEndpoint) > 0 {
ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
if ep == nil {
ctx.Data["Err_LFSEndpoint"] = true
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form)
return
}
err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
if err != nil {
ctx.Data["Err_LFSEndpoint"] = true
handleMigrateRemoteAddrError(ctx, err, tpl, form)
return
}
}
var opts = migrations.MigrateOptions{
OriginalURL: form.CloneAddr,
GitServiceType: serviceType,
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
LFS: form.LFS,
LFSEndpoint: form.LFSEndpoint,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Wiki: form.Wiki,
Issues: form.Issues,
Milestones: form.Milestones,
Labels: form.Labels,
Comments: form.Issues || form.PullRequests,
PullRequests: form.PullRequests,
Releases: form.Releases,
}
if opts.Mirror {
opts.Issues = false
opts.Milestones = false
opts.Labels = false
opts.Comments = false
opts.PullRequests = false
opts.Releases = false
}
err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, false)
if err != nil {
handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form)
return
}
err = task.MigrateRepository(ctx.User, ctxUser, opts)
if err == nil {
ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName)
return
}
handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form)
}
func setMigrationContextData(ctx *context.Context, serviceType structs.GitServiceType) {
ctx.Data["Title"] = ctx.Tr("new_migrate")
ctx.Data["LFSActive"] = setting.LFS.StartServer
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
// Plain git should be first
ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...)
ctx.Data["service"] = serviceType
}