diff --git a/ReadMe.md b/ReadMe.md index 5ce18c5..afb3151 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,4 +1,9 @@ -## matrix-synapse-state-groups-state-janitor +## matrix-synapse-diskspace-janitor + +![scruffy the janitor from futurama](frontend/static/images/scruffy.png) + +_toilets and boilers, boilers and toilets_ + Matrix-synapse (the matrix homeserver implementation) requires a postgres database server to operate. diff --git a/config.json b/config.json index a747fbb..b08fe1e 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,14 @@ { + "FrontendPort": 6712, + "FrontendDomain": "matrix-diskspace-janitor.cyberia.club", + "MatrixServerPublicDomain": "cyberia.club", "MatrixURL": "http://localhost:8080", "MatrixAdminToken": "changeme", "DatabaseType": "postgres", - "DatabaseConnectionString": "host=localhost port=5432 user=synapse_user password=changeme database=synapse sslmode=disable" + + "DatabaseConnectionString": + "host=localhost port=5432 user=synapse_user password=changeme database=synapse sslmode=disable", + + "MediaFolder": "/var/lib/matrix-synapse", + "PostgresFolder": "/var/lib/postgresql" } \ No newline at end of file diff --git a/db_model.go b/db_model.go index a8e3347..be61375 100644 --- a/db_model.go +++ b/db_model.go @@ -206,7 +206,7 @@ func (model *DBModel) DeleteStateGroupsState(stateGroupIds []int64, startAt int) } // https://dataedo.com/kb/query/postgresql/list-of-tables-by-their-size -func (model *DBModel) GetDBTableSizes(roomId string) (tables []DBTableSize, err error) { +func (model *DBModel) GetDBTableSizes() (tables []DBTableSize, err error) { rows, err := model.DB.Query( `select schemaname as table_schema, relname as table_name, pg_relation_size(relid) as data_size diff --git a/frontend.go b/frontend.go index 139597f..56e2f8f 100644 --- a/frontend.go +++ b/frontend.go @@ -1,2 +1,304 @@ +package main +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "html/template" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + errors "git.sequentialread.com/forest/pkg-errors" + + "github.com/shengdoushi/base58" +) + +type Session struct { + SessionId string + UserID string + ExpiresUnixMilli int64 + Flash *map[string]string +} + +type FrontendApp struct { + Port int + Domain string + Router *http.ServeMux + HTMLTemplates map[string]*template.Template + cssHash string + basicURLPathRegex *regexp.Regexp + base58Regex *regexp.Regexp +} + +func initFrontend(config *Config) FrontendApp { + + cssBytes, err := os.ReadFile(filepath.Join(".", "frontend", "static", "app.css")) + if err != nil { + panic(errors.Wrap(err, "can't initFrontend because can't read cssBytes:")) + } + hashArray := sha256.Sum256(cssBytes) + cssHash := base58.Encode(hashArray[:6], base58.BitcoinAlphabet) + + app := FrontendApp{ + Port: config.FrontendPort, + Domain: config.FrontendDomain, + Router: http.NewServeMux(), + HTMLTemplates: map[string]*template.Template{}, + basicURLPathRegex: regexp.MustCompile("(?i)[a-z0-9/?&_+-]+"), + base58Regex: regexp.MustCompile("(?i)[a-z0-9_-]+"), + cssHash: cssHash, + } + + // serve the homepage + app.handleWithSession("/", func(responseWriter http.ResponseWriter, request *http.Request, session Session) { + + userIsLoggedIn := session.UserID != "" + if userIsLoggedIn { + app.buildPageFromTemplate(responseWriter, request, session, "panel.html", nil) + } else { + if request.Method == "POST" { + username := request.PostFormValue("username") + password := request.PostFormValue("password") + + success, err := matrixAdmin.Login(username, password) + if err != nil { + (*session.Flash)["error"] += "an error was thrown by the login process 😧" + log.Println(errors.Wrap(err, "an error was thrown by the login process")) + } else if success { + session.UserID = username + session.ExpiresUnixMilli = time.Now().Add(time.Hour * 24).UnixMilli() + err = app.setSession(responseWriter, &session) + if err != nil { + log.Println(errors.Wrap(err, "setSession failed")) + } + } + } + + app.buildPageFromTemplate(responseWriter, request, session, "login.html", nil) + } + }) + + // registerHowtoRoutes(&app) + + // registerLoginRoutes(&app, emailService) + + // registerProfileRoutes(&app) + + // registerAdminPanelRoutes(&app) + + app.reloadTemplates() + + staticFilesDir := "./frontend/static" + log.Printf("serving static files from %s", staticFilesDir) + app.Router.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(staticFilesDir)))) + + return app +} + +func (app *FrontendApp) ListenAndServe() error { + return http.ListenAndServe(fmt.Sprintf(":%d", app.Port), app.Router) +} + +func (app *FrontendApp) setCookie(responseWriter http.ResponseWriter, name, value string, lifetimeSeconds int, sameSite http.SameSite) { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent + // The Domain attribute specifies which hosts are allowed to receive the cookie. + // If unspecified, it defaults to the same host that set the cookie, excluding subdomains. + // If Domain is specified, then subdomains are always included. + // Therefore, specifying Domain is less restrictive than omitting it. + // However, it can be helpful when subdomains need to share information about a user. + + toSet := &http.Cookie{ + Name: name, + HttpOnly: true, + Secure: true, + SameSite: sameSite, + Path: "/", + Value: value, + MaxAge: lifetimeSeconds, + } + + http.SetCookie(responseWriter, toSet) +} + +func (app *FrontendApp) deleteCookie(responseWriter http.ResponseWriter, name string) { + http.SetCookie(responseWriter, &http.Cookie{ + Name: name, + HttpOnly: true, + Secure: true, + SameSite: http.SameSiteLaxMode, + Path: "/", + Value: "", + MaxAge: -1, + }) +} + +func (app *FrontendApp) getSession(request *http.Request, domain string) (Session, error) { + toReturn := Session{ + Flash: &(map[string]string{}), + } + for _, cookie := range request.Cookies() { + if cookie.Name == "sessionId" && app.base58Regex.MatchString(cookie.Value) { + session, err := ReadJsonFile[Session](fmt.Sprintf("data/sessions/%s.json", cookie.Value)) + + if err == nil { + if session.ExpiresUnixMilli > time.Now().UnixMilli() { + toReturn.SessionId = cookie.Value + toReturn.UserID = session.UserID + } + } + //log.Printf("toReturn.SessionId %s\n", toReturn.SessionId) + } else if cookie.Name == "flash" && cookie.Value != "" { + bytes, err := base64.RawURLEncoding.DecodeString(cookie.Value) + if err != nil { + log.Printf("can't getSession because can't base64 decode flash cookie: %+v", err) + return toReturn, err + } + flash := map[string]string{} + err = json.Unmarshal(bytes, &flash) + if err != nil { + log.Printf("can't getSession because can't json parse the decoded flash cookie: %+v", err) + return toReturn, err + } + toReturn.Flash = &flash + } + } + return toReturn, nil +} + +func (app *FrontendApp) setSession(responseWriter http.ResponseWriter, session *Session) error { + sessionIdBuffer := make([]byte, 32) + rand.Read(sessionIdBuffer) + sessionId := base58.Encode(sessionIdBuffer, base58.BitcoinAlphabet) + + err := WriteJsonFile(fmt.Sprintf("data/sessions/%s.json", sessionId), *session) + if err != nil { + return err + } + + bytes, _ := json.MarshalIndent(session, "", " ") + log.Printf("setSession(): %s %s\n", sessionId, string(bytes)) + + exipreInSeconds := int(time.Until(time.UnixMilli(session.ExpiresUnixMilli)).Seconds()) + app.setCookie(responseWriter, "sessionId", sessionId, exipreInSeconds, http.SameSiteStrictMode) + + return nil +} + +func (app *FrontendApp) unhandledError(responseWriter http.ResponseWriter, request *http.Request, err error) { + log.Printf("500 internal server error: %+v\n", err) + + responseWriter.Header().Add("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusInternalServerError) + responseWriter.Write([]byte("500 internal server error")) +} + +func (app *FrontendApp) handleWithSession(path string, handler func(http.ResponseWriter, *http.Request, Session)) { + app.Router.HandleFunc(path, func(responseWriter http.ResponseWriter, request *http.Request) { + session, err := app.getSession(request, app.Domain) + + bytes, _ := json.MarshalIndent(session, "", " ") + log.Printf("handleWithSession(): %s\n", string(bytes)) + + if err != nil { + app.unhandledError(responseWriter, request, err) + } else { + handler(responseWriter, request, session) + } + }) +} + +func (app *FrontendApp) buildPage(responseWriter http.ResponseWriter, request *http.Request, session Session, highlight, page template.HTML) { + var buffer bytes.Buffer + templateName := "page.html" + pageTemplate, hasPageTemplate := app.HTMLTemplates[templateName] + if !hasPageTemplate { + panic(fmt.Errorf("template '%s' not found!", templateName)) + } + err := pageTemplate.Execute( + &buffer, + struct { + Session Session + Highlight template.HTML + Page template.HTML + CSSHash string + }{session, highlight, page, app.cssHash}, + ) + app.deleteCookie(responseWriter, "flash") + + if err != nil { + app.unhandledError(responseWriter, request, err) + } else { + io.Copy(responseWriter, &buffer) + } +} + +func (app *FrontendApp) renderTemplateToHTML(templateName string, data interface{}) (template.HTML, error) { + var buffer bytes.Buffer + desiredTemplate, hasTemplate := app.HTMLTemplates[templateName] + if !hasTemplate { + return "", fmt.Errorf("template '%s' not found!", templateName) + } + err := desiredTemplate.Execute(&buffer, data) + if err != nil { + return "", err + } + return template.HTML(buffer.String()), nil +} + +func (app *FrontendApp) buildPageFromTemplate(responseWriter http.ResponseWriter, request *http.Request, session Session, templateName string, data interface{}) { + content, err := app.renderTemplateToHTML(templateName, data) + if err != nil { + app.unhandledError(responseWriter, request, err) + } else { + app.buildPage(responseWriter, request, session, template.HTML(""), content) + } +} + +func (app *FrontendApp) setFlash(responseWriter http.ResponseWriter, session Session, key, value string) { + (*session.Flash)[key] += value + bytes, err := json.Marshal((*session.Flash)) + if err != nil { + log.Printf("can't setFlash because can't json marshal the flash map: %+v", err) + return + } + + app.setCookie(responseWriter, "flash", base64.RawURLEncoding.EncodeToString(bytes), 60, http.SameSiteStrictMode) +} + +func (app *FrontendApp) reloadTemplates() { + + loadTemplate := func(filename string) *template.Template { + newTemplateString, err := os.ReadFile(filename) + if err != nil { + panic(err) + } + newTemplate, err := template.New(filename).Parse(string(newTemplateString)) + if err != nil { + panic(err) + } + return newTemplate + } + + frontendDirectory := "./frontend" + //frontendVersion = hashTemplateAndStaticFiles(frontendDirectory)[:6] + + fileInfos, err := os.ReadDir(frontendDirectory) + if err != nil { + panic(err) + } + for _, fileInfo := range fileInfos { + if !fileInfo.IsDir() && strings.Contains(fileInfo.Name(), ".gotemplate") { + app.HTMLTemplates[strings.Replace(fileInfo.Name(), ".gotemplate", "", 1)] = loadTemplate(filepath.Join(frontendDirectory, fileInfo.Name())) + } + } + +} diff --git a/frontend/login.gotemplate.html b/frontend/login.gotemplate.html new file mode 100644 index 0000000..18a1f76 --- /dev/null +++ b/frontend/login.gotemplate.html @@ -0,0 +1,11 @@ + + +
+
+

login

+ + + + +
+
diff --git a/frontend/page.gotemplate.html b/frontend/page.gotemplate.html new file mode 100644 index 0000000..e51a63f --- /dev/null +++ b/frontend/page.gotemplate.html @@ -0,0 +1,51 @@ + + + + + matrix-synapse diskspace janitor + + + + + + + + +
+
+ +

+ matrix-synapse diskspace janitor +

+
+ {{if .Session.UserID }} + {{ .Session.UserID }} | logout + {{end}} +
+
+ + +
+
+
+ {{if index .Session.Flash "error"}} +
{{index .Session.Flash "error"}}
+ {{end}} + {{if index .Session.Flash "info"}} +
{{index .Session.Flash "info"}}
+ {{end}} +
+
+ {{.Page}} +
+
+ + + \ No newline at end of file diff --git a/frontend/panel.gotemplate.html b/frontend/panel.gotemplate.html new file mode 100644 index 0000000..e69de29 diff --git a/frontend/static/app.css b/frontend/static/app.css new file mode 100644 index 0000000..4ee493a --- /dev/null +++ b/frontend/static/app.css @@ -0,0 +1,831 @@ +html { + min-height: 100%; +} + +body { + font-family: -apple-system,system-ui,BlinkMacSystemFont,"Ubuntu",Roboto,"Segoe UI",sans-serif; + margin: 0; + padding: 0; + + background-repeat: no-repeat; + background-size: 100%; + background-image: linear-gradient(to bottom, #0074ba 0%, #6c9ad1 100%); + min-height: 100%; +} +header { + background-image: linear-gradient(to right, #85bb2e 0%, #35bd1a88 100%); + padding: 0.15rem 0.45rem; + + box-shadow: 0 0 6rem 1rem #27026b8a; + border-bottom-left-radius: 0.8rem; + border-bottom-right-radius: 0.8rem; +} +header .header-container { + padding: 0.25rem 0; + border-bottom: 2px solid #c5d6d6aa; +} + +header h1 { + font-size: 1.3rem; + background: #5f7a7a; + border-radius: 4px; + color: white; + display: inline; + letter-spacing: 0.2rem; + padding: 0.1rem 0.5rem; + margin: 0; +} +header .shine-container { + position: relative; +} +header .shine { + opacity: 1; + background-image: url(images/shine.png); + background-size: cover; + width: 1.8rem; + height: 1.8rem; + position: absolute; + right:-1.1rem; + top:-0.7rem; +} + +h1, h2, h3, h4, .price { + font-family: 'Nunito',-apple-system,system-ui,BlinkMacSystemFont,"Ubuntu",Roboto,"Segoe UI",sans-serif; +} + +blockquote { + display: inline-block; + margin: 0; + padding: 1em; + padding-left: 2em; + background-color: white; + border-left: 5px solid #ccc; + color: #333; +} + +hr { + width: 100%; +} + +p, li { + line-height: 1.5em; +} +li { + margin-top: 0.5em; + margin-bottom: 1em; +} +li::marker { + font-weight: bold; +} +a { + font-weight: 600; + color: #66811a; +} +a:visited { + font-weight: 600; + color: #406e14; +} +img { + max-width: 100%; +} +video { + max-height: 90%; +} +footer { + height: 100px; + background: none; + display: flex; + align-items: center; + justify-content: center; + color: #005b9c; + font-weight: bold; + font-style: italic; +} + +footer .cloud { + padding: 10px 20px; + border-radius: 20px; + background: #ffffff40; +} + +footer a { + font-weight: normal; + color: #101e58; +} +footer a:visited { + color: #2f1058; +} + +header .session-status, +header .session-status a { + font-weight: 600; + color: white; +} +header .session-status a { + text-decoration-thickness: 2px; +} +header .session-status { + margin-top: 0.2em; + margin-right: 0.8em; +} + + +code { + padding: 2px 4px; + font-size: 90%; + color: #862323; + background-color: #965f5f1c; + border-radius: 4px; +} + +pre code, +pre tt { + padding: 0; + font-size: inherit; + color: #222; + background-color: transparent; + border: 0; +} + +/* red-purple inline code links */ +code a, +a code { + font-weight: 600; + color: #af5214; + text-emphasis-color: #af5214; + text-decoration-thickness: 2px; +} +code a:hover, +a:hover code, +code a:visited, +a:visited code { + font-weight: 600; + color: #973213; + text-emphasis-color: #973213; + text-decoration-thickness: 2px; +} + +main { + /* background: #0000007e; */ +} +.highlight { + color: #eee4dd; + padding: 2em 1em; +} +.highlight a { + color: #b0e226; +} + +pre.flash { + border: 2px solid gray; + font-weight: bold; + padding: 2em; + border-radius: 8px; + font-size: 0.8em; +} +pre.flash.error { + border-color: #5e2416; + background: #da9871; + color: #5e2416; +} +pre.flash.info { + border-color: #000000aa; + background: #ffffff60; + color: #12496e; +} + + + +.admonition { + border-left: 8px dashed gray; + padding: 5px 20px; + /* display: flex; + flex-direction: row; + align-items: flex-start; */ + min-height: 100px; +} + +.admonition.warning { + border-left-color: #ffbb00; + background: #e7d8ae; +} +.admonition.info { + border-left-color: #289eff; + background: #c7dffa; +} +.admonition.info code { + background-color: #753d3d08; +} +.admonition.mascot { + border-left-color: #00ffaa; + background: #e8ffc3; +} + +.emoji-icon { + margin-right: 20px; + margin-top: 15px ; + float: left; + font-family: sans-serif; + font-size: 60px; + text-shadow: 2px 5px 20px #25202e40, 2px 5px 10px #25202e40, 2px 5px 3px #25202e40; +} +.emoji-icon img { + width: 70px; +} + +.install-command { + background: #554; + border-radius: 10px; + box-shadow: inset 1px 1px 40px 0 #222; + font-weight: bold; + color:#e5f879; + padding: 1em; + white-space: pre-wrap; + display: inline-block; + margin: 10px 0; +} +@media screen and (max-width: 800px) { + .install-command { + font-size: 12px; + } +} +@media screen and (max-width: 640px) { + .install-command { + font-size: 10px; + } +} +.install-command.small { + font-size: 12px; +} +.white-pill { + border: 1px solid gray; + border-radius: 12px; + background: white; + box-shadow: 0.1rem 0.1rem 0.5rem 0 #00000060; + padding: 2px 8px; + color: #444; + font-size: 16px; + display: inline-flex; + flex-direction: row; + align-items: center; + margin: 0 5px; +} + +.white-pill img { + height: 20px; + margin: -2px 5px -4px 5px; + filter: brightness(1.4); +} + + +.page { + padding: 2vw 4vw; + border-top: 0.2rem dashed #438bc8; + border-bottom: 0.2rem dashed #6999d0; + /* border-bottom: 0.2rem dashed #8a7232; */ + background: #d9d9d9; + color: #432; + min-height: 20em; +} +.page.no-horizontal-margin { + padding: 2vw 0; +} +@media screen and (max-width: 1024px) { + .page { + padding: 2vw 2vw; + } +} +@media screen and (max-width: 640px) { + .page { + padding: 0; + } +} +.pagewidth { + max-width: 1100px; +} + +.float-right { + float: right; +} +.image-large { + margin: 1rem; + width: 22rem; +} +.image-medium { + margin: 1rem; + width: 8rem; +} +.image-small { + margin: 0.5rem; + width: 4rem; +} + +.horizontal { + width: 100%; + display: flex; + align-items: flex-start; +} +.margin-bottom { + margin-bottom: 0.4em; +} +.align-center { + align-items: center; +} +.vertical { + display: flex; + flex-direction: column; + align-items: flex-start; +} + +.money-unit { + margin-right: 0.3em; +} + +input,button,textarea, +input:hover,button:hover,textarea:hover, +input:focus,button:focus,textarea:focus { + outline: 0; +} + +input, +.js-form-submit-button { + box-sizing: content-box; + border-radius: 4px; + border: 1px solid #ccc; + padding: 0.5rem; + color: #333; + font-size: 1em; +} +input:focus { + border: 2px solid #029dfd; + padding: calc(0.5rem - 1px); +} +.text-align-right { + text-align: right; +} +input.short { + width: 11em; +} +input.subdomain { + padding-right: 0; +} + +input::-webkit-outer-spin-button, +input::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; +} +/* Firefox */ +input[type=number] { + -moz-appearance: textfield; +} + +label { + margin-right: 1em; +} + +section.comments, +.homepage-markdown pre, +pre.telemetry { + background-color: white; + border-radius: 20px; + box-shadow: 5px 5px 20px 0 #00000020; +} + +.homepage-markdown pre, +pre.telemetry { + padding: 20px; + white-space: pre-wrap; +} + +pre.telemetry { + font-size: 10px; +} + +.homepage-mascot { + width: 25vw; + max-width: 180px; +} +@media screen and (max-width: 640px) { + .homepage-mascot { + display: none; + } +} + + +section.comments h1, +section.comments h2, +section.comments h3 { + padding-top: 20px; + padding-left: 20px; +} + +section.comments a { + font-weight: 600; + color: #7333e0; +} +section.comments a:visited { + font-weight: 600; + color: #7e1cac; +} + +#sqr-comment-container { + padding-left: 20px; + padding-right: 20px; + padding-bottom: 20px; +} + +#sqr-comment-container .sqr-comment-bottom-row { + padding-bottom: 0.4em; +} + +form.vertical input, +form.horizontal.wrap input, +form.vertical .js-form-submit-button { + margin-top: 0.4rem; + margin-bottom: 0.4rem; +} +.js-form-submit-button { + padding: 0.25em 1em; + cursor: pointer; + display: none; +} +.form-footer { + font-size: 0.85em; + padding-top: 1rem; +} +.fine-print { + font-size: 0.7em; +} + +.profile-form { + width: 100%; + background-color: #dfd5ca; + border-radius: 1em; + padding: 0.7em; +} + +.api-token, +.external-domain { + + border-bottom: 2px solid #44332255; + margin: 0.5em; + padding: 0.15em; +} + +.new-api-token { + background-color: #619d00; + color: white; + border-radius: 0.5em; + padding: 0.5em; + border-bottom: 0; + line-height: 2em; + font-weight: bold; +} + +.download-image { + height: 32px; + margin-right: 0.5em; +} +.os-image { + height: 1.5em; + margin-right: 0.5em; +} + + + +.invalid { + color: red; +} + + + + +@media screen and (max-width: 900px) { + .horizontal.flip { + flex-direction: column; + align-items: center; + } +} + + +.justify-center { + justify-content: center; +} + +.space-around { + justify-content: space-around; +} + +.space-between { + justify-content: space-around; +} + +.justify-left { + justify-content: flex-start; +} + +.justify-right { + justify-content: flex-end; +} + +.flex-grow-2 { + flex-grow: 2; +} + +.wrap { + flex-wrap: wrap; +} + +.box { + display: flex; + flex-direction: column; + align-items: center; + background: white; + margin: 1rem; + padding: 1rem 2rem; + border-radius: 1rem; + /* border: 0.2rem dashed #cba; */ + z-index: 1; +} +@media screen and (max-width: 640px) { + .box { + border-radius: 0; + border-left: 0; + border-right: 0; + margin-left: 0; + margin-right: 0; + padding-left: 0; + padding-right: 0; + } +} + +.box img.expand { + margin: 0 -6rem; + position: relative; + z-index: 2; +} + +.tab-container { + margin: 1rem 0.5rem; + display: grid; +} +.tab-container input[type="radio"] { + display: none; +} + +.tab-container > label:nth-of-type(1) { + grid-column: 1 / 2; +} +.tab-container > label:nth-of-type(2) { + grid-column: 2 / 4; +} +.tab-container > label:nth-of-type(3) { + grid-column: 4 / 5; +} + +.tab-container > label { + + margin: 0; + grid-row: 1 / 2; + border-top: 2px solid #00000000; + padding: 1rem; + border-bottom: 2px solid #c4c4c4; + cursor: pointer; + text-align: center; + white-space: nowrap; + + color: #432; + text-decoration: none; + + font-family: 'Nunito',-apple-system,system-ui,BlinkMacSystemFont,"Ubuntu",Roboto,"Segoe UI",sans-serif; + font-weight: bold; + font-size: 1.1em; +} +.tab-container input[type="radio"]:checked + label { + border-top: 2px solid #8a7232; + background: white; + border-bottom: 2px solid #00000000; + box-shadow: 0.1rem 0.5rem 1rem 0 #00000020; +} +.tab-container input[type="radio"]:checked + label + .tab-content { + display: block; +} + +.tab-container.two-tabs .tab-content { + grid-column: 1 / 4; +} +.tab-container.three-tabs .tab-content { + grid-column: 1 / 5; +} + +.tab-content { + grid-row: 2 / 3; + + display: none; + position: relative; + + padding: 1rem; + z-index: 1; + background: white; + box-shadow: 0.1rem 0.5rem 1rem 0 #00000020; +} + + + +/* this makes the tab-content for the selected tab visible */ +input[type="radio"].tab:checked+label { + display: block; +} + +@media screen and (max-width: 1150px) and (min-width: 901px) { + .service-pricing .box { + border-radius: 0; + margin-left: -0.1rem; + margin-right: -0.1rem; + } + .service-pricing .box:first-child { + border-top-left-radius: 1rem; + border-bottom-left-radius: 1rem; + } + .service-pricing .box:nth-child(2) { + border-bottom-right-radius: 1rem; + border-bottom-left-radius: 1rem; + } + .service-pricing .box:last-child { + border-top-right-radius: 1rem; + border-bottom-right-radius: 1rem; + } + .service-pricing .box.price-box { + border-radius: 0.5rem; + } +} + + +.box.price-box { + border-radius: 0.5rem; + min-width: 11rem; + padding: 1rem; + padding-bottom: 0rem; + margin-left: -1rem; + margin-right: -1rem; + box-shadow: 0 0.5rem 1rem 0 #00000070; + border: 2px solid #ccc; + width: 100%; +} + +.price { + display: flex; + align-items: center; + justify-content: center; + font-weight: bold; + font-size: 1.6rem; +} + +.big-price { + font-size: 3.6rem; +} + +.med-price { + font-size: 2.6rem; +} + +.price-box ul { + margin-left: -1rem; + align-self: baseline; +} + + + + + + + + + + + + + + + + + + +.admin-box { + border-radius: 0.5rem; + min-width: 11rem; + padding: 1rem; + padding-bottom: 0rem; + box-shadow: 0 0.5rem 1rem 0 #00000070; + border: 2px solid #ccc; +} + +.admin-box form { + display: inline; +} + +.instance-group { + min-height: 100px; + width: 100%; + border: 3px dashed rgb(173, 230, 109); + display: flex; + flex-wrap: wrap; + flex-direction: row; + justify-content: flex-start; + align-content: flex-start; +} + +.instance-group>h4, +.instance-group>h3, +.instance-group>h5 { + width: 100%; + text-align: center; +} + +.instance { + width: 100px; + height: 120px; + margin: 10px; + padding: 10px; + border-radius: 0.3em; + display: flex; + flex-direction: column; + + box-sizing: content-box; +} + +.instance.healthy { + background-color: rgb(102, 241, 176); + border: 1px solid rgb(56, 161, 165); +} +.instance.unhealthy { + background-color: rgb(255, 116, 92); + border: 1px solid rgb(165, 56, 74); +} + +.instance-name, +.instance-ip { + background-color: white; + color: #333; + margin-bottom: 5px; + font-size: 11px; + white-space: nowrap; + overflow: hidden; +} +.instance-content { + display: flex; + flex-direction: row; + justify-content: flex-end; + flex-grow: 1; + box-sizing: content-box; + max-height: calc(100% - 40px); +} +.current-tenants, .pinned-tenants { + display: inline-flex; + writing-mode: vertical-lr; + flex-wrap: wrap; + align-content: flex-start; + margin-right: 5px; + padding: 2px; + min-width: 10px; + min-height: 100%; +} +.current-tenants { + border: 1px solid rgb(51, 57, 116); +} +.pinned-tenants { + border: 1px dashed gray; +} +.tenant { + writing-mode: horizontal-tb; + color: white; + border-radius: 5px; + padding: 3px; + font-size: 10px; +} +.thermometer { + background-color: gray; + min-height: 100%; + width: 6px; + padding: 2px; + margin-right: 5px; + display: flex; + flex-direction: column-reverse; +} + +.thermometer.projected { + margin-right: 0; +} +.marker { + display: block; +} + +.delete-button { + border: 2px solid red; + color: #400; + background-color: #fcc; + border-radius: 4px; + outline: 0; + padding: 0.2em; +} \ No newline at end of file diff --git a/frontend/static/favicon.png b/frontend/static/favicon.png new file mode 100644 index 0000000..30c3e90 Binary files /dev/null and b/frontend/static/favicon.png differ diff --git a/frontend/static/images/scruffy.png b/frontend/static/images/scruffy.png new file mode 100644 index 0000000..f8b9d44 Binary files /dev/null and b/frontend/static/images/scruffy.png differ diff --git a/go.mod b/go.mod index 791c9a9..a89c531 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module git.cyberia.club/cyberia/matrix-synapse-state-groups-state-janitor +module git.cyberia.club/cyberia/matrix-synapse-diskspace-janitor go 1.19 @@ -6,6 +6,7 @@ require ( git.sequentialread.com/forest/config-lite v0.0.0-20220225195944-164dc71bce04 // indirect git.sequentialread.com/forest/pkg-errors v0.9.2 // indirect github.com/lib/pq v1.10.7 // indirect + github.com/shengdoushi/base58 v1.0.0 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect golang.org/x/sys v0.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5ef6c8e..f061eeb 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.sequentialread.com/forest/pkg-errors v0.9.2 h1:j6pwbL6E+TmE7TD0tqRtGwuoCbCfO git.sequentialread.com/forest/pkg-errors v0.9.2/go.mod h1:8TkJ/f8xLWFIAid20aoqgDZcCj9QQt+FU+rk415XO1w= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/shengdoushi/base58 v1.0.0 h1:tGe4o6TmdXFJWoI31VoSWvuaKxf0Px3gqa3sUWhAxBs= +github.com/shengdoushi/base58 v1.0.0/go.mod h1:m5uIILfzcKMw6238iWAhP4l3s5+uXyF3+bJKUNhAL9I= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= diff --git a/main.go b/main.go index cb131f4..45c488b 100644 --- a/main.go +++ b/main.go @@ -1,23 +1,46 @@ package main import ( - "encoding/json" "log" - "os" "reflect" + "strings" + "sync" "time" configlite "git.sequentialread.com/forest/config-lite" ) type Config struct { + FrontendPort int + FrontendDomain string MatrixURL string + MatrixServerPublicDomain string + AdminMatrixRoomId string MatrixAdminToken string DatabaseType string DatabaseConnectionString string + MediaFolder string + PostgresFolder string } +type JanitorState struct { + LastScheduledTaskRunUnixMilli int64 +} + +type DiskUsage struct { + DiskSizeBytes int64 + OtherBytes int64 + MediaBytes int64 + PostgresBytes int64 +} + +var isRunningScheduledTask bool +var mutex sync.Mutex +var matrixAdmin *MatrixAdmin + func main() { + mutex = sync.Mutex{} + config := Config{} ignoreCommandlineFlags := []string{} err := configlite.ReadConfiguration("config.json", "JANITOR", ignoreCommandlineFlags, reflect.ValueOf(&config)) @@ -25,7 +48,78 @@ func main() { panic(err) } - db := initDatabase(&config) + validateConfig(&config) + + //db := initDatabase(&config) + matrixAdmin = initMatrixAdmin(&config) + frontend := initFrontend(&config) + + go frontend.ListenAndServe() + + for { + janitorState, err := ReadJsonFile[JanitorState]("data/janitorState.json") + if err != nil { + log.Printf("ERROR!: can't read data/janitorState.json: %+v\n", err) + } else { + sinceLastScheduledTaskDuration := time.Since(time.UnixMilli(janitorState.LastScheduledTaskRunUnixMilli)) + if !isRunningScheduledTask && sinceLastScheduledTaskDuration > time.Hour*24 { + //go runScheduledTask(db, &config) + + } + } + + time.Sleep(time.Second * 10) + } +} + +func runScheduledTask(db *DBModel, config *Config) { + + isRunningScheduledTask = true + log.Println("starting runScheduledTask...") + + log.Println("GetDBTableSizes...") + tables, err := db.GetDBTableSizes() + if err != nil { + log.Printf("ERROR!: runScheduledTask can't GetDBTableSizes: %s\n", err) + } + log.Println("Saving data/dbTableSizes.json...") + err = WriteJsonFile[[]DBTableSize]("data/dbTableSizes.json", tables) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't write data/dbTableSizes.json: %s\n", err) + } + + log.Println("GetAvaliableDiskSpace...") + availableBytes, totalBytes, err := GetAvaliableDiskSpace(config.MediaFolder) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't GetAvaliableDiskSpace: %s\n", err) + } + + log.Printf("GetTotalFilesizeWithinFolder(\"%s\")...\n", config.MediaFolder) + mediaBytes, err := GetTotalFilesizeWithinFolder(config.MediaFolder) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't GetTotalFilesizeWithinFolder(\"%s\"): %s\n", config.MediaFolder, err) + } + + log.Printf("GetTotalFilesizeWithinFolder(\"%s\")...\n", config.PostgresFolder) + postgresBytes, err := GetTotalFilesizeWithinFolder(config.PostgresFolder) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't GetTotalFilesizeWithinFolder(\"%s\"): %s\n", config.PostgresFolder, err) + } + + diskUsage := DiskUsage{ + DiskSizeBytes: totalBytes, + OtherBytes: (totalBytes - availableBytes) - (mediaBytes + postgresBytes), + MediaBytes: mediaBytes, + PostgresBytes: postgresBytes, + } + + log.Println("Saving data/diskUsage.json...") + err = WriteJsonFile[DiskUsage]("data/diskUsage.json", diskUsage) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't write data/diskUsage.json: %s\n", err) + } + + log.Println("starting db.StateGroupsStateStream()...") stream, err := db.StateGroupsStateStream() if err != nil { log.Fatalf("Can't start because %+v\n", err) @@ -41,25 +135,74 @@ func main() { updateCounter += 1 rowCounter += 1 if updateCounter > 10000 { - if time.Now().After(lastUpdateTime.Add(time.Second)) { + if time.Now().After(lastUpdateTime.Add(time.Second * 60)) { lastUpdateTime = time.Now() percent := int((float64(rowCounter) / float64(stream.EstimatedCount)) * float64(100)) - log.Printf("%d/%d (%d%s) ... \n", rowCounter, stream.EstimatedCount, percent, "%") + log.Printf("state_groups_state table scan %d/%d (%d%s) ... \n", rowCounter, stream.EstimatedCount, percent, "%") } updateCounter = 0 } } - output, err := json.MarshalIndent(rowCountByRoom, "", " ") - + err = WriteJsonFile[map[string]int]("data/stateGroupsStateRowCountByRoom.json", rowCountByRoom) if err != nil { - log.Printf("Can't save rooms.json because json.MarshalIndent returned %+v\n", err) + log.Printf("ERROR!: runScheduledTask can't write data/stateGroupsStateRowCountByRoom.json: %s\n", err) } - err = os.WriteFile("./rooms.json", output, 0755) + log.Println("updating data/janitorState.json...") + janitorState, err := ReadJsonFile[JanitorState]("data/janitorState.json") if err != nil { - log.Printf("Can't save rooms.json because os.WriteFile returned %+v\n", err) + log.Printf("ERROR!: runScheduledTask can't read data/janitorState.json: %+v\n", err) } + janitorState.LastScheduledTaskRunUnixMilli = time.Now().UnixMilli() + + err = WriteJsonFile[JanitorState]("data/janitorState.json", janitorState) + if err != nil { + log.Printf("ERROR!: runScheduledTask can't write data/janitorState.json: %s\n", err) + } + + log.Println("runScheduledTask completed!") + isRunningScheduledTask = false +} + +func validateConfig(config *Config) { + + errors := []string{} + + if config.FrontendPort == 0 { + errors = append(errors, "Can't start because FrontendPort is required") + } + if config.FrontendDomain == "" { + errors = append(errors, "Can't start because FrontendDomain is required") + } + if config.MatrixURL == "" { + errors = append(errors, "Can't start because MatrixURL is required") + } + if config.MatrixAdminToken == "" || config.MatrixAdminToken == "changeme" { + errors = append(errors, "Can't start because MatrixAdminToken is required") + } + if config.MatrixServerPublicDomain == "" { + errors = append(errors, "Can't start because MatrixServerPublicDomain is required") + } + if config.AdminMatrixRoomId == "" { + errors = append(errors, "Can't start because AdminMatrixRoomId is required") + } + if config.DatabaseType == "" { + errors = append(errors, "Can't start because DatabaseType is required") + } + if config.DatabaseConnectionString == "" { + errors = append(errors, "Can't start because DatabaseConnectionString is required") + } + if config.MediaFolder == "" { + errors = append(errors, "Can't start because MediaFolder is required") + } + if config.PostgresFolder == "" { + errors = append(errors, "Can't start because PostgresFolder is required") + } + + if len(errors) > 0 { + log.Fatalln(strings.Join(errors, "\n")) + } } diff --git a/matrix_admin_service.go b/matrix_admin_service.go index 0076d2d..ab8642e 100644 --- a/matrix_admin_service.go +++ b/matrix_admin_service.go @@ -13,9 +13,11 @@ import ( ) type MatrixAdmin struct { - Client http.Client - URL string - Token string + Client http.Client + AdminMatrixRoomId string + MatrixServerPublicDomain string + URL string + Token string } type DeleteRoomRequest struct { @@ -45,14 +47,36 @@ type ShutdownRoom struct { NewRoomId string `json:"new_room_id"` } +type LoginRequestBody struct { + Identifier LoginIdentifier `json:"identifier"` + DeviceDisplayName string `json:"initial_device_display_name"` + Password string `json:"password"` + Type string `json:"type"` +} + +type LoginIdentifier struct { + Type string `json:"type"` + User string `json:"user"` +} + +type LoginResponseBody struct { + AccessToken string `json:"access_token"` +} + +type RoomMembersResponseBody struct { + Members []string `json:"members"` +} + func initMatrixAdmin(config *Config) *MatrixAdmin { return &MatrixAdmin{ Client: http.Client{ Timeout: 10 * time.Second, }, - URL: config.MatrixURL, - Token: config.MatrixAdminToken, + AdminMatrixRoomId: config.AdminMatrixRoomId, + MatrixServerPublicDomain: config.MatrixServerPublicDomain, + URL: config.MatrixURL, + Token: config.MatrixAdminToken, } } @@ -187,3 +211,77 @@ func (admin *MatrixAdmin) GetDeleteRoomStatus(roomId string) (string, []string, return mostCompleteStatus, usersSlice, nil } + +// curl 'https://matrix.cyberia.club/_matrix/client/r0/login' -X POST -H 'Accept: application/json' -H 'content-type: application/json' +// --data-raw '{"type":"m.login.password","password":"xxxxxxxxx","identifier":{"type":"m.id.user","user":"forestjohnson"},"initial_device_display_name":"chat.cyberia.club (Firefox, Ubuntu)"}' + +func (admin *MatrixAdmin) Login(username, password string) (bool, error) { + + loginURL := fmt.Sprintf("%s/_matrix/client/v3/login", admin.URL) + + loginRequestBodyObject := LoginRequestBody{ + Identifier: LoginIdentifier{ + Type: "m.id.user", + User: username, + }, + DeviceDisplayName: "matrix-synapse-diskspace-janitor", + Password: password, + Type: "m.login.password", + } + loginRequestBody, err := json.Marshal(loginRequestBodyObject) + if err != nil { + return false, errors.Wrap(err, "can't serialize LoginRequestBody to json") + } + loginResponse, err := admin.Client.Post(loginURL, "application/json", bytes.NewBuffer(loginRequestBody)) + if err != nil { + return false, err + } + + if loginResponse.StatusCode > 200 { + return false, nil + } + + responseBody, err := ioutil.ReadAll(loginResponse.Body) + if err != nil { + return false, errors.Wrapf(err, "HTTP POST %s read error", loginURL) + } + var responseObject LoginResponseBody + err = json.Unmarshal(responseBody, &responseObject) + if err != nil { + return false, errors.Wrapf(err, "HTTP POST %s response json parse error", loginURL) + } + + logoutURL := fmt.Sprintf("%s/_matrix/client/v3/logout", admin.URL) + logoutRequest, err := http.NewRequest("POST", logoutURL, nil) + if err != nil { + return false, errors.Wrap(err, "matrixAdmin.Login(...) cannot create logoutRequest") + } + + logoutRequest.Header.Add("Authorization", fmt.Sprintf("Bearer %s", responseObject.AccessToken)) + + _, err = admin.Client.Do(logoutRequest) + if err != nil { + return false, err + } + + roomMembersURLWithoutToken := fmt.Sprintf("%s/_synapse/admin/v1/rooms/%s/members", admin.URL, admin.AdminMatrixRoomId) + roomMembersURL := fmt.Sprintf("%s%s", roomMembersURLWithoutToken, admin.Token) + roomMembersResponse, err := admin.Client.Get(roomMembersURL) + + roomMembersResponseBody, err := ioutil.ReadAll(roomMembersResponse.Body) + if err != nil { + return false, errors.Wrapf(err, "HTTP POST %sxxxxxxx read error", roomMembersURLWithoutToken) + } + var roomMembersResponseObject RoomMembersResponseBody + err = json.Unmarshal(roomMembersResponseBody, &roomMembersResponseObject) + if err != nil { + return false, errors.Wrapf(err, "HTTP POST %sxxxxxxxxx response json parse error", roomMembersURLWithoutToken) + } + for _, member := range roomMembersResponseObject.Members { + if member == fmt.Sprintf("@%s:%s", username, admin.MatrixServerPublicDomain) { + return true, nil + } + } + + return false, nil +} diff --git a/storage_service.go b/storage_service.go new file mode 100644 index 0000000..13e0e38 --- /dev/null +++ b/storage_service.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "os" + + errors "git.sequentialread.com/forest/pkg-errors" +) + +func WriteJsonFile[T any](path string, object T) error { + mutex.Lock() + defer mutex.Unlock() + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + return json.NewEncoder(file).Encode(object) +} + +func ReadJsonFile[T any](path string) (T, error) { + mutex.Lock() + defer mutex.Unlock() + var object T + file, err := os.OpenFile(path, os.O_RDONLY, 0644) + if err != nil && os.IsNotExist(err) { + return object, nil + } + if err != nil { + return object, err + } + defer file.Close() + + err = json.NewDecoder(file).Decode(&object) + if err != nil { + return object, errors.Wrapf(err, "json parse error on %s", path) + } + return object, nil +}