Save remote profiles avatars & banners locally

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2020-12-15 17:17:42 +01:00
parent ae03f84950
commit 9b27e70eb0
No known key found for this signature in database
GPG key ID: A061B9DDE0CA0773
41 changed files with 1424 additions and 716 deletions

View file

@ -20,6 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Docker * Docker
`docker-compose exec mobilizon mobilizon_ctl maintenance.fix_unattached_media_in_body` `docker-compose exec mobilizon mobilizon_ctl maintenance.fix_unattached_media_in_body`
* **Refresh remote profiles to save avatars locally**
Profile avatars and banners were previously only proxified and cached. Now we save them locally. Refreshing all remote actors will save profile media locally instead.
* Source install
`MIX_ENV=prod mix mobilizon.actors.refresh --all`
* Docker
`docker-compose exec mobilizon mobilizon_ctl actors.refresh --all`
### Added ### Added
- **Add a command to clean orphan media files**. There's a `--dry-run` option to see what files would have been deleted. - **Add a command to clean orphan media files**. There's a `--dry-run` option to see what files would have been deleted.

View file

@ -81,17 +81,6 @@ config :mobilizon, Mobilizon.Web.Upload,
config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads" config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads"
config :mobilizon, :media_proxy,
enabled: true,
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
http: [
follow_redirect: true,
pool: :media
]
]
config :mobilizon, Mobilizon.Web.Email.Mailer, config :mobilizon, Mobilizon.Web.Email.Mailer,
adapter: Bamboo.SMTPAdapter, adapter: Bamboo.SMTPAdapter,
server: "localhost", server: "localhost",

View file

@ -766,7 +766,10 @@ defmodule Mobilizon.Federation.ActivityPub do
res = res =
with {:ok, %{status: 200, body: body}} <- with {:ok, %{status: 200, body: body}} <-
Tesla.get(url, headers: [{"Accept", "application/activity+json"}]), Tesla.get(url,
headers: [{"Accept", "application/activity+json"}],
follow_redirect: true
),
:ok <- Logger.debug("response okay, now decoding json"), :ok <- Logger.debug("response okay, now decoding json"),
{:ok, data} <- Jason.decode(body) do {:ok, data} <- Jason.decode(body) do
Logger.debug("Got activity+json response at actor's endpoint, now converting data") Logger.debug("Got activity+json response at actor's endpoint, now converting data")

View file

@ -382,7 +382,7 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier do
{:ok, activity, new_actor} {:ok, activity, new_actor}
else else
e -> e ->
Logger.debug(inspect(e)) Logger.error(inspect(e))
:error :error
end end
end end

View file

@ -11,7 +11,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
alias Mobilizon.Federation.ActivityPub.Utils alias Mobilizon.Federation.ActivityPub.Utils
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible} alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Web.MediaProxy alias Mobilizon.Service.HTTP.RemoteMediaDownloaderClient
alias Mobilizon.Service.RichMedia.Parser
alias Mobilizon.Web.Upload
@behaviour Converter @behaviour Converter
@ -30,18 +32,10 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
@spec as_to_model_data(map()) :: {:ok, map()} @spec as_to_model_data(map()) :: {:ok, map()}
def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do
avatar = avatar =
data["icon"]["url"] && download_picture(get_in(data, ["icon", "url"]), get_in(data, ["icon", "name"]), "avatar")
%{
"name" => data["icon"]["name"] || "avatar",
"url" => MediaProxy.url(data["icon"]["url"])
}
banner = banner =
data["image"]["url"] && download_picture(get_in(data, ["image", "url"]), get_in(data, ["image", "name"]), "banner")
%{
"name" => data["image"]["name"] || "banner",
"url" => MediaProxy.url(data["image"]["url"])
}
%{ %{
url: data["id"], url: data["id"],
@ -140,4 +134,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
}) })
end end
end end
@spec download_picture(String.t() | nil, String.t(), String.t()) :: map()
defp download_picture(nil, _name, _default_name), do: nil
defp download_picture(url, name, default_name) do
with {:ok, %{body: body, status: code, headers: response_headers}}
when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
{:ok, file} <- Upload.store(%{body: body, name: name}) do
file
end
end
end end

View file

@ -10,7 +10,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
alias Mobilizon.Users.User alias Mobilizon.Users.User
alias Mobilizon.GraphQL.API alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Federation.ActivityPub.Activity alias Mobilizon.Federation.ActivityPub.Activity
import Mobilizon.Web.Gettext import Mobilizon.Web.Gettext
@ -35,7 +34,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
) do ) do
case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do
{:has_event, %Event{} = event} -> {:has_event, %Event{} = event} ->
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))} {:ok, event}
{:has_event, _} -> {:has_event, _} ->
{:error, :event_not_found} {:error, :event_not_found}
@ -51,7 +50,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
{:has_event, Events.get_public_event_by_uuid_with_preload(uuid)}, {:has_event, Events.get_public_event_by_uuid_with_preload(uuid)},
{:access_valid, true} <- {:access_valid, true} <-
{:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do {:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))} {:ok, event}
else else
{:has_event, _} -> {:has_event, _} ->
find_private_event(parent, args, resolution) find_private_event(parent, args, resolution)

View file

@ -8,7 +8,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
alias Mobilizon.Actors.{Actor, Member} alias Mobilizon.Actors.{Actor, Member}
alias Mobilizon.Federation.ActivityPub alias Mobilizon.Federation.ActivityPub
alias Mobilizon.GraphQL.API alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User alias Mobilizon.Users.User
alias Mobilizon.Web.Upload alias Mobilizon.Web.Upload
import Mobilizon.Web.Gettext import Mobilizon.Web.Gettext
@ -30,8 +29,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
with {:ok, %Actor{id: group_id} = group} <- with {:ok, %Actor{id: group_id} = group} <-
ActivityPub.find_or_make_group_from_nickname(name), ActivityPub.find_or_make_group_from_nickname(name),
{:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)}, {:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)}, {:member, true} <- {:member, Actors.is_member?(actor_id, group_id)} do
group <- Person.proxify_pictures(group) do
{:ok, group} {:ok, group}
else else
{:member, false} -> {:member, false} ->
@ -44,7 +42,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
def find_group(_parent, %{preferred_username: name}, _resolution) do def find_group(_parent, %{preferred_username: name}, _resolution) do
with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name), with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name),
%Actor{} = actor <- Person.proxify_pictures(actor),
%Actor{} = actor <- restrict_fields_for_non_member_request(actor) do %Actor{} = actor <- restrict_fields_for_non_member_request(actor) do
{:ok, actor} {:ok, actor}
else else

View file

@ -6,7 +6,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
alias Mobilizon.Actors.Actor alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant} alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.GraphQL.API.Participations alias Mobilizon.GraphQL.API.Participations
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User alias Mobilizon.Users.User
alias Mobilizon.Web.Email alias Mobilizon.Web.Email
alias Mobilizon.Web.Email.Checker alias Mobilizon.Web.Email.Checker
@ -114,7 +113,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
%Participant{} = participant <- %Participant{} = participant <-
participant participant
|> Map.put(:event, event) |> Map.put(:event, event)
|> Map.put(:actor, Person.proxify_pictures(actor)) do |> Map.put(:actor, actor) do
{:ok, participant} {:ok, participant}
else else
{:maximum_attendee_capacity, _} -> {:maximum_attendee_capacity, _} ->

View file

@ -15,15 +15,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
alias Mobilizon.Federation.ActivityPub alias Mobilizon.Federation.ActivityPub
require Logger require Logger
alias Mobilizon.Web.{MediaProxy, Upload} alias Mobilizon.Web.Upload
@doc """ @doc """
Get a person Get a person
""" """
def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do
with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true), with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true),
true <- suspended == false or is_moderator(role), true <- suspended == false or is_moderator(role) do
actor <- proxify_pictures(actor) do
{:ok, actor} {:ok, actor}
else else
_ -> _ ->
@ -31,6 +30,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end end
end end
def get_person(_parent, _args, _resolution), do: {:error, :unauthorized}
@doc """ @doc """
Find a person Find a person
""" """
@ -39,8 +40,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
}) do }) do
with {:ok, %Actor{id: actor_id} = actor} <- with {:ok, %Actor{id: actor_id} = actor} <-
ActivityPub.find_or_make_actor_from_nickname(preferred_username), ActivityPub.find_or_make_actor_from_nickname(preferred_username),
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)}, {:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)} do
actor <- proxify_pictures(actor) do
{:ok, actor} {:ok, actor}
else else
{:own, nil} -> {:own, nil} ->
@ -120,9 +120,12 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
args = Map.put(args, :user_id, user.id) args = Map.put(args, :user_id, user.id)
with args <- Map.update(args, :preferred_username, "", &String.downcase/1), with args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- save_attached_pictures(args), {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args) do {:ok, %Actor{} = new_person} <- Actors.new_person(args) do
{:ok, new_person} {:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
end end
end end
@ -144,10 +147,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
with {:find_actor, %Actor{} = actor} <- with {:find_actor, %Actor{} = actor} <-
{:find_actor, Actors.get_actor(id)}, {:find_actor, Actors.get_actor(id)},
{:is_owned, %Actor{}} <- User.owns_actor(user, actor.id), {:is_owned, %Actor{}} <- User.owns_actor(user, actor.id),
args <- save_attached_pictures(args), {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do {:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do
{:ok, actor} {:ok, actor}
else else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:find_actor, nil} -> {:find_actor, nil} ->
{:error, dgettext("errors", "Profile not found")} {:error, dgettext("errors", "Profile not found")}
@ -199,18 +205,27 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end end
defp save_attached_pictures(args) do defp save_attached_pictures(args) do
Enum.reduce([:avatar, :banner], args, fn key, args -> with args when is_map(args) <- save_attached_picture(args, :avatar),
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do args when is_map(args) <- save_attached_picture(args, :banner) do
media = args[key][:media] args
end
end
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <- defp save_attached_picture(args, key) do
Upload.store(media.file, type: key, description: media.alt) do if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
Map.put(args, key, %{"name" => name, "url" => url, "mediaType" => content_type}) with media when is_map(media) <- save_picture(args[key][:media], key) do
Map.put(args, key, media)
end end
else else
args args
end end
end) end
defp save_picture(media, key) do
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
%{"name" => name, "url" => url, "mediaType" => content_type}
end
end end
@doc """ @doc """
@ -223,10 +238,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
{:no_actor, true} <- {:no_actor, no_actor}, {:no_actor, true} <- {:no_actor, no_actor},
args <- Map.update(args, :preferred_username, "", &String.downcase/1), args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- Map.put(args, :user_id, user.id), args <- Map.put(args, :user_id, user.id),
args <- save_attached_pictures(args), {:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do {:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do
{:ok, new_person} {:ok, new_person}
else else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:error, :user_not_found} -> {:error, :user_not_found} ->
{:error, dgettext("errors", "No user with this email was found")} {:error, dgettext("errors", "No user with this email was found")}
@ -298,12 +316,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end end
end end
def proxify_pictures(%Actor{} = actor) do
actor
|> proxify_avatar
|> proxify_banner
end
def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{ def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{
context: %{current_user: %User{role: role}} context: %{current_user: %User{role: role}}
}) })
@ -343,20 +355,4 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
defp last_admin_of_a_group?(actor_id) do defp last_admin_of_a_group?(actor_id) do
length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0 length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0
end end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{avatar: %{url: avatar_url} = avatar} = actor) do
actor |> Map.put(:avatar, avatar |> Map.put(:url, MediaProxy.url(avatar_url)))
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{} = actor), do: actor
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{banner: %{url: banner_url} = banner} = actor) do
actor |> Map.put(:banner, banner |> Map.put(:url, MediaProxy.url(banner_url)))
end
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{} = actor), do: actor
end end

View file

@ -73,6 +73,12 @@ defmodule Mix.Tasks.Mobilizon.Actors.Refresh do
{:actor, nil} -> {:actor, nil} ->
shell_error("Error: No such actor") shell_error("Error: No such actor")
{:error, err} when is_binary(err) ->
shell_error(err)
_err ->
shell_error("Error while refreshing actor #{preferred_username}")
end end
end end

View file

@ -13,11 +13,13 @@ defmodule Mobilizon.Actors do
alias Mobilizon.Actors.{Actor, Bot, Follower, Member} alias Mobilizon.Actors.{Actor, Bot, Follower, Member}
alias Mobilizon.Addresses.Address alias Mobilizon.Addresses.Address
alias Mobilizon.{Crypto, Events} alias Mobilizon.{Crypto, Events}
alias Mobilizon.Events.FeedToken
alias Mobilizon.Federation.ActivityPub alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Medias.File alias Mobilizon.Medias.File
alias Mobilizon.Service.Workers alias Mobilizon.Service.Workers
alias Mobilizon.Storage.{Page, Repo} alias Mobilizon.Storage.{Page, Repo}
alias Mobilizon.Users alias Mobilizon.Users
alias Mobilizon.Users.User
alias Mobilizon.Web.Email.Group alias Mobilizon.Web.Email.Group
alias Mobilizon.Web.Upload alias Mobilizon.Web.Upload
@ -215,18 +217,35 @@ defmodule Mobilizon.Actors do
def new_person(args, default_actor \\ false) do def new_person(args, default_actor \\ false) do
args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key()) args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key())
with {:ok, %Actor{id: person_id} = person} <- multi =
%Actor{} Multi.new()
|> Actor.registration_changeset(args) |> Multi.insert(:person, Actor.registration_changeset(%Actor{}, args))
|> Repo.insert() do |> Multi.insert(:token, fn %{person: person} ->
Events.create_feed_token(%{user_id: args.user_id, actor_id: person.id}) FeedToken.changeset(%FeedToken{}, %{
user_id: args.user_id,
actor_id: person.id,
token: Ecto.UUID.generate()
})
end)
multi =
if default_actor do if default_actor do
user = Users.get_user!(args.user_id) user = Users.get_user!(args.user_id)
Users.update_user(user, %{default_actor_id: person_id})
Multi.update(multi, :user, fn %{person: person} ->
User.changeset(user, %{default_actor_id: person.id})
end)
else
multi
end end
case Repo.transaction(multi) do
{:ok, %{person: %Actor{} = person}} ->
{:ok, person} {:ok, person}
{:error, _step, err, _} ->
Logger.error("Error while creating a new person")
{:error, err}
end end
end end

View file

@ -13,7 +13,7 @@ defmodule Mobilizon.Service.Export.Feed do
alias Mobilizon.Storage.Page alias Mobilizon.Storage.Page
alias Mobilizon.Users.User alias Mobilizon.Users.User
alias Mobilizon.Web.{Endpoint, MediaProxy} alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.Router.Helpers, as: Routes alias Mobilizon.Web.Router.Helpers, as: Routes
require Logger require Logger
@ -85,14 +85,14 @@ defmodule Mobilizon.Service.Export.Feed do
feed = feed =
if actor.avatar do if actor.avatar do
feed |> Feed.icon(actor.avatar.url |> MediaProxy.url()) feed |> Feed.icon(actor.avatar.url)
else else
feed feed
end end
feed = feed =
if actor.banner do if actor.banner do
feed |> Feed.logo(actor.banner.url |> MediaProxy.url()) feed |> Feed.logo(actor.banner.url)
else else
feed feed
end end

View file

@ -0,0 +1,22 @@
defmodule Mobilizon.Service.HTTP.RemoteMediaDownloaderClient do
@moduledoc """
Tesla HTTP Basic Client that fetches HTML to extract metadata preview
"""
use Tesla
alias Mobilizon.Config
@default_opts [
recv_timeout: 20_000
]
adapter(Tesla.Adapter.Hackney, @default_opts)
@user_agent Config.instance_user_agent()
plug(Tesla.Middleware.FollowRedirects)
plug(Tesla.Middleware.Timeout, timeout: 10_000)
plug(Tesla.Middleware.Headers, [{"User-Agent", @user_agent}])
end

View file

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
alias Phoenix.HTML.Tag alias Phoenix.HTML.Tag
alias Mobilizon.Actors.Actor alias Mobilizon.Actors.Actor
alias Mobilizon.Web.JsonLD.ObjectView alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1] import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1]
def build_tags(_actor, _locale \\ "en") def build_tags(_actor, _locale \\ "en")
@ -36,7 +35,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
tags tags
else else
tags ++ tags ++
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url |> MediaProxy.url())] [Tag.tag(:meta, property: "og:image", content: actor.avatar.url)]
end end
end end

View file

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
alias Phoenix.HTML.Tag alias Phoenix.HTML.Tag
alias Mobilizon.Events.Event alias Mobilizon.Events.Event
alias Mobilizon.Web.JsonLD.ObjectView alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1] import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1]
def build_tags(%Event{} = event, locale \\ "en") do def build_tags(%Event{} = event, locale \\ "en") do
@ -28,7 +27,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
[ [
Tag.tag(:meta, Tag.tag(:meta,
property: "og:image", property: "og:image",
content: event.picture.file.url |> MediaProxy.url() content: event.picture.file.url
) )
] ]
end end

View file

@ -47,6 +47,14 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:error, "Cachex error: #{inspect(e)}"} {:error, "Cachex error: #{inspect(e)}"}
end end
@doc """
Get a filename for the fetched data, using the response header or the last part of the URL
"""
@spec get_filename_from_response(Enum.t(), String.t()) :: String.t() | nil
def get_filename_from_response(response_headers, url) do
get_filename_from_headers(response_headers) || get_filename_from_url(url)
end
@spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()} @spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()}
defp parse_url(url, options \\ []) do defp parse_url(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent()) user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())

View file

@ -1,50 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/controller.ex
defmodule Mobilizon.Web.MediaProxyController do
use Mobilizon.Web, :controller
alias Plug.Conn
alias Mobilizon.Config
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Web.ReverseProxy
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
:ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
end
end
def filename_matches(has_filename, path, url) do
filename =
url
|> MediaProxy.filename()
|> URI.decode()
path = URI.decode(path)
if has_filename && filename && Path.basename(path) != filename do
{:wrong_filename, filename}
else
:ok
end
end
end

View file

@ -1,91 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/media_proxy.ex
defmodule Mobilizon.Web.MediaProxy do
@moduledoc """
Handles proxifying media files
"""
alias Mobilizon.Config
alias Mobilizon.Web.Endpoint
@base64_opts [padding: false]
def url(nil), do: nil
def url(""), do: nil
def url("/" <> _ = url), do: url
def url(url) do
config = Application.get_env(:mobilizon, :media_proxy, [])
if !Keyword.get(config, :enabled, false) or
String.starts_with?(url, Endpoint.url()) do
url
else
encode_url(url)
end
end
def encode_url(url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
# Must preserve `%2F` for compatibility with S3
# https://git.pleroma.social/pleroma/pleroma/issues/580
replacement = get_replacement(url, ":2F:")
# The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
base64 =
url
|> String.replace("%2F", replacement)
|> URI.decode()
|> URI.encode()
|> String.replace(replacement, "%2F")
|> Base.url_encode64(@base64_opts)
sig = :crypto.hmac(:sha, secret, base64)
sig64 = sig |> Base.url_encode64(@base64_opts)
build_url(sig64, base64, filename(url))
end
def decode_url(sig, url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
sig = Base.url_decode64!(sig, @base64_opts)
local_sig = :crypto.hmac(:sha, secret, url)
if local_sig == sig do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
{:error, :invalid_signature}
end
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
[
Config.get([:media_proxy, :base_url], Endpoint.url()),
"proxy",
sig_base64,
url_base64,
filename
]
|> Enum.filter(fn value -> value end)
|> Path.join()
end
defp get_replacement(url, replacement) do
if String.contains?(url, replacement) do
get_replacement(url, replacement <> replacement)
else
replacement
end
end
end

View file

@ -69,8 +69,6 @@ defmodule Mobilizon.Web.ReverseProxy do
alias Plug.Conn alias Plug.Conn
alias Mobilizon.Web.MediaProxy
require Logger require Logger
@type option :: @type option ::
@ -111,7 +109,7 @@ defmodule Mobilizon.Web.ReverseProxy do
req_headers = build_req_headers(conn.req_headers, opts) req_headers = build_req_headers(conn.req_headers, opts)
opts = opts =
if filename = MediaProxy.filename(url) do if filename = filename(url) do
Keyword.put_new(opts, :attachment_name, filename) Keyword.put_new(opts, :attachment_name, filename)
else else
opts opts
@ -388,4 +386,8 @@ defmodule Mobilizon.Web.ReverseProxy do
defp increase_read_duration(_) do defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit} {:ok, :no_duration_limit, :no_duration_limit}
end end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
end end

View file

@ -162,13 +162,6 @@ defmodule Mobilizon.Web.Router do
post("/auth/:provider/callback", AuthController, :callback) post("/auth/:provider/callback", AuthController, :callback)
end end
scope "/proxy/", Mobilizon.Web do
pipe_through(:remote_media)
get("/:sig/:url", MediaProxyController, :remote)
get("/:sig/:url/:filename", MediaProxyController, :remote)
end
if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do
# If using Phoenix # If using Phoenix
forward("/sent_emails", Bamboo.SentEmailViewerPlug) forward("/sent_emails", Bamboo.SentEmailViewerPlug)

View file

@ -5,7 +5,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
alias Mobilizon.Addresses.Address alias Mobilizon.Addresses.Address
alias Mobilizon.Events.Event alias Mobilizon.Events.Event
alias Mobilizon.Posts.Post alias Mobilizon.Posts.Post
alias Mobilizon.Web.{Endpoint, MediaProxy} alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.JsonLD.ObjectView alias Mobilizon.Web.JsonLD.ObjectView
def render("group.json", %{group: %Actor{} = group}) do def render("group.json", %{group: %Actor{} = group}) do
@ -41,7 +41,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
"image" => "image" =>
if(event.picture, if(event.picture,
do: [ do: [
event.picture.file.url |> MediaProxy.url() event.picture.file.url
], ],
else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"] else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"]
) )

View file

@ -264,7 +264,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Plugs.UploadedMedia, Mobilizon.Web.Plugs.UploadedMedia,
Mobilizon.Web.FallbackController, Mobilizon.Web.FallbackController,
Mobilizon.Web.FeedController, Mobilizon.Web.FeedController,
Mobilizon.Web.MediaProxyController,
Mobilizon.Web.PageController, Mobilizon.Web.PageController,
Mobilizon.Web.ChangesetView, Mobilizon.Web.ChangesetView,
Mobilizon.Web.JsonLD.ObjectView, Mobilizon.Web.JsonLD.ObjectView,
@ -295,7 +294,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Upload.MIME, Mobilizon.Web.Upload.MIME,
Mobilizon.Web.Upload.Uploader, Mobilizon.Web.Upload.Uploader,
Mobilizon.Web.Upload.Uploader.Local, Mobilizon.Web.Upload.Uploader.Local,
Mobilizon.Web.MediaProxy,
Mobilizon.Web.ReverseProxy Mobilizon.Web.ReverseProxy
], ],
Geospatial: [ Geospatial: [

View file

@ -45,13 +45,13 @@ defmodule Mobilizon.Federation.ActivityPubTest do
use_cassette "activity_pub/fetch_tcit@framapiaf.org" do use_cassette "activity_pub/fetch_tcit@framapiaf.org" do
assert {:ok, assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :public} = %Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :public} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org") _actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end end
use_cassette "activity_pub/fetch_tcit@framapiaf.org_not_discoverable" do use_cassette "activity_pub/fetch_tcit@framapiaf.org_not_discoverable" do
assert {:ok, assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :unlisted} = %Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :unlisted} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org") _actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end end
end end

View file

@ -39,7 +39,7 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
Transmogrifier.handle_incoming(data) Transmogrifier.handle_incoming(data)
assert data["id"] == assert data["id"] ==
"https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity" "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity"
assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@ -49,12 +49,12 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
# "http://localtesting.pleroma.lol/users/lain" # "http://localtesting.pleroma.lol/users/lain"
# ] # ]
assert data["actor"] == "https://test.mobilizon.org/@Alicia" assert data["actor"] == "https://mobilizon.fr/@metacartes"
object = data["object"] object = data["object"]
assert object["id"] == assert object["id"] ==
"https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0" "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93"
assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"] assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@ -63,9 +63,9 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
# "http://localtesting.pleroma.lol/users/lain" # "http://localtesting.pleroma.lol/users/lain"
# ] # ]
assert object["actor"] == "https://test.mobilizon.org/@Alicia" assert object["actor"] == "https://mobilizon.fr/@metacartes"
assert object["location"]["name"] == "Locaux de Framasoft" assert object["location"]["name"] == "Locaux de Framasoft"
# assert object["attributedTo"] == "https://test.mobilizon.org/@Alicia" # assert object["attributedTo"] == "https://mobilizon.fr/@metacartes"
assert event.physical_address.street == "10 Rue Jangot" assert event.physical_address.street == "10 Rue Jangot"
@ -84,8 +84,8 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
%Actor{url: actor_url, id: actor_id} = %Actor{url: actor_url, id: actor_id} =
actor = actor =
insert(:actor, insert(:actor,
domain: "test.mobilizon.org", domain: "mobilizon.fr",
url: "https://test.mobilizon.org/@member", url: "https://mobilizon.fr/@member",
preferred_username: "member" preferred_username: "member"
) )

View file

@ -20,15 +20,15 @@
"endpoints": { "endpoints": {
"sharedInbox": "https://framapiaf.org/inbox" "sharedInbox": "https://framapiaf.org/inbox"
}, },
"icon":{ "icon": {
"type":"Image", "type": "Image",
"mediaType":"image/png", "mediaType": "image/png",
"url":"https://files.mastodon.social/accounts/avatars/000/000/001/original/a285c086605e4182.png" "url": "https://files.mastodon.social/accounts/avatars/000/000/001/original/d96d39a0abb45b92.jpg"
}, },
"image":{ "image": {
"type":"Image", "type": "Image",
"mediaType":"image/png", "mediaType": "image/png",
"url":"https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png" "url": "https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png"
} }
}, },
"id": "https://framapiaf.org/users/gargron#updates/1519563538", "id": "https://framapiaf.org/users/gargron#updates/1519563538",

View file

@ -28,12 +28,12 @@
"uuid": "sc:identifier" "uuid": "sc:identifier"
} }
], ],
"actor": "https://test.mobilizon.org/@Alicia", "actor": "https://mobilizon.fr/@metacartes",
"cc": [ "cc": [
"https://framapiaf.org/users/admin/followers", "https://framapiaf.org/users/admin/followers",
"https://framapiaf.org/users/tcit" "https://framapiaf.org/users/tcit"
], ],
"id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity", "id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity",
"object": { "object": {
"attachment": [ "attachment": [
{ {
@ -49,7 +49,7 @@
"type": "Link" "type": "Link"
} }
], ],
"attributedTo": "https://test.mobilizon.org/@Alicia", "attributedTo": "https://mobilizon.fr/@metacartes",
"startTime": "2018-02-12T14:08:20Z", "startTime": "2018-02-12T14:08:20Z",
"cc": [ "cc": [
"https://framapiaf.org/users/admin/followers", "https://framapiaf.org/users/admin/followers",
@ -57,7 +57,7 @@
], ],
"content": "<p><span class=\"h-card\"><a href=\"https://framapiaf.org/users/tcit\" class=\"u-url mention\">@<span>tcit</span></a></span></p>", "content": "<p><span class=\"h-card\"><a href=\"https://framapiaf.org/users/tcit\" class=\"u-url mention\">@<span>tcit</span></a></span></p>",
"category": "TODO remove me", "category": "TODO remove me",
"id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0", "id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
"inReplyTo": null, "inReplyTo": null,
"location": { "location": {
"type": "Place", "type": "Place",
@ -81,16 +81,12 @@
"type": "Mention" "type": "Mention"
} }
], ],
"to": [ "to": ["https://www.w3.org/ns/activitystreams#Public"],
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Event", "type": "Event",
"url": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0", "url": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
"uuid": "109ccdfd-ee3e-46e1-a877-6c228763df0c" "uuid": "109ccdfd-ee3e-46e1-a877-6c228763df0c"
}, },
"published": "2018-02-12T14:08:20Z", "published": "2018-02-12T14:08:20Z",
"to": [ "to": ["https://www.w3.org/ns/activitystreams#Public"],
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Create" "type": "Create"
} }

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -108,7 +108,7 @@ defmodule Mobilizon.ActorsTest do
avatar: %FileModel{name: picture_name} = _picture avatar: %FileModel{name: picture_name} = _picture
} = _actor} = ActivityPub.get_or_fetch_actor_by_url(@remote_account_url) } = _actor} = ActivityPub.get_or_fetch_actor_by_url(@remote_account_url)
assert picture_name == "avatar" assert picture_name == "a28c50ce5f2b13fd.jpg"
%Actor{ %Actor{
id: actor_found_id, id: actor_found_id,
@ -116,7 +116,7 @@ defmodule Mobilizon.ActorsTest do
} = Actors.get_actor_by_name("#{preferred_username}@#{domain}") } = Actors.get_actor_by_name("#{preferred_username}@#{domain}")
assert actor_found_id == actor_id assert actor_found_id == actor_id
assert picture_name == "avatar" assert picture_name == "a28c50ce5f2b13fd.jpg"
end end
end end

View file

@ -52,7 +52,7 @@ defmodule Mobilizon.Web.Plugs.MappedSignatureToIdentityTest do
use_cassette "activity_pub/signature/invalid_not_found" do use_cassette "activity_pub/signature/invalid_not_found" do
conn = conn =
build_conn(:post, "/doesntmattter", %{"actor" => "https://framapiaf.org/users/admin"}) build_conn(:post, "/doesntmattter", %{"actor" => "https://framapiaf.org/users/admin"})
|> set_signature("http://niu.moe/users/rye") |> set_signature("https://mastodon.social/users/gargron")
|> MappedSignatureToIdentity.call(%{}) |> MappedSignatureToIdentity.call(%{})
assert %{valid_signature: false} == conn.assigns assert %{valid_signature: false} == conn.assigns

View file

@ -1,185 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/test/media_proxy_test.ex
defmodule Mobilizon.Web.MediaProxyTest do
use ExUnit.Case
import Mobilizon.Web.MediaProxy
alias Mobilizon.Config
alias Mobilizon.Web.{Endpoint, MediaProxyController}
setup do
enabled = Config.get([:media_proxy, :enabled])
on_exit(fn -> Config.put([:media_proxy, :enabled], enabled) end)
:ok
end
describe "when enabled" do
setup do
Config.put([:media_proxy, :enabled], true)
:ok
end
test "ignores invalid url" do
assert url(nil) == nil
assert url("") == nil
end
test "ignores relative url" do
assert url("/local") == "/local"
assert url("/") == "/"
end
test "ignores local url" do
local_url = Endpoint.url() <> "/hello"
local_root = Endpoint.url()
assert url(local_url) == local_url
assert url(local_root) == local_root
end
test "encodes and decodes URL" do
url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url)
assert String.starts_with?(
encoded,
Config.get([:media_proxy, :base_url], Endpoint.url())
)
assert String.ends_with?(encoded, "/logo.png")
assert decode_result(encoded) == url
end
test "encodes and decodes URL without a path" do
url = "https://pleroma.soykaf.com"
encoded = url(url)
assert decode_result(encoded) == url
end
test "encodes and decodes URL without an extension" do
url = "https://pleroma.soykaf.com/path/"
encoded = url(url)
assert String.ends_with?(encoded, "/path")
assert decode_result(encoded) == url
end
test "encodes and decodes URL and ignores query params for the path" do
url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true"
encoded = url(url)
assert String.ends_with?(encoded, "/logo.png")
assert decode_result(encoded) == url
end
test "ensures urls are url-encoded" do
assert decode_result(url("https://pleroma.social/Hello world.jpg")) ==
"https://pleroma.social/Hello%20world.jpg"
assert decode_result(url("https://pleroma.social/Hello%20world.jpg")) ==
"https://pleroma.social/Hello%20world.jpg"
end
test "validates signature" do
secret_key_base = Config.get([Endpoint, :secret_key_base])
on_exit(fn ->
Config.put([Endpoint, :secret_key_base], secret_key_base)
end)
encoded = url("https://pleroma.social")
Config.put(
[Endpoint, :secret_key_base],
"00000000000000000000000000000000000000000000000"
)
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
assert decode_url(sig, base64) == {:error, :invalid_signature}
end
test "filename_matches matches url encoded paths" do
assert MediaProxyController.filename_matches(
true,
"/Hello%20world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
true,
"/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
end
test "filename_matches matches non-url encoded paths" do
assert MediaProxyController.filename_matches(
true,
"/Hello world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
true,
"/Hello world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
end
test "uses the configured base_url" do
base_url = Config.get([:media_proxy, :base_url])
if base_url do
on_exit(fn ->
Config.put([:media_proxy, :base_url], base_url)
end)
end
Config.put([:media_proxy, :base_url], "https://cache.pleroma.social")
url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url)
assert String.starts_with?(encoded, Config.get([:media_proxy, :base_url]))
end
# https://git.pleroma.social/pleroma/pleroma/issues/580
test "encoding S3 links (must preserve `%2F`)" do
url =
"https://s3.amazonaws.com/example/test.png?X-Amz-Credential=your-access-key-id%2F20130721%2Fus-east-1%2Fs3%2Faws4_request"
encoded = url(url)
assert decode_result(encoded) == url
end
end
describe "when disabled" do
setup do
enabled = Config.get([:media_proxy, :enabled])
if enabled do
Config.put([:media_proxy, :enabled], false)
on_exit(fn ->
Config.put([:media_proxy, :enabled], enabled)
:ok
end)
end
:ok
end
test "does not encode remote urls" do
assert url("https://google.fr") == "https://google.fr"
end
end
defp decode_result(encoded) do
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
{:ok, decoded} = decode_url(sig, base64)
decoded
end
end