Introduce Sitemaps

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2020-08-12 16:05:34 +02:00
parent 9a0068dfea
commit c56fb710b6
No known key found for this signature in database
GPG key ID: A061B9DDE0CA0773
21 changed files with 163 additions and 39 deletions

View file

@ -236,7 +236,10 @@ config :mobilizon, :anonymous,
config :mobilizon, Oban,
repo: Mobilizon.Storage.Repo,
log: false,
queues: [default: 10, search: 5, mailers: 10, background: 5]
queues: [default: 10, search: 5, mailers: 10, background: 5],
crontab: [
{"@daily", Mobilizon.Service.Workers.BuildSiteMap, queue: :background}
]
config :mobilizon, :rich_media,
parsers: [

View file

@ -56,7 +56,6 @@ defmodule Mobilizon.Actors do
:rejected
])
@public_visibility [:public, :unlisted]
@administrator_roles [:creator, :administrator]
@moderator_roles [:moderator] ++ @administrator_roles
@member_roles [:member] ++ @moderator_roles
@ -537,6 +536,15 @@ defmodule Mobilizon.Actors do
|> Page.build_page(page, limit)
end
@doc """
Lists the groups.
"""
@spec list_groups_for_stream :: Enum.t()
def list_groups_for_stream do
groups_query()
|> Repo.stream()
end
@doc """
Returns the list of groups an actor is member of.
"""
@ -1212,7 +1220,7 @@ defmodule Mobilizon.Actors do
from(
a in Actor,
where: a.type == ^:Group,
where: a.visibility in ^@public_visibility
where: a.visibility == ^:public
)
end

View file

@ -366,6 +366,15 @@ defmodule Mobilizon.Events do
|> Repo.all()
end
@spec stream_events_for_sitemap :: Enum.t()
def stream_events_for_sitemap do
Event
|> filter_public_visibility()
|> filter_draft()
|> filter_local()
|> Repo.stream()
end
@doc """
Returns the list of events with the same tags.
"""
@ -1583,6 +1592,11 @@ defmodule Mobilizon.Events do
defp filter_future_events(query, false), do: query
@spec filter_local(Ecto.Query.t()) :: Ecto.Query.t()
defp filter_local(query) do
where(query, [q], q.local == true)
end
@spec filter_local_or_from_followed_instances_events(Ecto.Query.t()) :: Ecto.Query.t()
defp filter_local_or_from_followed_instances_events(query) do
from(q in query,

View file

@ -21,6 +21,13 @@ defmodule Mobilizon.Posts do
:private
])
@spec list_posts_for_stream :: Enum.t()
def list_posts_for_stream do
Post
|> filter_public()
|> Repo.stream()
end
@doc """
Returns the list of recent posts for a group
"""
@ -35,7 +42,7 @@ defmodule Mobilizon.Posts do
def get_public_posts_for_group(%Actor{id: group_id}, page \\ nil, limit \\ nil) do
group_id
|> do_get_posts_for_group()
|> where([p], p.visibility == ^:public and not p.draft)
|> filter_public()
|> Page.build_page(page, limit)
end
@ -132,4 +139,9 @@ defmodule Mobilizon.Posts do
where: p.post_id == ^post_id
)
end
@spec filter_public(Ecto.Query.t()) :: Ecto.Query.t()
defp filter_public(query) do
where(query, [p], p.visibility == ^:public and not p.draft)
end
end

View file

@ -43,7 +43,7 @@ defmodule Mobilizon.Storage.Ecto do
|> put_change(:id, uuid)
|> put_change(
:url,
apply(Routes, String.to_existing_atom("#{to_string(route)}_url"), [Endpoint, route, uuid])
apply(Routes, String.to_existing_atom("page_url"), [Endpoint, route, uuid])
)
end
end

66
lib/service/site_map.ex Normal file
View file

@ -0,0 +1,66 @@
defmodule Mobilizon.Service.SiteMap do
@moduledoc """
Generates a sitemap
"""
alias Mobilizon.{Actors, Events, Posts}
alias Mobilizon.Storage.Repo
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.Router.Helpers, as: Routes
@default_static_frequency :monthly
def generate_sitemap do
static_routes = [
{Routes.page_url(Endpoint, :index, []), :daily},
"#{Endpoint.url()}/about/instance",
"#{Endpoint.url()}/about/mobilizon",
"#{Endpoint.url()}/terms",
"#{Endpoint.url()}/privacy",
"#{Endpoint.url()}/rules",
"#{Endpoint.url()}/glossary"
]
config = [
store: Sitemapper.FileStore,
store_config: [path: "priv/static"],
sitemap_url: Endpoint.url(),
gzip: false
]
Repo.transaction(fn ->
Events.stream_events_for_sitemap()
|> Stream.concat(Actors.list_groups_for_stream())
|> Stream.concat(Posts.list_posts_for_stream())
|> Stream.concat(
Enum.map(static_routes, fn route ->
{url, frequency} =
case route do
{url, frequency} -> {url, frequency}
url when is_binary(url) -> {url, @default_static_frequency}
end
%{url: url, updated_at: nil, frequence: frequency}
end)
)
|> Stream.map(fn %{url: url, updated_at: updated_at} = args ->
frequence = Map.get(args, :frequence, :weekly)
%Sitemapper.URL{
loc: url,
changefreq: frequence,
lastmod: check_date_time(updated_at)
}
end)
|> Sitemapper.generate(config)
|> Sitemapper.persist(config)
|> Sitemapper.ping(config)
|> Stream.run()
end)
end
# Sometimes we use naive datetimes
defp check_date_time(%NaiveDateTime{} = datetime), do: DateTime.from_naive!(datetime, "Etc/UTC")
defp check_date_time(%DateTime{} = datetime), do: datetime
defp check_date_time(_), do: nil
end

View file

@ -0,0 +1,12 @@
defmodule Mobilizon.Service.Workers.BuildSiteMap do
@moduledoc """
Worker to build sitemap
"""
alias Mobilizon.Service.SiteMap
use Oban.Worker, queue: "background"
@impl Oban.Worker
def perform(%Job{}), do: SiteMap.generate_sitemap()
end

View file

@ -8,11 +8,21 @@ defmodule Mobilizon.Web.PageController do
alias Mobilizon.Events.Event
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Tombstone
alias Mobilizon.Web.{ActivityPubController, Cache}
alias Mobilizon.Web.{ActivityPubController, Cache, PageController}
plug(:put_layout, false)
action_fallback(Mobilizon.Web.FallbackController)
defdelegate my_events(conn, params), to: PageController, as: :index
defdelegate create_event(conn, params), to: PageController, as: :index
defdelegate list_events(conn, params), to: PageController, as: :index
defdelegate explore_events(conn, params), to: PageController, as: :index
defdelegate edit_event(conn, params), to: PageController, as: :index
defdelegate moderation_report(conn, params), to: PageController, as: :index
defdelegate participation_email_confirmation(conn, params), to: PageController, as: :index
defdelegate user_email_validation(conn, params), to: PageController, as: :index
defdelegate my_groups(conn, params), to: PageController, as: :index
@spec index(Plug.Conn.t(), any) :: Plug.Conn.t()
def index(conn, _params), do: render(conn, :index)

View file

@ -81,12 +81,12 @@ defmodule Mobilizon.Web.Router do
pipe_through(:activity_pub_signature)
get("/@:name", PageController, :actor)
get("/events/me", PageController, :index)
get("/events/me", PageController, :my_events)
get("/events/:uuid", PageController, :event)
get("/comments/:uuid", PageController, :comment)
get("/resource/:uuid", PageController, :resource, as: "resource")
get("/todo-list/:uuid", PageController, :todo_list, as: "todo_list")
get("/todo/:uuid", PageController, :todo, as: "todo")
get("/resource/:uuid", PageController, :resource)
get("/todo-list/:uuid", PageController, :todo_list)
get("/todo/:uuid", PageController, :todo)
get("/@:name/todos", PageController, :todos)
get("/@:name/resources", PageController, :resources)
get("/@:name/posts", PageController, :posts)
@ -139,21 +139,20 @@ defmodule Mobilizon.Web.Router do
pipe_through(:browser)
# Because the "/events/:uuid" route caches all these, we need to force them
get("/events/create", PageController, :index)
get("/events/list", PageController, :index)
get("/events/me", PageController, :index)
get("/events/explore", PageController, :index)
get("/events/:uuid/edit", PageController, :index)
get("/events/create", PageController, :create_event)
get("/events/list", PageController, :list_events)
get("/events/me", PageController, :my_events)
get("/events/explore", PageController, :explore_events)
get("/events/:uuid/edit", PageController, :edit_event)
# This is a hack to ease link generation into emails
get("/moderation/reports/:id", PageController, :index, as: "moderation_report")
get("/moderation/reports/:id", PageController, :moderation_report)
get("/participation/email/confirm/:token", PageController, :index,
as: "participation_email_confirmation"
)
get("/participation/email/confirm/:token", PageController, :participation_email_confirmation)
get("/validate/email/:token", PageController, :index, as: "user_email_validation")
get("/groups/me", PageController, :index, as: "my_groups")
get("/validate/email/:token", PageController, :user_email_validation)
get("/groups/me", PageController, :my_groups)
get("/interact", PageController, :interact)

View file

@ -54,7 +54,7 @@
<td bgcolor="#ffffff" align="center" style="padding: 20px 30px 60px 30px;">
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E"><a href="<%= participation_email_confirmation_url(Mobilizon.Web.Endpoint, :index, @participant.metadata.confirmation_token) %>" target="_blank" style="font-size: 20px; font-family: 'Roboto', Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E"><a href="<%= page_url(Mobilizon.Web.Endpoint, :participation_email_confirmation, @participant.metadata.confirmation_token) %>" target="_blank" style="font-size: 20px; font-family: 'Roboto', Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<%= gettext "Confirm my e-mail address" %>
</a></td>
</tr>

View file

@ -2,5 +2,5 @@
==
<%= gettext "Hi there! You just registered to join this event: « %{title} ». Please confirm the e-mail address you provided:", title: @participant.event.title %>
<%= gettext "If you didn't trigger this email, you may safely ignore it." %>
<%= participation_email_confirmation_url(Mobilizon.Web.Endpoint, :index, @participant.metadata.confirmation_token) %>
<%= page_url(Mobilizon.Web.Endpoint, :participation_email_confirmation, @participant.metadata.confirmation_token) %>
<%= ngettext "Would you wish to cancel your attendance, visit the event page through the link above and click the « Attending » button.", "Would you wish to cancel your attendance to one or several events, visit the event pages through the links above and click the « Attending » button.", 1 %>

View file

@ -47,7 +47,7 @@
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E">
<a href="<%= user_email_validation_url(Mobilizon.Web.Endpoint, :index, @token) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<a href="<%= page_url(Mobilizon.Web.Endpoint, :user_email_validation, @token) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<%= gettext "Verify your email address" %>
</a>
</td>

View file

@ -1,5 +1,5 @@
<%= gettext "Confirm new email" %>
==
<%= gettext "Hi there! It seems like you wanted to change the email address linked to your account on <b>%{instance}</b>. If you still wish to do so, please click the button below to confirm the change. You will then be able to log in to %{instance} with this new email address.", %{instance: @instance_name} %>
<%= user_email_validation_url(Mobilizon.Web.Endpoint, :index, @token) %>
<%= page_url(Mobilizon.Web.Endpoint, :user_email_validation, @token) %>
<%= gettext "If you didn't trigger the change yourself, please ignore this message." %>

View file

@ -55,7 +55,7 @@
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E">
<a href="<%= my_groups_url(Mobilizon.Web.Endpoint, :index) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<a href="<%= page_url(Mobilizon.Web.Endpoint, :my_groups) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<%= gettext "See my groups" %>
</a>
</td>

View file

@ -1,9 +1,5 @@
<%= gettext "Come along!" %>
==
<%= gettext "%{inviter} just invited you to join their group %{group}", group: @group.name, inviter: @inviter.name %>
<%= gettext "To accept this invitation, head over to your groups." %>
<%= my_groups_url(Mobilizon.Web.Endpoint, :index) %>
<%= page_url(Mobilizon.Web.Endpoint, :my_groups) %>

View file

@ -100,7 +100,7 @@
<td bgcolor="#ffffff" align="center" style="padding: 20px 30px 60px 30px;">
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E"><a href="<%= moderation_report_url(Mobilizon.Web.Endpoint, :index, @report.id) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<td align="center" style="border-radius: 3px;" bgcolor="#3C376E"><a href="<%= page_url(Mobilizon.Web.Endpoint, :moderation_report, @report.id) %>" target="_blank" style="font-size: 20px; font-family: Helvetica, Arial, sans-serif; color: #ffffff; text-decoration: none; padding: 15px 25px; border-radius: 2px; border: 1px solid #3C376E; display: inline-block;">
<%= gettext "View report" %>
</a></td>
</tr>

View file

@ -14,4 +14,4 @@
<%= gettext "Reason" %>
<%= @report.content %>
<% end %>
<%= gettext "View report:" %> <%= moderation_report_url(Mobilizon.Web.Endpoint, :index, @report.id) %>
<%= gettext "View report:" %> <%= page_url(Mobilizon.Web.Endpoint, :moderation_report, @report.id) %>

View file

@ -132,6 +132,8 @@ defmodule Mobilizon.Mixfile do
git: "https://github.com/tcitworld/ueberauth_gitlab.git", branch: "upgrade-deps"},
{:ecto_shortuuid, "~> 0.1"},
{:tesla, "~> 1.3.0"},
{:sitemapper, "~> 0.4.0"},
{:xml_builder, "~> 2.1.1", override: true},
# Dev and test dependencies
{:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
{:ex_machina, "~> 2.3", only: [:dev, :test]},

View file

@ -119,6 +119,7 @@
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
"rsa_ex": {:hex, :rsa_ex, "0.4.0", "e28dd7dc5236e156df434af0e4aa822384c8866c928e17b785d4edb7c253b558", [:mix], [], "hexpm", "40e1f08e8401da4be59a6dd0f4da30c42d5bb01703161f0208d839d97db27f4e"},
"shortuuid": {:hex, :shortuuid, "2.1.2", "14dbafdb2f6c7213fdfcc05c7572384b5051a7b1621170018ad4c05504bd96c1", [:mix], [], "hexpm", "d9b0c4f37500ea5199b6275ece872e213e9f45a015caf4aa777cec84f63ad353"},
"sitemapper": {:hex, :sitemapper, "0.4.0", "50061503ddc306aabcb984b377415961ff49696d70cd95081b20fa2a86f18ac4", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "9bfe778635a6801e7762b185564df6a174d6016b15cbeaf50746e94ee55138c3"},
"sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},
"slugger": {:hex, :slugger, "0.3.0", "efc667ab99eee19a48913ccf3d038b1fb9f165fa4fbf093be898b8099e61b6ed", [:mix], [], "hexpm", "20d0ded0e712605d1eae6c5b4889581c3460d92623a930ddda91e0e609b5afba"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
@ -139,5 +140,5 @@
"unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"},
"unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
"uuid": {:git, "git://github.com/botsunit/erlang-uuid", "1effbbbd200f9f5d9d5154e81b83fe8e4c3fe714", [branch: "master"]},
"xml_builder": {:hex, :xml_builder, "2.0.0", "371ed27bb63bf0598dbaf3f0c466e5dc7d16cb4ecb68f06a67f953654062e21b", [:mix], [], "hexpm", "baeb5c8d42204bac2b856ffd50e8cda42d63b622984538d18d92733e4e790fbd"},
"xml_builder": {:hex, :xml_builder, "2.1.2", "90cb9ad382958934c78c6ddfbe6d385a8ce147d84b61cbfa83ec93a169d0feab", [:mix], [], "hexpm", "b89046041da2fbc1d51d31493ba31b9d5fc6223c93384bf513a1a9e1df9ec081"},
}

View file

@ -87,8 +87,9 @@ defmodule Mobilizon.Web.Resolvers.GroupTest do
end
describe "list groups" do
test "list_groups/3 returns all public or unlisted groups", %{conn: conn} do
group = insert(:group, visibility: :unlisted)
test "list_groups/3 returns all public groups", %{conn: conn} do
group = insert(:group, visibility: :public)
insert(:group, visibility: :unlisted)
insert(:group, visibility: :private)
query = """

View file

@ -285,7 +285,7 @@ defmodule Mobilizon.Factory do
title: sequence("todo list"),
actor: build(:group),
id: uuid,
url: Routes.todo_list_url(Endpoint, :todo_list, uuid)
url: Routes.page_url(Endpoint, :todo_list, uuid)
}
end
@ -299,7 +299,7 @@ defmodule Mobilizon.Factory do
status: false,
due_date: Timex.shift(DateTime.utc_now(), hours: 2),
assigned_to: build(:actor),
url: Routes.todo_url(Endpoint, :todo, uuid),
url: Routes.page_url(Endpoint, :todo, uuid),
creator: build(:actor)
}
end
@ -316,7 +316,7 @@ defmodule Mobilizon.Factory do
actor: build(:group),
creator: build(:actor),
parent: nil,
url: Routes.resource_url(Endpoint, :resource, uuid),
url: Routes.page_url(Endpoint, :resource, uuid),
path: "/#{title}"
}
end