Fix rich media parsers

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2021-05-03 14:52:37 +02:00
parent 46120b16b6
commit 5b36e71581
No known key found for this signature in database
GPG key ID: A061B9DDE0CA0773
5 changed files with 39 additions and 17 deletions

View file

@ -3,25 +3,25 @@
exports[`CommentTree renders a comment tree 1`] = `
<div>
<!---->
<transition-stub name="comment-empty-list" mode="out-in">
<transition-group-stub name="comment-empty-list" mode="out-in">
<transition-group-stub tag="ul" name="comment-list" class="comment-list">
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
</transition-group-stub>
<div class="no-comments"><span>No comments yet</span></div>
</transition-stub>
</transition-group-stub>
</div>
`;
exports[`CommentTree renders a comment tree 2`] = `
<div>
<!---->
<transition-stub name="comment-empty-list" mode="out-in">
<transition-group-stub name="comment-empty-list" mode="out-in">
<transition-group-stub tag="ul" name="comment-list" class="comment-list">
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
<comment-stub comment="[object Object]" event="[object Object]" class="root-comment"></comment-stub>
</transition-group-stub>
<div class="no-comments"><span>No comments yet</span></div>
</transition-stub>
</transition-group-stub>
</div>
`;

View file

@ -74,12 +74,11 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:is_html, _response_headers, true} <-
{:is_html, response_headers, is_html(response_headers)} do
body
|> parse_html()
|> maybe_parse()
|> Map.put(:url, url)
|> maybe_add_favicon()
|> clean_parsed_data()
|> check_parsed_data()
|> check_parsed_data(body)
|> check_remote_picture_path()
else
{:is_html, response_headers, false} ->
@ -193,8 +192,7 @@ defmodule Mobilizon.Service.RichMedia.Parser do
end
end
defp parse_html(html), do: Floki.parse_document!(html)
@spec maybe_parse(String.t()) :: {:halt, map()} | {:cont, map()}
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do
@ -207,7 +205,9 @@ defmodule Mobilizon.Service.RichMedia.Parser do
end)
end
defp check_parsed_data(%{title: title} = data)
defp check_parsed_data(data, html, first_run \\ true)
defp check_parsed_data(%{title: title} = data, _html, _first_run)
when is_binary(title) and byte_size(title) > 0 do
data
end

View file

@ -29,11 +29,19 @@ defmodule Mobilizon.Service.RichMedia.Parsers.Fallback do
end
defp get_page(html, :title) do
html |> Floki.find("html head title") |> List.first() |> Floki.text() |> String.trim()
html
|> Floki.parse_document!()
|> Floki.find("html title")
|> List.first()
|> Floki.text()
|> String.trim()
end
defp get_page(html, :description) do
case html |> Floki.find("html head meta[name='description']") |> List.first() do
case html
|> Floki.parse_document!()
|> Floki.find("html meta[name='description']")
|> List.first() do
nil -> ""
elem -> elem |> Floki.attribute("content") |> List.first() |> String.trim()
end

View file

@ -36,7 +36,7 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
end
defp get_elements(html, key_name, prefix) do
html |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']")
html |> Floki.parse_document!() |> Floki.find("meta[#{to_string(key_name)}^='#{prefix}:']")
end
defp normalize_attributes(html_node, prefix, key_name, value_name, allowed_attributes) do
@ -83,14 +83,26 @@ defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
defp maybe_put_description(meta, _), do: meta
@spec get_page_title(String.t()) :: String.t()
defp get_page_title(html) do
html |> Floki.find("html head title") |> List.first() |> Floki.text()
with {:ok, document} <- Floki.parse_document(html),
elem when not is_nil(elem) <- document |> Floki.find("html head title") |> List.first(),
title when is_binary(title) <- Floki.text(elem) do
title
else
_ -> ""
end
end
@spec get_page_description(String.t()) :: String.t()
defp get_page_description(html) do
case html |> Floki.find("html head meta[name='description']") |> List.first() do
nil -> ""
elem -> Floki.attribute(elem, "content")
with {:ok, document} <- Floki.parse_document(html),
elem when not is_nil(elem) <-
document |> Floki.find("html head meta[name='description']") |> List.first(),
description when is_binary(description) <- Floki.attribute(elem, "content") do
description
else
_ -> ""
end
end
end

View file

@ -32,7 +32,9 @@ defmodule Mobilizon.Service.RichMedia.Parsers.OEmbed do
end
defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
with {:ok, document} <- Floki.parse_document(html) do
Floki.find(document, "link[type='application/json+oembed']")
end
end
defp get_oembed_url(nodes) do