Introduce group basic federation, event new page and notifications

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

lib/service/rich_media/favicon.ex

@@ -0,0 +1,104 @@
+defmodule Mobilizon.Service.RichMedia.Favicon do
+  @moduledoc """
+  Module to fetch favicon information from a website
+
+  Taken and adapted from https://github.com/ricn/favicon
+  """
+
+  require Logger
+  alias Mobilizon.Config
+
+  @options [
+    max_body: 2_000_000,
+    timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    ssl: [{:versions, [:"tlsv1.2"]}]
+  ]
+
+  @spec fetch(String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
+  def fetch(url, options \\ []) do
+    user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())
+    headers = [{"User-Agent", user_agent}]
+
+    case HTTPoison.get(url, headers, @options) do
+      {:ok, %HTTPoison.Response{status_code: code, body: body}} when code in 200..299 ->
+        find_favicon_url(url, body, headers)
+
+      {:ok, %HTTPoison.Response{}} ->
+        {:error, "Error while fetching the page"}
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        {:error, reason}
+    end
+  end
+
+  @spec find_favicon_url(String.t(), String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
+  defp find_favicon_url(url, body, headers) do
+    Logger.debug("finding favicon URL for #{url}")
+
+    case find_favicon_link_tag(body) do
+      {:ok, tag} ->
+        Logger.debug("Found link #{inspect(tag)}")
+        {"link", attrs, _} = tag
+
+        {"href", path} =
+          Enum.find(attrs, fn {name, _} ->
+            name == "href"
+          end)
+
+        {:ok, format_url(url, path)}
+
+      _ ->
+        find_favicon_in_root(url, headers)
+    end
+  end
+
+  @spec format_url(String.t(), String.t()) :: String.t()
+  defp format_url(url, path) do
+    image_uri = URI.parse(path)
+    uri = URI.parse(url)
+
+    cond do
+      is_nil(image_uri.host) -> "#{uri.scheme}://#{uri.host}#{path}"
+      is_nil(image_uri.scheme) -> "#{uri.scheme}:#{path}"
+      true -> path
+    end
+  end
+
+  @spec find_favicon_link_tag(String.t()) :: {:ok, tuple()} | {:error, any()}
+  defp find_favicon_link_tag(html) do
+    with {:ok, html} <- Floki.parse_document(html),
+         links <- Floki.find(html, "link"),
+         {:link, link} when not is_nil(link) <-
+           {:link,
+            Enum.find(links, fn {"link", attrs, _} ->
+              Enum.any?(attrs, fn {name, value} ->
+                name == "rel" && String.contains?(value, "icon") &&
+                  !String.contains?(value, "-icon-")
+              end)
+            end)} do
+      {:ok, link}
+    else
+      {:link, nil} -> {:error, "No link found"}
+      err -> err
+    end
+  end
+
+  @spec find_favicon_in_root(String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
+  defp find_favicon_in_root(url, headers) do
+    uri = URI.parse(url)
+    favicon_url = "#{uri.scheme}://#{uri.host}/favicon.ico"
+
+    case HTTPoison.head(favicon_url, headers, @options) do
+      {:ok, %HTTPoison.Response{status_code: code}} when code in 200..299 ->
+        {:ok, favicon_url}
+
+      {:ok, %HTTPoison.Response{}} ->
+        {:error, "Error while doing a HEAD request on the favicon"}
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        {:error, reason}
+    end
+  end
+end

lib/service/rich_media/parser.ex

@@ -0,0 +1,278 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parser do
+  @moduledoc """
+  Module to parse data in HTML pages
+  """
+  @options [
+    max_body: 2_000_000,
+    timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    # TODO: Remove me once Hackney/HTTPoison fixes their shit with TLS1.3 and OTP 23
+    ssl: [{:versions, [:"tlsv1.2"]}]
+  ]
+
+  alias Mobilizon.Config
+  alias Mobilizon.Service.RichMedia.Favicon
+  alias Plug.Conn.Utils
+  require Logger
+
+  defp parsers do
+    Mobilizon.Config.get([:rich_media, :parsers])
+  end
+
+  def parse(nil), do: {:error, "No URL provided"}
+
+  @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
+  def parse(url) do
+    case Cachex.fetch(:rich_media_cache, url, fn _ ->
+           case parse_url(url) do
+             {:ok, data} -> {:commit, data}
+             {:error, err} -> {:ignore, err}
+           end
+         end) do
+      {status, value} when status in [:ok, :commit] ->
+        {:ok, value}
+
+      {_, err} ->
+        {:error, err}
+    end
+  rescue
+    e ->
+      {:error, "Cachex error: #{inspect(e)}"}
+  end
+
+  @spec parse_url(String.t(), List.t()) :: {:ok, map()} | {:error, any()}
+  defp parse_url(url, options \\ []) do
+    user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())
+    headers = [{"User-Agent", user_agent}]
+    Logger.debug("Fetching content at address #{inspect(url)}")
+
+    try do
+      with {:ok, _} <- prevent_local_address(url),
+           {:ok, %HTTPoison.Response{body: body, status_code: code, headers: response_headers}}
+           when code in 200..299 <-
+             HTTPoison.get(
+               url,
+               headers,
+               @options
+             ),
+           {:is_html, _response_headers, true} <-
+             {:is_html, response_headers, is_html(response_headers)} do
+        body
+        |> parse_html()
+        |> maybe_parse()
+        |> Map.put(:url, url)
+        |> maybe_add_favicon()
+        |> clean_parsed_data()
+        |> check_parsed_data()
+        |> check_remote_picture_path()
+      else
+        {:is_html, response_headers, false} ->
+          data = get_data_for_media(response_headers, url)
+
+          {:ok, data}
+
+        {:error, err} ->
+          Logger.debug("HTTP error: #{inspect(err)}")
+          {:error, "HTTP error: #{inspect(err)}"}
+      end
+    rescue
+      e ->
+        {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
+    end
+  end
+
+  @spec get_data_for_media(List.t(), String.t()) :: map()
+  defp get_data_for_media(response_headers, url) do
+    data = %{title: get_filename_from_headers(response_headers) || get_filename_from_url(url)}
+
+    if is_image(response_headers) do
+      Map.put(data, :image_remote_url, url)
+    else
+      data
+    end
+  end
+
+  @spec is_html(List.t()) :: boolean
+  defp is_html(headers) do
+    headers
+    |> get_header("Content-Type")
+    |> content_type_header_matches(["text/html", "application/xhtml"])
+  end
+
+  @spec is_image(List.t()) :: boolean
+  defp is_image(headers) do
+    headers
+    |> get_header("Content-Type")
+    |> content_type_header_matches(["image/"])
+  end
+
+  @spec content_type_header_matches(String.t() | nil, List.t()) :: boolean
+  defp content_type_header_matches(header, content_types)
+  defp content_type_header_matches(nil, _content_types), do: false
+
+  defp content_type_header_matches(header, content_types) when is_binary(header) do
+    Enum.any?(content_types, fn content_type -> String.starts_with?(header, content_type) end)
+  end
+
+  @spec get_header(List.t(), String.t()) :: String.t() | nil
+  defp get_header(headers, key) do
+    case List.keyfind(headers, key, 0) do
+      {^key, value} -> String.downcase(value)
+      nil -> nil
+    end
+  end
+
+  @spec get_filename_from_headers(List.t()) :: String.t() | nil
+  defp get_filename_from_headers(headers) do
+    case get_header(headers, "Content-Disposition") do
+      nil -> nil
+      content_disposition -> parse_content_disposition(content_disposition)
+    end
+  end
+
+  @spec get_filename_from_url(String.t()) :: String.t()
+  defp get_filename_from_url(url) do
+    %URI{path: path} = URI.parse(url)
+
+    path
+    |> String.split("/", trim: true)
+    |> Enum.at(-1)
+    |> URI.decode()
+  end
+
+  # The following is taken from https://github.com/elixir-plug/plug/blob/65986ad32f9aaae3be50dc80cbdd19b326578da7/lib/plug/parsers/multipart.ex#L207
+  @spec parse_content_disposition(String.t()) :: String.t() | nil
+  defp parse_content_disposition(disposition) do
+    with [_, params] <- :binary.split(disposition, ";"),
+         %{"name" => _name} = params <- Utils.params(params) do
+      handle_disposition(params)
+    else
+      _ -> nil
+    end
+  end
+
+  @spec handle_disposition(map()) :: String.t() | nil
+  defp handle_disposition(params) do
+    case params do
+      %{"filename" => ""} ->
+        nil
+
+      %{"filename" => filename} ->
+        filename
+
+      %{"filename*" => ""} ->
+        nil
+
+      %{"filename*" => "utf-8''" <> filename} ->
+        URI.decode(filename)
+
+      _ ->
+        nil
+    end
+  end
+
+  defp parse_html(html), do: Floki.parse_document!(html)
+
+  defp maybe_parse(html) do
+    Enum.reduce_while(parsers(), %{}, fn parser, acc ->
+      case parser.parse(html, acc) do
+        {:ok, data} -> {:halt, data}
+        {:error, _msg} -> {:cont, acc}
+      end
+    end)
+  end
+
+  defp check_parsed_data(%{title: title} = data)
+       when is_binary(title) and byte_size(title) > 0 do
+    {:ok, data}
+  end
+
+  defp check_parsed_data(data) do
+    {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
+  end
+
+  defp clean_parsed_data(data) do
+    data
+    |> Enum.reject(fn {key, val} ->
+      case Jason.encode(%{key => val}) do
+        {:ok, _} -> false
+        _ -> true
+      end
+    end)
+    |> Map.new()
+  end
+
+  defp prevent_local_address(url) do
+    case URI.parse(url) do
+      %URI{host: host} when not is_nil(host) ->
+        host = String.downcase(host)
+
+        if validate_hostname_not_localhost(host) && validate_hostname_only(host) &&
+             validate_ip(host) do
+          {:ok, url}
+        else
+          {:error, "Host violates local access rules"}
+        end
+
+      _ ->
+        {:error, "Could not detect any host"}
+    end
+  end
+
+  defp validate_hostname_not_localhost(hostname),
+    do:
+      hostname != "localhost" && !String.ends_with?(hostname, ".local") &&
+        !String.ends_with?(hostname, ".localhost")
+
+  defp validate_hostname_only(hostname),
+    do: hostname |> String.graphemes() |> Enum.count(&(&1 == "o")) > 0
+
+  defp validate_ip(hostname) do
+    case hostname |> String.to_charlist() |> :inet.parse_address() do
+      {:ok, address} ->
+        !IpReserved.is_reserved?(address)
+
+      # Not a valid IP
+      {:error, _} ->
+        true
+    end
+  end
+
+  @spec maybe_add_favicon(map()) :: map()
+  defp maybe_add_favicon(%{url: url} = data) do
+    case Favicon.fetch(url) do
+      {:ok, favicon_url} ->
+        Logger.debug("Adding favicon #{favicon_url} to metadata")
+        Map.put(data, :favicon_url, favicon_url)
+
+      err ->
+        Logger.debug("Failed to add favicon to metadata")
+        Logger.debug(inspect(err))
+        data
+    end
+  end
+
+  @spec check_remote_picture_path(map()) :: map()
+  defp check_remote_picture_path(%{image_remote_url: image_remote_url, url: url} = data) do
+    Logger.debug("Checking image_remote_url #{image_remote_url}")
+    image_uri = URI.parse(image_remote_url)
+    uri = URI.parse(url)
+
+    image_remote_url =
+      cond do
+        is_nil(image_uri.host) -> "#{uri.scheme}://#{uri.host}#{image_remote_url}"
+        is_nil(image_uri.scheme) -> "#{uri.scheme}:#{image_remote_url}"
+        true -> image_remote_url
+      end
+
+    Map.put(data, :image_remote_url, image_remote_url)
+  end
+
+  defp check_remote_picture_path(data), do: data
+end

lib/service/rich_media/parsers/fallback.ex

@@ -0,0 +1,41 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parsers.Fallback do
+  @moduledoc """
+  Module to parse fallback data in HTML pages (plain old title and meta description)
+  """
+  @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
+  def parse(html, data) do
+    data =
+      data
+      |> maybe_put(html, :title)
+      |> maybe_put(html, :description)
+
+    if Enum.empty?(data) do
+      {:error, "Not even a title"}
+    else
+      {:ok, data}
+    end
+  end
+
+  defp maybe_put(meta, html, attr) do
+    case get_page(html, attr) do
+      "" -> meta
+      content -> Map.put_new(meta, attr, content)
+    end
+  end
+
+  defp get_page(html, :title) do
+    html |> Floki.find("html head title") |> List.first() |> Floki.text() |> String.trim()
+  end
+
+  defp get_page(html, :description) do
+    case html |> Floki.find("html head meta[name='description']") |> List.first() do
+      nil -> ""
+      elem -> elem |> Floki.attribute("content") |> List.first() |> String.trim()
+    end
+  end
+end

lib/service/rich_media/parsers/meta_tags_parser.ex

@@ -0,0 +1,76 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
+  @moduledoc """
+  Module to parse meta tags data in HTML pages
+  """
+  def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do
+    meta_data =
+      html
+      |> get_elements(key_name, prefix)
+      |> Enum.reduce(data, fn el, acc ->
+        attributes = normalize_attributes(el, prefix, key_name, value_name)
+
+        Map.merge(acc, attributes)
+      end)
+      |> maybe_put_title(html)
+      |> maybe_put_description(html)
+
+    if Enum.empty?(meta_data) do
+      {:error, error_message}
+    else
+      {:ok, meta_data}
+    end
+  end
+
+  defp get_elements(html, key_name, prefix) do
+    html |> Floki.find("meta[#{key_name}^='#{prefix}:']")
+  end
+
+  defp normalize_attributes(html_node, prefix, key_name, value_name) do
+    {_tag, attributes, _children} = html_node
+
+    data =
+      Enum.into(attributes, %{}, fn {name, value} ->
+        {name, String.trim_leading(value, "#{prefix}:")}
+      end)
+
+    %{String.to_atom(data[key_name]) => data[value_name]}
+  end
+
+  defp maybe_put_title(%{title: _} = meta, _), do: meta
+
+  defp maybe_put_title(meta, html) when meta != %{} do
+    case get_page_title(html) do
+      "" -> meta
+      title -> Map.put_new(meta, :title, title)
+    end
+  end
+
+  defp maybe_put_title(meta, _), do: meta
+
+  defp maybe_put_description(%{description: _} = meta, _), do: meta
+
+  defp maybe_put_description(meta, html) when meta != %{} do
+    case get_page_description(html) do
+      "" -> meta
+      description -> Map.put_new(meta, :description, description)
+    end
+  end
+
+  defp maybe_put_description(meta, _), do: meta
+
+  defp get_page_title(html) do
+    html |> Floki.find("html head title") |> List.first() |> Floki.text()
+  end
+
+  defp get_page_description(html) do
+    case html |> Floki.find("html head meta[name='description']") |> List.first() do
+      nil -> ""
+      elem -> Floki.attribute(elem, "content")
+    end
+  end
+end

lib/service/rich_media/parsers/oembed_parser.ex

@@ -0,0 +1,83 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parsers.OEmbed do
+  @moduledoc """
+  Module to parse OEmbed data in HTML pages
+  """
+  alias Mobilizon.Service.Formatter.HTML
+  require Logger
+
+  @http_options [
+    follow_redirect: true,
+    ssl: [{:versions, [:"tlsv1.2"]}]
+  ]
+
+  def parse(html, _data) do
+    Logger.debug("Using OEmbed parser")
+
+    with elements = [_ | _] <- get_discovery_data(html),
+         {:ok, oembed_url} <- get_oembed_url(elements),
+         {:ok, oembed_data} <- get_oembed_data(oembed_url),
+         oembed_data <- filter_oembed_data(oembed_data) do
+      Logger.debug("Data found with OEmbed parser")
+      Logger.debug(inspect(oembed_data))
+      {:ok, oembed_data}
+    else
+      _e ->
+        {:error, "No OEmbed data found"}
+    end
+  end
+
+  defp get_discovery_data(html) do
+    html |> Floki.find("link[type='application/json+oembed']")
+  end
+
+  defp get_oembed_url(nodes) do
+    {"link", attributes, _children} = nodes |> hd()
+
+    {:ok, Enum.into(attributes, %{})["href"]}
+  end
+
+  defp get_oembed_data(url) do
+    with {:ok, %HTTPoison.Response{body: json}} <- HTTPoison.get(url, [], @http_options),
+         {:ok, data} <- Jason.decode(json),
+         data <- data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end) do
+      {:ok, data}
+    end
+  end
+
+  defp filter_oembed_data(data) do
+    case Map.get(data, :type) do
+      nil ->
+        {:error, "No type declared for OEmbed data"}
+
+      "link" ->
+        Map.put(data, :image_remote_url, Map.get(data, :thumbnail_url))
+
+      "photo" ->
+        if Map.get(data, :url, "") == "" do
+          {:error, "No URL for photo OEmbed data"}
+        else
+          data
+          |> Map.put(:image_remote_url, Map.get(data, :url))
+          |> Map.put(:width, Map.get(data, :width, 0))
+          |> Map.put(:height, Map.get(data, :height, 0))
+        end
+
+      "video" ->
+        {:ok, html} = data |> Map.get(:html, "") |> HTML.filter_tags_for_oembed()
+
+        data
+        |> Map.put(:html, html)
+        |> Map.put(:width, Map.get(data, :width, 0))
+        |> Map.put(:height, Map.get(data, :height, 0))
+        |> Map.put(:image_remote_url, Map.get(data, :thumbnail_url))
+
+      "rich" ->
+        {:error, "OEmbed data has rich type, which we don't support"}
+    end
+  end
+end

lib/service/rich_media/parsers/ogp.ex

@@ -0,0 +1,36 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parsers.OGP do
+  @moduledoc """
+  Module to parse OpenGraph data in HTML pages
+  """
+  require Logger
+  alias Mobilizon.Service.RichMedia.Parsers.MetaTagsParser
+
+  def parse(html, data) do
+    Logger.debug("Using OpenGraph card parser")
+
+    with {:ok, data} <-
+           MetaTagsParser.parse(
+             html,
+             data,
+             "og",
+             "No OGP metadata found",
+             "property"
+           ) do
+      data = transform_tags(data)
+      Logger.debug("Data found with OpenGraph card parser")
+      {:ok, data}
+    end
+  end
+
+  defp transform_tags(data) do
+    data
+    |> Map.put(:image_remote_url, Map.get(data, :image))
+    |> Map.put(:width, Map.get(data, :"image:width"))
+    |> Map.put(:height, Map.get(data, :"image:height"))
+  end
+end

lib/service/rich_media/parsers/twitter_card.ex

@@ -0,0 +1,34 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Service.RichMedia.Parsers.TwitterCard do
+  @moduledoc """
+  Module to parse Twitter tags data in HTML pages
+  """
+  alias Mobilizon.Service.RichMedia.Parsers.MetaTagsParser
+  require Logger
+
+  @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
+  def parse(html, data) do
+    Logger.debug("Using Twitter card parser")
+
+    res =
+      data
+      |> parse_name_attrs(html)
+      |> parse_property_attrs(html)
+
+    Logger.debug("Data found with Twitter card parser")
+    Logger.debug(inspect(res))
+    res
+  end
+
+  defp parse_name_attrs(data, html) do
+    MetaTagsParser.parse(html, data, "twitter", %{}, "name")
+  end
+
+  defp parse_property_attrs({_, data}, html) do
+    MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property")
+  end
+end