hamlab/mobilizon-frontend
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Introduce group basic federation, event new page and notifications

Browse Source 
Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel
2020-02-18 08:57:00 +01:00
parent 300ef8f245
commit 4144e9ffd0
416 changed files with 32220 additions and 16750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
lib/service/formatter/default_scrubbler.ex
View File

@@ -8,19 +8,19 @@ defmodule Mobilizon.Service.Formatter.DefaultScrubbler do
Custom strategy to filter HTML content.
"""
alias HtmlSanitizeEx.Scrubber.Meta
require HtmlSanitizeEx.Scrubber.Meta
require FastSanitize.Sanitizer.Meta
alias FastSanitize.Sanitizer.Meta
# credo:disable-for-previous-line
# No idea how to fix this one…
Meta.remove_cdata_sections_before_scrub()
@valid_schemes ~w(https http)
Meta.strip_comments()
Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], ["https", "http"])
Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes)
Meta.allow_tag_with_this_attribute_values("a", "class", [
Meta.allow_tag_with_this_attribute_values(:a, "class", [
"hashtag",
"u-url",
"mention",
@@ -28,7 +28,7 @@ defmodule Mobilizon.Service.Formatter.DefaultScrubbler do
"mention u-url"
])
Meta.allow_tag_with_this_attribute_values("a", "rel", [
Meta.allow_tag_with_this_attribute_values(:a, "rel", [
"tag",
"nofollow",
"noopener",
@@ -36,34 +36,42 @@ defmodule Mobilizon.Service.Formatter.DefaultScrubbler do
"ugc"
])
Meta.allow_tag_with_these_attributes("a", ["name", "title"])
Meta.allow_tag_with_these_attributes(:a, ["name", "title"])
Meta.allow_tag_with_these_attributes("abbr", ["title"])
Meta.allow_tag_with_these_attributes(:abbr, ["title"])
Meta.allow_tag_with_these_attributes("b", [])
Meta.allow_tag_with_these_attributes("blockquote", [])
Meta.allow_tag_with_these_attributes("br", [])
Meta.allow_tag_with_these_attributes("code", [])
Meta.allow_tag_with_these_attributes("del", [])
Meta.allow_tag_with_these_attributes("em", [])
Meta.allow_tag_with_these_attributes("i", [])
Meta.allow_tag_with_these_attributes("li", [])
Meta.allow_tag_with_these_attributes("ol", [])
Meta.allow_tag_with_these_attributes("p", [])
Meta.allow_tag_with_these_attributes("pre", [])
Meta.allow_tag_with_these_attributes("strong", [])
Meta.allow_tag_with_these_attributes("u", [])
Meta.allow_tag_with_these_attributes("ul", [])
Meta.allow_tag_with_these_attributes("img", ["src", "alt"])
Meta.allow_tag_with_these_attributes(:b, [])
Meta.allow_tag_with_these_attributes(:blockquote, [])
Meta.allow_tag_with_these_attributes(:br, [])
Meta.allow_tag_with_these_attributes(:code, [])
Meta.allow_tag_with_these_attributes(:del, [])
Meta.allow_tag_with_these_attributes(:em, [])
Meta.allow_tag_with_these_attributes(:i, [])
Meta.allow_tag_with_these_attributes(:li, [])
Meta.allow_tag_with_these_attributes(:ol, [])
Meta.allow_tag_with_these_attributes(:p, [])
Meta.allow_tag_with_these_attributes(:pre, [])
Meta.allow_tag_with_these_attributes(:strong, [])
Meta.allow_tag_with_these_attributes(:u, [])
Meta.allow_tag_with_these_attributes(:ul, [])
Meta.allow_tag_with_uri_attributes(:img, ["src"], @valid_schemes)
Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card", "mention"])
Meta.allow_tag_with_these_attributes("span", ["data-user"])
Meta.allow_tag_with_these_attributes(:img, [
"width",
"height",
"class",
"title",
"alt"
])
Meta.allow_tag_with_these_attributes("h1", [])
Meta.allow_tag_with_these_attributes("h2", [])
Meta.allow_tag_with_these_attributes("h3", [])
Meta.allow_tag_with_these_attributes("h4", [])
Meta.allow_tag_with_these_attributes("h5", [])
Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "mention"])
Meta.allow_tag_with_these_attributes(:span, ["data-user"])
Meta.allow_tag_with_these_attributes(:h1, [])
Meta.allow_tag_with_these_attributes(:h2, [])
Meta.allow_tag_with_these_attributes(:h3, [])
Meta.allow_tag_with_these_attributes(:h4, [])
Meta.allow_tag_with_these_attributes(:h5, [])
Meta.strip_everything_not_covered()
end

4
lib/service/formatter/formatter.ex
View File

@@ -95,7 +95,9 @@ defmodule Mobilizon.Service.Formatter do
end
def html_escape(text, "text/html") do
HTML.filter_tags(text)
with {:ok, content} <- HTML.filter_tags(text) do
content
end
end
def html_escape(text, "text/plain") do

8
lib/service/formatter/html.ex
View File

@@ -8,9 +8,11 @@ defmodule Mobilizon.Service.Formatter.HTML do
Service to filter tags out of HTML content.
"""
alias HtmlSanitizeEx.Scrubber
alias FastSanitize.Sanitizer
alias Mobilizon.Service.Formatter.DefaultScrubbler
alias Mobilizon.Service.Formatter.{DefaultScrubbler, OEmbed}
def filter_tags(html), do: Scrubber.scrub(html, DefaultScrubbler)
def filter_tags(html), do: Sanitizer.scrub(html, DefaultScrubbler)
def filter_tags_for_oembed(html), do: Sanitizer.scrub(html, OEmbed)
end

34
lib/service/formatter/oembed.ex Normal file
View File

@@ -0,0 +1,34 @@
defmodule Mobilizon.Service.Formatter.OEmbed do
@moduledoc """
Custom strategy to filter HTML content in OEmbed html
"""
require FastSanitize.Sanitizer.Meta
alias FastSanitize.Sanitizer.Meta
@valid_schemes ~w(https http)
Meta.strip_comments()
Meta.allow_tag_with_uri_attributes(:a, ["href"], @valid_schemes)
Meta.allow_tag_with_uri_attributes(:img, ["src"], @valid_schemes)
Meta.allow_tag_with_these_attributes(:audio, ["controls"])
Meta.allow_tag_with_uri_attributes(:embed, ["src"], @valid_schemes)
Meta.allow_tag_with_these_attributes(:embed, ["height type width"])
Meta.allow_tag_with_uri_attributes(:iframe, ["src"], @valid_schemes)
Meta.allow_tag_with_these_attributes(
:iframe,
["allowfullscreen frameborder allow height scrolling width"]
)
Meta.allow_tag_with_uri_attributes(:source, ["src"], @valid_schemes)
Meta.allow_tag_with_these_attributes(:source, ["type"])
Meta.allow_tag_with_these_attributes(:video, ["controls height loop width"])
Meta.strip_everything_not_covered()
end

9
lib/service/geospatial/addok.ex
View File

@@ -13,6 +13,11 @@ defmodule Mobilizon.Service.Geospatial.Addok do
@endpoint Application.get_env(:mobilizon, __MODULE__) |> get_in([:endpoint])
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Addok implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -26,7 +31,7 @@ defmodule Mobilizon.Service.Geospatial.Addok do
Logger.debug("Asking addok for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else
@@ -46,7 +51,7 @@ defmodule Mobilizon.Service.Geospatial.Addok do
Logger.debug("Asking addok for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else

11
lib/service/geospatial/google_maps.ex
View File

@@ -28,6 +28,11 @@ defmodule Mobilizon.Service.Geospatial.GoogleMaps do
@api_key_missing_message "API Key required to use Google Maps"
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Google Maps implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -39,7 +44,7 @@ defmodule Mobilizon.Service.Geospatial.GoogleMaps do
Logger.debug("Asking Google Maps for reverse geocode with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url),
HTTPoison.get(url, [], @http_options),
{:ok, %{"results" => results, "status" => "OK"}} <- Poison.decode(body) do
Enum.map(results, fn entry -> process_data(entry, options) end)
else
@@ -59,7 +64,7 @@ defmodule Mobilizon.Service.Geospatial.GoogleMaps do
Logger.debug("Asking Google Maps for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url),
HTTPoison.get(url, [], @http_options),
{:ok, %{"results" => results, "status" => "OK"}} <- Poison.decode(body) do
results |> Enum.map(fn entry -> process_data(entry, options) end)
else
@@ -161,7 +166,7 @@ defmodule Mobilizon.Service.Geospatial.GoogleMaps do
Logger.debug("Asking Google Maps for details with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url),
HTTPoison.get(url, [], @http_options),
{:ok, %{"result" => %{"name" => name}, "status" => "OK"}} <- Poison.decode(body) do
name
else

10
lib/service/geospatial/map_quest.ex
View File

@@ -21,6 +21,11 @@ defmodule Mobilizon.Service.Geospatial.MapQuest do
@api_key_missing_message "API Key required to use MapQuest"
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
MapQuest implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -42,7 +47,8 @@ defmodule Mobilizon.Service.Geospatial.MapQuest do
"https://#{prefix}.mapquestapi.com/geocoding/v1/reverse?key=#{api_key}&location=#{
lat
},#{lon}&maxResults=#{limit}",
headers
headers,
@http_options
),
{:ok, %{"results" => results, "info" => %{"statuscode" => 0}}} <- Poison.decode(body) do
results |> Enum.map(&process_data/1)
@@ -77,7 +83,7 @@ defmodule Mobilizon.Service.Geospatial.MapQuest do
Logger.debug("Asking MapQuest for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"results" => results, "info" => %{"statuscode" => 0}}} <- Poison.decode(body) do
results |> Enum.map(&process_data/1)
else

9
lib/service/geospatial/mimirsbrunn.ex
View File

@@ -17,6 +17,11 @@ defmodule Mobilizon.Service.Geospatial.Mimirsbrunn do
@endpoint Application.get_env(:mobilizon, __MODULE__) |> get_in([:endpoint])
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Mimirsbrunn implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -29,7 +34,7 @@ defmodule Mobilizon.Service.Geospatial.Mimirsbrunn do
Logger.debug("Asking Mimirsbrunn for reverse geocoding with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else
@@ -49,7 +54,7 @@ defmodule Mobilizon.Service.Geospatial.Mimirsbrunn do
Logger.debug("Asking Mimirsbrunn for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else

9
lib/service/geospatial/nominatim.ex
View File

@@ -14,6 +14,11 @@ defmodule Mobilizon.Service.Geospatial.Nominatim do
@endpoint Application.get_env(:mobilizon, __MODULE__) |> get_in([:endpoint])
@api_key Application.get_env(:mobilizon, __MODULE__) |> get_in([:api_key])
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Nominatim implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -26,7 +31,7 @@ defmodule Mobilizon.Service.Geospatial.Nominatim do
Logger.debug("Asking Nominatim for geocode with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
features |> process_data() |> Enum.filter(& &1)
else
@@ -46,7 +51,7 @@ defmodule Mobilizon.Service.Geospatial.Nominatim do
Logger.debug("Asking Nominatim for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
features |> process_data() |> Enum.filter(& &1)
else

9
lib/service/geospatial/pelias.ex
View File

@@ -15,6 +15,11 @@ defmodule Mobilizon.Service.Geospatial.Pelias do
@endpoint Application.get_env(:mobilizon, __MODULE__) |> get_in([:endpoint])
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Pelias implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -27,7 +32,7 @@ defmodule Mobilizon.Service.Geospatial.Pelias do
Logger.debug("Asking Pelias for reverse geocoding with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else
@@ -47,7 +52,7 @@ defmodule Mobilizon.Service.Geospatial.Pelias do
Logger.debug("Asking Pelias for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else

9
lib/service/geospatial/photon.ex
View File

@@ -13,6 +13,11 @@ defmodule Mobilizon.Service.Geospatial.Photon do
@endpoint Application.get_env(:mobilizon, __MODULE__) |> get_in([:endpoint])
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@impl Provider
@doc """
Photon implementation for `c:Mobilizon.Service.Geospatial.Provider.geocode/3`.
@@ -27,7 +32,7 @@ defmodule Mobilizon.Service.Geospatial.Photon do
Logger.debug("Asking photon for reverse geocoding with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else
@@ -47,7 +52,7 @@ defmodule Mobilizon.Service.Geospatial.Photon do
Logger.debug("Asking photon for addresses with #{url}")
with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
HTTPoison.get(url, headers),
HTTPoison.get(url, headers, @http_options),
{:ok, %{"features" => features}} <- Poison.decode(body) do
process_data(features)
else

4
lib/service/metadata/comment.ex
View File

@@ -1,6 +1,6 @@
defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Comment do
defimpl Mobilizon.Service.Metadata, for: Mobilizon.Conversations.Comment do
alias Phoenix.HTML.Tag
alias Mobilizon.Events.Comment
alias Mobilizon.Conversations.Comment
def build_tags(%Comment{} = comment) do
[

82
lib/service/notifications/scheduler.ex Normal file
View File

@@ -0,0 +1,82 @@
defmodule Mobilizon.Service.Notifications.Scheduler do
@moduledoc """
Allows to insert jobs
"""
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.Service.Workers.Notification
alias Mobilizon.Users
alias Mobilizon.Users.Setting
require Logger
def before_event_notification(%Participant{
id: participant_id,
event: %Event{begins_on: begins_on},
actor: %Actor{user_id: user_id}
})
when not is_nil(user_id) do
case Users.get_setting(user_id) do
%Setting{notification_before_event: true} ->
Notification.enqueue(:before_event_notification, %{participant_id: participant_id},
scheduled_at: DateTime.add(begins_on, -3600, :second)
)
_ ->
{:ok, nil}
end
end
def before_event_notification(_), do: {:ok, nil}
def on_day_notification(%Participant{
event: %Event{begins_on: begins_on},
actor: %Actor{user_id: user_id}
})
when not is_nil(user_id) do
case Users.get_setting(user_id) do
%Setting{notification_on_day: true, timezone: timezone} ->
%DateTime{hour: hour} = begins_on_shifted = shift_zone(begins_on, timezone)
Logger.debug("Participation event start at #{inspect(begins_on_shifted)} (user timezone)")
send_date =
cond do
begins_on < DateTime.utc_now() ->
nil
hour > 8 ->
# If the event is after 8 o'clock
%{begins_on_shifted | hour: 8, minute: 0, second: 0, microsecond: {0, 0}}
true ->
# If the event is before 8 o'clock, we send the notification the day before,
# unless this is already passed
begins_on_shifted
|> DateTime.add(-24 * 3_600)
|> (&%{&1 | hour: 8, minute: 0, second: 0, microsecond: {0, 0}}).()
end
Logger.debug(
"Participation notification should be sent at #{inspect(send_date)} (user timezone)"
)
if DateTime.utc_now() > send_date do
{:ok, "Too late to send same day notifications"}
else
Notification.enqueue(:on_day_notification, %{user_id: user_id}, scheduled_at: send_date)
end
_ ->
{:ok, "User has disable on day notifications"}
end
end
def on_day_notification(_), do: {:ok, nil}
defp shift_zone(datetime, timezone) do
case DateTime.shift_zone(datetime, timezone) do
{:ok, shift_datetime} -> shift_datetime
{:error, _} -> datetime
end
end
end

104
lib/service/rich_media/favicon.ex Normal file
View File

@@ -0,0 +1,104 @@
defmodule Mobilizon.Service.RichMedia.Favicon do
@moduledoc """
Module to fetch favicon information from a website
Taken and adapted from https://github.com/ricn/favicon
"""
require Logger
alias Mobilizon.Config
@options [
max_body: 2_000_000,
timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
@spec fetch(String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
def fetch(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())
headers = [{"User-Agent", user_agent}]
case HTTPoison.get(url, headers, @options) do
{:ok, %HTTPoison.Response{status_code: code, body: body}} when code in 200..299 ->
find_favicon_url(url, body, headers)
{:ok, %HTTPoison.Response{}} ->
{:error, "Error while fetching the page"}
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
end
end
@spec find_favicon_url(String.t(), String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
defp find_favicon_url(url, body, headers) do
Logger.debug("finding favicon URL for #{url}")
case find_favicon_link_tag(body) do
{:ok, tag} ->
Logger.debug("Found link #{inspect(tag)}")
{"link", attrs, _} = tag
{"href", path} =
Enum.find(attrs, fn {name, _} ->
name == "href"
end)
{:ok, format_url(url, path)}
_ ->
find_favicon_in_root(url, headers)
end
end
@spec format_url(String.t(), String.t()) :: String.t()
defp format_url(url, path) do
image_uri = URI.parse(path)
uri = URI.parse(url)
cond do
is_nil(image_uri.host) -> "#{uri.scheme}://#{uri.host}#{path}"
is_nil(image_uri.scheme) -> "#{uri.scheme}:#{path}"
true -> path
end
end
@spec find_favicon_link_tag(String.t()) :: {:ok, tuple()} | {:error, any()}
defp find_favicon_link_tag(html) do
with {:ok, html} <- Floki.parse_document(html),
links <- Floki.find(html, "link"),
{:link, link} when not is_nil(link) <-
{:link,
Enum.find(links, fn {"link", attrs, _} ->
Enum.any?(attrs, fn {name, value} ->
name == "rel" && String.contains?(value, "icon") &&
!String.contains?(value, "-icon-")
end)
end)} do
{:ok, link}
else
{:link, nil} -> {:error, "No link found"}
err -> err
end
end
@spec find_favicon_in_root(String.t(), List.t()) :: {:ok, String.t()} | {:error, any()}
defp find_favicon_in_root(url, headers) do
uri = URI.parse(url)
favicon_url = "#{uri.scheme}://#{uri.host}/favicon.ico"
case HTTPoison.head(favicon_url, headers, @options) do
{:ok, %HTTPoison.Response{status_code: code}} when code in 200..299 ->
{:ok, favicon_url}
{:ok, %HTTPoison.Response{}} ->
{:error, "Error while doing a HEAD request on the favicon"}
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
end
end
end

278
lib/service/rich_media/parser.ex Normal file
View File

@@ -0,0 +1,278 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parser do
@moduledoc """
Module to parse data in HTML pages
"""
@options [
max_body: 2_000_000,
timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
# TODO: Remove me once Hackney/HTTPoison fixes their shit with TLS1.3 and OTP 23
ssl: [{:versions, [:"tlsv1.2"]}]
]
alias Mobilizon.Config
alias Mobilizon.Service.RichMedia.Favicon
alias Plug.Conn.Utils
require Logger
defp parsers do
Mobilizon.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
case Cachex.fetch(:rich_media_cache, url, fn _ ->
case parse_url(url) do
{:ok, data} -> {:commit, data}
{:error, err} -> {:ignore, err}
end
end) do
{status, value} when status in [:ok, :commit] ->
{:ok, value}
{_, err} ->
{:error, err}
end
rescue
e ->
{:error, "Cachex error: #{inspect(e)}"}
end
@spec parse_url(String.t(), List.t()) :: {:ok, map()} | {:error, any()}
defp parse_url(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())
headers = [{"User-Agent", user_agent}]
Logger.debug("Fetching content at address #{inspect(url)}")
try do
with {:ok, _} <- prevent_local_address(url),
{:ok, %HTTPoison.Response{body: body, status_code: code, headers: response_headers}}
when code in 200..299 <-
HTTPoison.get(
url,
headers,
@options
),
{:is_html, _response_headers, true} <-
{:is_html, response_headers, is_html(response_headers)} do
body
|> parse_html()
|> maybe_parse()
|> Map.put(:url, url)
|> maybe_add_favicon()
|> clean_parsed_data()
|> check_parsed_data()
|> check_remote_picture_path()
else
{:is_html, response_headers, false} ->
data = get_data_for_media(response_headers, url)
{:ok, data}
{:error, err} ->
Logger.debug("HTTP error: #{inspect(err)}")
{:error, "HTTP error: #{inspect(err)}"}
end
rescue
e ->
{:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
end
end
@spec get_data_for_media(List.t(), String.t()) :: map()
defp get_data_for_media(response_headers, url) do
data = %{title: get_filename_from_headers(response_headers) || get_filename_from_url(url)}
if is_image(response_headers) do
Map.put(data, :image_remote_url, url)
else
data
end
end
@spec is_html(List.t()) :: boolean
defp is_html(headers) do
headers
|> get_header("Content-Type")
|> content_type_header_matches(["text/html", "application/xhtml"])
end
@spec is_image(List.t()) :: boolean
defp is_image(headers) do
headers
|> get_header("Content-Type")
|> content_type_header_matches(["image/"])
end
@spec content_type_header_matches(String.t() | nil, List.t()) :: boolean
defp content_type_header_matches(header, content_types)
defp content_type_header_matches(nil, _content_types), do: false
defp content_type_header_matches(header, content_types) when is_binary(header) do
Enum.any?(content_types, fn content_type -> String.starts_with?(header, content_type) end)
end
@spec get_header(List.t(), String.t()) :: String.t() | nil
defp get_header(headers, key) do
case List.keyfind(headers, key, 0) do
{^key, value} -> String.downcase(value)
nil -> nil
end
end
@spec get_filename_from_headers(List.t()) :: String.t() | nil
defp get_filename_from_headers(headers) do
case get_header(headers, "Content-Disposition") do
nil -> nil
content_disposition -> parse_content_disposition(content_disposition)
end
end
@spec get_filename_from_url(String.t()) :: String.t()
defp get_filename_from_url(url) do
%URI{path: path} = URI.parse(url)
path
|> String.split("/", trim: true)
|> Enum.at(-1)
|> URI.decode()
end
# The following is taken from https://github.com/elixir-plug/plug/blob/65986ad32f9aaae3be50dc80cbdd19b326578da7/lib/plug/parsers/multipart.ex#L207
@spec parse_content_disposition(String.t()) :: String.t() | nil
defp parse_content_disposition(disposition) do
with [_, params] <- :binary.split(disposition, ";"),
%{"name" => _name} = params <- Utils.params(params) do
handle_disposition(params)
else
_ -> nil
end
end
@spec handle_disposition(map()) :: String.t() | nil
defp handle_disposition(params) do
case params do
%{"filename" => ""} ->
nil
%{"filename" => filename} ->
filename
%{"filename*" => ""} ->
nil
%{"filename*" => "utf-8''" <> filename} ->
URI.decode(filename)
_ ->
nil
end
end
defp parse_html(html), do: Floki.parse_document!(html)
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do
{:ok, data} -> {:halt, data}
{:error, _msg} -> {:cont, acc}
end
end)
end
defp check_parsed_data(%{title: title} = data)
when is_binary(title) and byte_size(title) > 0 do
{:ok, data}
end
defp check_parsed_data(data) do
{:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
end
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
case Jason.encode(%{key => val}) do
{:ok, _} -> false
_ -> true
end
end)
|> Map.new()
end
defp prevent_local_address(url) do
case URI.parse(url) do
%URI{host: host} when not is_nil(host) ->
host = String.downcase(host)
if validate_hostname_not_localhost(host) && validate_hostname_only(host) &&
validate_ip(host) do
{:ok, url}
else
{:error, "Host violates local access rules"}
end
_ ->
{:error, "Could not detect any host"}
end
end
defp validate_hostname_not_localhost(hostname),
do:
hostname != "localhost" && !String.ends_with?(hostname, ".local") &&
!String.ends_with?(hostname, ".localhost")
defp validate_hostname_only(hostname),
do: hostname |> String.graphemes() |> Enum.count(&(&1 == "o")) > 0
defp validate_ip(hostname) do
case hostname |> String.to_charlist() |> :inet.parse_address() do
{:ok, address} ->
!IpReserved.is_reserved?(address)
# Not a valid IP
{:error, _} ->
true
end
end
@spec maybe_add_favicon(map()) :: map()
defp maybe_add_favicon(%{url: url} = data) do
case Favicon.fetch(url) do
{:ok, favicon_url} ->
Logger.debug("Adding favicon #{favicon_url} to metadata")
Map.put(data, :favicon_url, favicon_url)
err ->
Logger.debug("Failed to add favicon to metadata")
Logger.debug(inspect(err))
data
end
end
@spec check_remote_picture_path(map()) :: map()
defp check_remote_picture_path(%{image_remote_url: image_remote_url, url: url} = data) do
Logger.debug("Checking image_remote_url #{image_remote_url}")
image_uri = URI.parse(image_remote_url)
uri = URI.parse(url)
image_remote_url =
cond do
is_nil(image_uri.host) -> "#{uri.scheme}://#{uri.host}#{image_remote_url}"
is_nil(image_uri.scheme) -> "#{uri.scheme}:#{image_remote_url}"
true -> image_remote_url
end
Map.put(data, :image_remote_url, image_remote_url)
end
defp check_remote_picture_path(data), do: data
end

41
lib/service/rich_media/parsers/fallback.ex Normal file
View File

@@ -0,0 +1,41 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parsers.Fallback do
@moduledoc """
Module to parse fallback data in HTML pages (plain old title and meta description)
"""
@spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
def parse(html, data) do
data =
data
|> maybe_put(html, :title)
|> maybe_put(html, :description)
if Enum.empty?(data) do
{:error, "Not even a title"}
else
{:ok, data}
end
end
defp maybe_put(meta, html, attr) do
case get_page(html, attr) do
"" -> meta
content -> Map.put_new(meta, attr, content)
end
end
defp get_page(html, :title) do
html |> Floki.find("html head title") |> List.first() |> Floki.text() |> String.trim()
end
defp get_page(html, :description) do
case html |> Floki.find("html head meta[name='description']") |> List.first() do
nil -> ""
elem -> elem |> Floki.attribute("content") |> List.first() |> String.trim()
end
end
end

76
lib/service/rich_media/parsers/meta_tags_parser.ex Normal file
View File

@@ -0,0 +1,76 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parsers.MetaTagsParser do
@moduledoc """
Module to parse meta tags data in HTML pages
"""
def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do
meta_data =
html
|> get_elements(key_name, prefix)
|> Enum.reduce(data, fn el, acc ->
attributes = normalize_attributes(el, prefix, key_name, value_name)
Map.merge(acc, attributes)
end)
|> maybe_put_title(html)
|> maybe_put_description(html)
if Enum.empty?(meta_data) do
{:error, error_message}
else
{:ok, meta_data}
end
end
defp get_elements(html, key_name, prefix) do
html |> Floki.find("meta[#{key_name}^='#{prefix}:']")
end
defp normalize_attributes(html_node, prefix, key_name, value_name) do
{_tag, attributes, _children} = html_node
data =
Enum.into(attributes, %{}, fn {name, value} ->
{name, String.trim_leading(value, "#{prefix}:")}
end)
%{String.to_atom(data[key_name]) => data[value_name]}
end
defp maybe_put_title(%{title: _} = meta, _), do: meta
defp maybe_put_title(meta, html) when meta != %{} do
case get_page_title(html) do
"" -> meta
title -> Map.put_new(meta, :title, title)
end
end
defp maybe_put_title(meta, _), do: meta
defp maybe_put_description(%{description: _} = meta, _), do: meta
defp maybe_put_description(meta, html) when meta != %{} do
case get_page_description(html) do
"" -> meta
description -> Map.put_new(meta, :description, description)
end
end
defp maybe_put_description(meta, _), do: meta
defp get_page_title(html) do
html |> Floki.find("html head title") |> List.first() |> Floki.text()
end
defp get_page_description(html) do
case html |> Floki.find("html head meta[name='description']") |> List.first() do
nil -> ""
elem -> Floki.attribute(elem, "content")
end
end
end

83
lib/service/rich_media/parsers/oembed_parser.ex Normal file
View File

@@ -0,0 +1,83 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parsers.OEmbed do
@moduledoc """
Module to parse OEmbed data in HTML pages
"""
alias Mobilizon.Service.Formatter.HTML
require Logger
@http_options [
follow_redirect: true,
ssl: [{:versions, [:"tlsv1.2"]}]
]
def parse(html, _data) do
Logger.debug("Using OEmbed parser")
with elements = [_ | _] <- get_discovery_data(html),
{:ok, oembed_url} <- get_oembed_url(elements),
{:ok, oembed_data} <- get_oembed_data(oembed_url),
oembed_data <- filter_oembed_data(oembed_data) do
Logger.debug("Data found with OEmbed parser")
Logger.debug(inspect(oembed_data))
{:ok, oembed_data}
else
_e ->
{:error, "No OEmbed data found"}
end
end
defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
end
defp get_oembed_url(nodes) do
{"link", attributes, _children} = nodes |> hd()
{:ok, Enum.into(attributes, %{})["href"]}
end
defp get_oembed_data(url) do
with {:ok, %HTTPoison.Response{body: json}} <- HTTPoison.get(url, [], @http_options),
{:ok, data} <- Jason.decode(json),
data <- data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end) do
{:ok, data}
end
end
defp filter_oembed_data(data) do
case Map.get(data, :type) do
nil ->
{:error, "No type declared for OEmbed data"}
"link" ->
Map.put(data, :image_remote_url, Map.get(data, :thumbnail_url))
"photo" ->
if Map.get(data, :url, "") == "" do
{:error, "No URL for photo OEmbed data"}
else
data
|> Map.put(:image_remote_url, Map.get(data, :url))
|> Map.put(:width, Map.get(data, :width, 0))
|> Map.put(:height, Map.get(data, :height, 0))
end
"video" ->
{:ok, html} = data |> Map.get(:html, "") |> HTML.filter_tags_for_oembed()
data
|> Map.put(:html, html)
|> Map.put(:width, Map.get(data, :width, 0))
|> Map.put(:height, Map.get(data, :height, 0))
|> Map.put(:image_remote_url, Map.get(data, :thumbnail_url))
"rich" ->
{:error, "OEmbed data has rich type, which we don't support"}
end
end
end

36
lib/service/rich_media/parsers/ogp.ex Normal file
View File

@@ -0,0 +1,36 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parsers.OGP do
@moduledoc """
Module to parse OpenGraph data in HTML pages
"""
require Logger
alias Mobilizon.Service.RichMedia.Parsers.MetaTagsParser
def parse(html, data) do
Logger.debug("Using OpenGraph card parser")
with {:ok, data} <-
MetaTagsParser.parse(
html,
data,
"og",
"No OGP metadata found",
"property"
) do
data = transform_tags(data)
Logger.debug("Data found with OpenGraph card parser")
{:ok, data}
end
end
defp transform_tags(data) do
data
|> Map.put(:image_remote_url, Map.get(data, :image))
|> Map.put(:width, Map.get(data, :"image:width"))
|> Map.put(:height, Map.get(data, :"image:height"))
end
end

34
lib/service/rich_media/parsers/twitter_card.ex Normal file
View File

@@ -0,0 +1,34 @@
# Portions of this file are derived from Pleroma:
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mobilizon.Service.RichMedia.Parsers.TwitterCard do
@moduledoc """
Module to parse Twitter tags data in HTML pages
"""
alias Mobilizon.Service.RichMedia.Parsers.MetaTagsParser
require Logger
@spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
def parse(html, data) do
Logger.debug("Using Twitter card parser")
res =
data
|> parse_name_attrs(html)
|> parse_property_attrs(html)
Logger.debug("Data found with Twitter card parser")
Logger.debug(inspect(res))
res
end
defp parse_name_attrs(data, html) do
MetaTagsParser.parse(html, data, "twitter", %{}, "name")
end
defp parse_property_attrs({_, data}, html) do
MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property")
end
end

4
lib/service/statistics/statistics.ex
View File

@@ -3,7 +3,7 @@ defmodule Mobilizon.Service.Statistics do
A module that provides cached statistics
"""
alias Mobilizon.{Events, Users}
alias Mobilizon.{Conversations, Events, Users}
def get_cached_value(key) do
case Cachex.fetch(:statistics, key, fn key ->
@@ -26,6 +26,6 @@ defmodule Mobilizon.Service.Statistics do
end
defp create_cache(:local_comments) do
Events.count_local_comments()
Conversations.count_local_comments()
end
end

2
lib/service/workers/background.ex
View File

@@ -1,6 +1,6 @@
defmodule Mobilizon.Service.Workers.Background do
@moduledoc """
Worker to build search results
Worker to perform various actions in the background
"""
alias Mobilizon.Actors

3
lib/service/workers/helper.ex
View File

@@ -10,7 +10,6 @@ defmodule Mobilizon.Service.Workers.Helper do
alias Mobilizon.Config
alias Mobilizon.Service.Workers.Helper
alias Mobilizon.Storage.Repo
def worker_args(queue) do
case Config.get([:workers, :retries, queue]) do
@@ -45,7 +44,7 @@ defmodule Mobilizon.Service.Workers.Helper do
unquote(caller_module)
|> apply(:new, [params, worker_args])
|> Repo.insert()
|> Oban.insert()
end
end
end

63
lib/service/workers/notification.ex Normal file
View File

@@ -0,0 +1,63 @@
defmodule Mobilizon.Service.Workers.Notification do
@moduledoc """
Worker to send notifications
"""
alias Mobilizon.Actors.Actor
alias Mobilizon.Events
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.Storage.Page
alias Mobilizon.Users
alias Mobilizon.Users.{Setting, User}
alias Mobilizon.Web.Email.{Mailer, Notification}
use Mobilizon.Service.Workers.Helper, queue: "mailers"
@impl Oban.Worker
def perform(%{"op" => "before_event_notification", "participant_id" => participant_id}, _job) do
with %Participant{actor: %Actor{user_id: user_id}, event: %Event{status: :confirmed}} =
participant <- Events.get_participant(participant_id),
%User{email: email, locale: locale, settings: %Setting{notification_before_event: true}} <-
Users.get_user_with_settings!(user_id) do
email
|> Notification.before_event_notification(participant, locale)
|> Mailer.deliver_later()
:ok
end
end
def perform(%{"op" => "on_day_notification", "user_id" => user_id}, _job) do
%User{locale: locale, settings: %Setting{timezone: timezone, notification_on_day: true}} =
user = Users.get_user_with_settings!(user_id)
now = DateTime.utc_now()
%DateTime{} = now_shifted = shift_zone(now, timezone)
start = %{now_shifted | hour: 8, minute: 0, second: 0, microsecond: {0, 0}}
tomorrow = DateTime.add(start, 3600 * 24)
with %Page{
elements: participations,
total: total
} <-
Events.list_participations_for_user(user_id, start, tomorrow, 1, 5),
true <-
Enum.all?(participations, fn participation ->
participation.event.status == :confirmed
end),
true <- total > 0 do
user
|> Notification.on_day_notification(participations, total, locale)
|> Mailer.deliver_later()
:ok
end
end
defp shift_zone(datetime, timezone) do
case DateTime.shift_zone(datetime, timezone) do
{:ok, shift_datetime} -> shift_datetime
{:error, _} -> datetime
end
end
end