Save remote profiles avatars & banners locally

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel
2020-12-15 17:17:42 +01:00
parent ae03f84950
commit 9b27e70eb0
41 changed files with 1424 additions and 716 deletions

View File

@@ -766,7 +766,10 @@ defmodule Mobilizon.Federation.ActivityPub do
res =
with {:ok, %{status: 200, body: body}} <-
Tesla.get(url, headers: [{"Accept", "application/activity+json"}]),
Tesla.get(url,
headers: [{"Accept", "application/activity+json"}],
follow_redirect: true
),
:ok <- Logger.debug("response okay, now decoding json"),
{:ok, data} <- Jason.decode(body) do
Logger.debug("Got activity+json response at actor's endpoint, now converting data")

View File

@@ -382,7 +382,7 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier do
{:ok, activity, new_actor}
else
e ->
Logger.debug(inspect(e))
Logger.error(inspect(e))
:error
end
end

View File

@@ -11,7 +11,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
alias Mobilizon.Federation.ActivityPub.Utils
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Service.HTTP.RemoteMediaDownloaderClient
alias Mobilizon.Service.RichMedia.Parser
alias Mobilizon.Web.Upload
@behaviour Converter
@@ -30,18 +32,10 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
@spec as_to_model_data(map()) :: {:ok, map()}
def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do
avatar =
data["icon"]["url"] &&
%{
"name" => data["icon"]["name"] || "avatar",
"url" => MediaProxy.url(data["icon"]["url"])
}
download_picture(get_in(data, ["icon", "url"]), get_in(data, ["icon", "name"]), "avatar")
banner =
data["image"]["url"] &&
%{
"name" => data["image"]["name"] || "banner",
"url" => MediaProxy.url(data["image"]["url"])
}
download_picture(get_in(data, ["image", "url"]), get_in(data, ["image", "name"]), "banner")
%{
url: data["id"],
@@ -140,4 +134,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
})
end
end
@spec download_picture(String.t() | nil, String.t(), String.t()) :: map()
defp download_picture(nil, _name, _default_name), do: nil
defp download_picture(url, name, default_name) do
with {:ok, %{body: body, status: code, headers: response_headers}}
when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
{:ok, file} <- Upload.store(%{body: body, name: name}) do
file
end
end
end

View File

@@ -10,7 +10,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
alias Mobilizon.Users.User
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Federation.ActivityPub.Activity
import Mobilizon.Web.Gettext
@@ -35,7 +34,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
) do
case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do
{:has_event, %Event{} = event} ->
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
{:has_event, _} ->
{:error, :event_not_found}
@@ -51,7 +50,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
{:has_event, Events.get_public_event_by_uuid_with_preload(uuid)},
{:access_valid, true} <-
{:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
else
{:has_event, _} ->
find_private_event(parent, args, resolution)

View File

@@ -8,7 +8,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
alias Mobilizon.Actors.{Actor, Member}
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Upload
import Mobilizon.Web.Gettext
@@ -30,8 +29,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
with {:ok, %Actor{id: group_id} = group} <-
ActivityPub.find_or_make_group_from_nickname(name),
{:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)},
group <- Person.proxify_pictures(group) do
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)} do
{:ok, group}
else
{:member, false} ->
@@ -44,7 +42,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
def find_group(_parent, %{preferred_username: name}, _resolution) do
with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name),
%Actor{} = actor <- Person.proxify_pictures(actor),
%Actor{} = actor <- restrict_fields_for_non_member_request(actor) do
{:ok, actor}
else

View File

@@ -6,7 +6,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.GraphQL.API.Participations
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Email
alias Mobilizon.Web.Email.Checker
@@ -114,7 +113,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
%Participant{} = participant <-
participant
|> Map.put(:event, event)
|> Map.put(:actor, Person.proxify_pictures(actor)) do
|> Map.put(:actor, actor) do
{:ok, participant}
else
{:maximum_attendee_capacity, _} ->

View File

@@ -15,15 +15,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
alias Mobilizon.Federation.ActivityPub
require Logger
alias Mobilizon.Web.{MediaProxy, Upload}
alias Mobilizon.Web.Upload
@doc """
Get a person
"""
def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do
with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true),
true <- suspended == false or is_moderator(role),
actor <- proxify_pictures(actor) do
true <- suspended == false or is_moderator(role) do
{:ok, actor}
else
_ ->
@@ -31,6 +30,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def get_person(_parent, _args, _resolution), do: {:error, :unauthorized}
@doc """
Find a person
"""
@@ -39,8 +40,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
}) do
with {:ok, %Actor{id: actor_id} = actor} <-
ActivityPub.find_or_make_actor_from_nickname(preferred_username),
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)},
actor <- proxify_pictures(actor) do
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)} do
{:ok, actor}
else
{:own, nil} ->
@@ -120,9 +120,12 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
args = Map.put(args, :user_id, user.id)
with args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
end
end
@@ -144,10 +147,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
with {:find_actor, %Actor{} = actor} <-
{:find_actor, Actors.get_actor(id)},
{:is_owned, %Actor{}} <- User.owns_actor(user, actor.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do
{:ok, actor}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:find_actor, nil} ->
{:error, dgettext("errors", "Profile not found")}
@@ -199,18 +205,27 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
defp save_attached_pictures(args) do
Enum.reduce([:avatar, :banner], args, fn key, args ->
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
media = args[key][:media]
with args when is_map(args) <- save_attached_picture(args, :avatar),
args when is_map(args) <- save_attached_picture(args, :banner) do
args
end
end
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
Map.put(args, key, %{"name" => name, "url" => url, "mediaType" => content_type})
end
else
args
defp save_attached_picture(args, key) do
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
with media when is_map(media) <- save_picture(args[key][:media], key) do
Map.put(args, key, media)
end
end)
else
args
end
end
defp save_picture(media, key) do
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
%{"name" => name, "url" => url, "mediaType" => content_type}
end
end
@doc """
@@ -223,10 +238,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
{:no_actor, true} <- {:no_actor, no_actor},
args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- Map.put(args, :user_id, user.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:error, :user_not_found} ->
{:error, dgettext("errors", "No user with this email was found")}
@@ -298,12 +316,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def proxify_pictures(%Actor{} = actor) do
actor
|> proxify_avatar
|> proxify_banner
end
def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{
context: %{current_user: %User{role: role}}
})
@@ -343,20 +355,4 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
defp last_admin_of_a_group?(actor_id) do
length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{avatar: %{url: avatar_url} = avatar} = actor) do
actor |> Map.put(:avatar, avatar |> Map.put(:url, MediaProxy.url(avatar_url)))
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{} = actor), do: actor
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{banner: %{url: banner_url} = banner} = actor) do
actor |> Map.put(:banner, banner |> Map.put(:url, MediaProxy.url(banner_url)))
end
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{} = actor), do: actor
end

View File

@@ -73,6 +73,12 @@ defmodule Mix.Tasks.Mobilizon.Actors.Refresh do
{:actor, nil} ->
shell_error("Error: No such actor")
{:error, err} when is_binary(err) ->
shell_error(err)
_err ->
shell_error("Error while refreshing actor #{preferred_username}")
end
end

View File

@@ -13,11 +13,13 @@ defmodule Mobilizon.Actors do
alias Mobilizon.Actors.{Actor, Bot, Follower, Member}
alias Mobilizon.Addresses.Address
alias Mobilizon.{Crypto, Events}
alias Mobilizon.Events.FeedToken
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Medias.File
alias Mobilizon.Service.Workers
alias Mobilizon.Storage.{Page, Repo}
alias Mobilizon.Users
alias Mobilizon.Users.User
alias Mobilizon.Web.Email.Group
alias Mobilizon.Web.Upload
@@ -215,18 +217,35 @@ defmodule Mobilizon.Actors do
def new_person(args, default_actor \\ false) do
args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key())
with {:ok, %Actor{id: person_id} = person} <-
%Actor{}
|> Actor.registration_changeset(args)
|> Repo.insert() do
Events.create_feed_token(%{user_id: args.user_id, actor_id: person.id})
multi =
Multi.new()
|> Multi.insert(:person, Actor.registration_changeset(%Actor{}, args))
|> Multi.insert(:token, fn %{person: person} ->
FeedToken.changeset(%FeedToken{}, %{
user_id: args.user_id,
actor_id: person.id,
token: Ecto.UUID.generate()
})
end)
multi =
if default_actor do
user = Users.get_user!(args.user_id)
Users.update_user(user, %{default_actor_id: person_id})
Multi.update(multi, :user, fn %{person: person} ->
User.changeset(user, %{default_actor_id: person.id})
end)
else
multi
end
{:ok, person}
case Repo.transaction(multi) do
{:ok, %{person: %Actor{} = person}} ->
{:ok, person}
{:error, _step, err, _} ->
Logger.error("Error while creating a new person")
{:error, err}
end
end

View File

@@ -13,7 +13,7 @@ defmodule Mobilizon.Service.Export.Feed do
alias Mobilizon.Storage.Page
alias Mobilizon.Users.User
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.Router.Helpers, as: Routes
require Logger
@@ -85,14 +85,14 @@ defmodule Mobilizon.Service.Export.Feed do
feed =
if actor.avatar do
feed |> Feed.icon(actor.avatar.url |> MediaProxy.url())
feed |> Feed.icon(actor.avatar.url)
else
feed
end
feed =
if actor.banner do
feed |> Feed.logo(actor.banner.url |> MediaProxy.url())
feed |> Feed.logo(actor.banner.url)
else
feed
end

View File

@@ -0,0 +1,22 @@
defmodule Mobilizon.Service.HTTP.RemoteMediaDownloaderClient do
@moduledoc """
Tesla HTTP Basic Client that fetches HTML to extract metadata preview
"""
use Tesla
alias Mobilizon.Config
@default_opts [
recv_timeout: 20_000
]
adapter(Tesla.Adapter.Hackney, @default_opts)
@user_agent Config.instance_user_agent()
plug(Tesla.Middleware.FollowRedirects)
plug(Tesla.Middleware.Timeout, timeout: 10_000)
plug(Tesla.Middleware.Headers, [{"User-Agent", @user_agent}])
end

View File

@@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
alias Phoenix.HTML.Tag
alias Mobilizon.Actors.Actor
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1]
def build_tags(_actor, _locale \\ "en")
@@ -36,7 +35,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
tags
else
tags ++
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url |> MediaProxy.url())]
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url)]
end
end

View File

@@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
alias Phoenix.HTML.Tag
alias Mobilizon.Events.Event
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1]
def build_tags(%Event{} = event, locale \\ "en") do
@@ -28,7 +27,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
[
Tag.tag(:meta,
property: "og:image",
content: event.picture.file.url |> MediaProxy.url()
content: event.picture.file.url
)
]
end

View File

@@ -47,6 +47,14 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:error, "Cachex error: #{inspect(e)}"}
end
@doc """
Get a filename for the fetched data, using the response header or the last part of the URL
"""
@spec get_filename_from_response(Enum.t(), String.t()) :: String.t() | nil
def get_filename_from_response(response_headers, url) do
get_filename_from_headers(response_headers) || get_filename_from_url(url)
end
@spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()}
defp parse_url(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())

View File

@@ -1,50 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/controller.ex
defmodule Mobilizon.Web.MediaProxyController do
use Mobilizon.Web, :controller
alias Plug.Conn
alias Mobilizon.Config
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Web.ReverseProxy
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
:ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
end
end
def filename_matches(has_filename, path, url) do
filename =
url
|> MediaProxy.filename()
|> URI.decode()
path = URI.decode(path)
if has_filename && filename && Path.basename(path) != filename do
{:wrong_filename, filename}
else
:ok
end
end
end

View File

@@ -1,91 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/media_proxy.ex
defmodule Mobilizon.Web.MediaProxy do
@moduledoc """
Handles proxifying media files
"""
alias Mobilizon.Config
alias Mobilizon.Web.Endpoint
@base64_opts [padding: false]
def url(nil), do: nil
def url(""), do: nil
def url("/" <> _ = url), do: url
def url(url) do
config = Application.get_env(:mobilizon, :media_proxy, [])
if !Keyword.get(config, :enabled, false) or
String.starts_with?(url, Endpoint.url()) do
url
else
encode_url(url)
end
end
def encode_url(url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
# Must preserve `%2F` for compatibility with S3
# https://git.pleroma.social/pleroma/pleroma/issues/580
replacement = get_replacement(url, ":2F:")
# The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
base64 =
url
|> String.replace("%2F", replacement)
|> URI.decode()
|> URI.encode()
|> String.replace(replacement, "%2F")
|> Base.url_encode64(@base64_opts)
sig = :crypto.hmac(:sha, secret, base64)
sig64 = sig |> Base.url_encode64(@base64_opts)
build_url(sig64, base64, filename(url))
end
def decode_url(sig, url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
sig = Base.url_decode64!(sig, @base64_opts)
local_sig = :crypto.hmac(:sha, secret, url)
if local_sig == sig do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
{:error, :invalid_signature}
end
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
[
Config.get([:media_proxy, :base_url], Endpoint.url()),
"proxy",
sig_base64,
url_base64,
filename
]
|> Enum.filter(fn value -> value end)
|> Path.join()
end
defp get_replacement(url, replacement) do
if String.contains?(url, replacement) do
get_replacement(url, replacement <> replacement)
else
replacement
end
end
end

View File

@@ -69,8 +69,6 @@ defmodule Mobilizon.Web.ReverseProxy do
alias Plug.Conn
alias Mobilizon.Web.MediaProxy
require Logger
@type option ::
@@ -111,7 +109,7 @@ defmodule Mobilizon.Web.ReverseProxy do
req_headers = build_req_headers(conn.req_headers, opts)
opts =
if filename = MediaProxy.filename(url) do
if filename = filename(url) do
Keyword.put_new(opts, :attachment_name, filename)
else
opts
@@ -388,4 +386,8 @@ defmodule Mobilizon.Web.ReverseProxy do
defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit}
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
end

View File

@@ -162,13 +162,6 @@ defmodule Mobilizon.Web.Router do
post("/auth/:provider/callback", AuthController, :callback)
end
scope "/proxy/", Mobilizon.Web do
pipe_through(:remote_media)
get("/:sig/:url", MediaProxyController, :remote)
get("/:sig/:url/:filename", MediaProxyController, :remote)
end
if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do
# If using Phoenix
forward("/sent_emails", Bamboo.SentEmailViewerPlug)

View File

@@ -5,7 +5,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
alias Mobilizon.Addresses.Address
alias Mobilizon.Events.Event
alias Mobilizon.Posts.Post
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.JsonLD.ObjectView
def render("group.json", %{group: %Actor{} = group}) do
@@ -41,7 +41,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
"image" =>
if(event.picture,
do: [
event.picture.file.url |> MediaProxy.url()
event.picture.file.url
],
else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"]
)