Export participants to different formats

* CSV
* PDF (requires Python dependency `weasyprint`)
* ODS (requires Python dependency `pyexcel_ods3`)

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel
2021-10-04 18:59:41 +02:00
parent 5dd24e1c9e
commit 0c667b13ae
121 changed files with 10817 additions and 6872 deletions

View File

@@ -26,7 +26,7 @@ defmodule Mobilizon.Federation.ActivityPub.Actions.Accept do
accept_join_entities | accept_follow_entities | accept_invite_entities
@spec accept(acceptable_types, acceptable_entities, boolean, map) ::
{:ok, ActivityStream.t(), acceptable_entities}
{:ok, ActivityStream.t(), acceptable_entities} | {:error, Ecto.Changeset.t()}
def accept(type, entity, local \\ true, additional \\ %{}) do
Logger.debug("We're accepting something")

View File

@@ -31,7 +31,8 @@ defmodule Mobilizon.GraphQL.API.Participations do
@doc """
Update participation status
"""
@spec update(Participant.t(), Actor.t(), atom()) :: {:ok, Activity.t(), Participant.t()}
@spec update(Participant.t(), Actor.t(), atom()) ::
{:ok, Activity.t(), Participant.t()} | {:error, Ecto.Changeset.t()}
def update(%Participant{} = participation, %Actor{} = moderator, :participant),
do: accept(participation, moderator)
@@ -46,7 +47,8 @@ defmodule Mobilizon.GraphQL.API.Participations do
def update(%Participant{} = participation, %Actor{} = moderator, :rejected),
do: reject(participation, moderator)
@spec accept(Participant.t(), Actor.t()) :: {:ok, Activity.t(), Participant.t()}
@spec accept(Participant.t(), Actor.t()) ::
{:ok, Activity.t(), Participant.t()} | {:error, Ecto.Changeset.t()}
defp accept(
%Participant{} = participation,
%Actor{} = moderator

View File

@@ -153,7 +153,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Config do
enabled: !is_nil(Application.get_env(:web_push_encryption, :vapid_details)),
public_key:
get_in(Application.get_env(:web_push_encryption, :vapid_details), [:public_key])
}
},
export_formats: Config.instance_export_formats()
}
end
end

View File

@@ -8,7 +8,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event.Utils do
alias Mobilizon.Federation.ActivityPub.Permission
import Mobilizon.Service.Guards, only: [is_valid_string: 1]
@spec can_event_be_updated_by?(%Event{id: String.t()}, Actor.t()) ::
@spec can_event_be_updated_by?(Event.t(), Actor.t()) ::
boolean
def can_event_be_updated_by?(
%Event{attributed_to: %Actor{type: :Group}} = event,
@@ -24,7 +24,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event.Utils do
Event.can_be_managed_by?(event, actor_member_id)
end
@spec can_event_be_deleted_by?(%Event{id: String.t(), url: String.t()}, Actor.t()) ::
@spec can_event_be_deleted_by?(Event.t(), Actor.t()) ::
boolean
def can_event_be_deleted_by?(
%Event{attributed_to: %Actor{type: :Group}, id: event_id, url: event_url} = event,

View File

@@ -6,6 +6,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.GraphQL.API.Participations
alias Mobilizon.Service.Export.Participants.{CSV, ODS, PDF}
alias Mobilizon.Users.User
alias Mobilizon.Web.Email
alias Mobilizon.Web.Email.Checker
@@ -225,7 +226,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
end
@spec update_participation(any(), map(), Absinthe.Resolution.t()) ::
{:ok, Participation.t()} | {:error, String.t()}
{:ok, Participation.t()} | {:error, String.t() | Ecto.Changeset.t()}
def update_participation(
_parent,
%{id: participation_id, role: new_role},
@@ -236,28 +237,29 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
}
) do
# Check that participation already exists
with {:has_participation, %Participant{role: old_role, event_id: event_id} = participation} <-
{:has_participation, Events.get_participant(participation_id)},
{:same_role, false} <- {:same_role, new_role == old_role},
# Check that moderator has right
{:event, %Event{} = event} <- {:event, Events.get_event_with_preload!(event_id)},
{:event_can_be_managed, true} <-
{:event_can_be_managed, can_event_be_updated_by?(event, moderator_actor)},
{:ok, _activity, participation} <-
Participations.update(participation, moderator_actor, new_role) do
{:ok, participation}
else
{:has_participation, nil} ->
{:error, dgettext("errors", "Participant not found")}
{:event_can_be_managed, _} ->
{:error,
dgettext("errors", "Provided profile doesn't have moderator permissions on this event")}
case Events.get_participant(participation_id) do
%Participant{role: old_role, event_id: event_id} = participation ->
if new_role != old_role do
%Event{} = event = Events.get_event_with_preload!(event_id)
{:same_role, true} ->
{:error, dgettext("errors", "Participant already has role %{role}", role: new_role)}
if can_event_be_updated_by?(event, moderator_actor) do
with {:ok, _activity, participation} <-
Participations.update(participation, moderator_actor, new_role) do
{:ok, participation}
end
else
{:error,
dgettext(
"errors",
"Provided profile doesn't have moderator permissions on this event"
)}
end
else
{:error, dgettext("errors", "Participant already has role %{role}", role: new_role)}
end
{:error, :participant_not_found} ->
nil ->
{:error, dgettext("errors", "Participant not found")}
end
end
@@ -272,16 +274,71 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
with {:has_participant,
%Participant{actor: actor, role: :not_confirmed, event: event} = participant} <-
{:has_participant, Events.get_participant_by_confirmation_token(confirmation_token)},
default_role <- Events.get_default_participant_role(event),
{:ok, _activity, %Participant{} = participant} <-
Participations.update(participant, actor, default_role) do
Participations.update(participant, actor, Events.get_default_participant_role(event)) do
{:ok, participant}
else
{:has_participant, _} ->
{:has_participant, nil} ->
{:error, dgettext("errors", "This token is invalid")}
{:error, %Ecto.Changeset{} = err} ->
{:error, err}
end
end
@spec export_event_participants(any(), map(), Absinthe.Resolution.t()) :: {:ok, String.t()}
def export_event_participants(_parent, %{event_id: event_id, roles: roles, format: format}, %{
context: %{
current_user: %User{locale: locale},
current_actor: %Actor{} = moderator_actor
}
}) do
case Events.get_event_with_preload(event_id) do
{:ok, %Event{} = event} ->
if can_event_be_updated_by?(event, moderator_actor) do
case export_format(format, event, roles, locale) do
{:ok, path} ->
{:ok, path}
{:error, :export_dependency_not_installed} ->
{:error,
dgettext(
"errors",
"A dependency needed to export to %{format} is not installed",
format: format
)}
{:error, :failed_to_save_upload} ->
{:error,
dgettext(
"errors",
"An error occured while saving export",
format: format
)}
{:error, :format_not_supported} ->
{:error,
dgettext(
"errors",
"Format not supported"
)}
end
else
{:error,
dgettext(
"errors",
"Provided profile doesn't have moderator permissions on this event"
)}
end
{:error, :event_not_found} ->
{:error,
dgettext("errors", "Event with this ID %{id} doesn't exist", id: inspect(event_id))}
end
end
def export_event_participants(_, _, _), do: {:error, :unauthorized}
@spec valid_email?(String.t() | nil) :: boolean
defp valid_email?(email) when is_nil(email), do: false
@@ -290,4 +347,24 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
|> String.trim()
|> Checker.valid?()
end
@spec export_format(atom(), Event.t(), list(), String.t()) ::
{:ok, String.t()}
| {:error,
:format_not_supported | :export_dependency_not_installed | :failed_to_save_upload}
defp export_format(format, event, roles, locale) do
case format do
:csv ->
CSV.export(event, roles: roles, locale: locale)
:pdf ->
PDF.export(event, roles: roles, locale: locale)
:ods ->
ODS.export(event, roles: roles, locale: locale)
_ ->
{:error, :format_not_supported}
end
end
end

View File

@@ -65,6 +65,8 @@ defmodule Mobilizon.GraphQL.Schema.ConfigType do
field(:auth, :auth, description: "The instance auth methods")
field(:instance_feeds, :instance_feeds, description: "The instance's feed settings")
field(:web_push, :web_push, description: "Web Push settings for the instance")
field(:export_formats, :export_formats, description: "The instance list of export formats")
end
@desc """
@@ -307,6 +309,15 @@ defmodule Mobilizon.GraphQL.Schema.ConfigType do
field(:public_key, :string, description: "The server's public WebPush VAPID key")
end
@desc """
Export formats configuration
"""
object :export_formats do
field(:event_participants, list_of(:string),
description: "The list of formats the event participants can be exported to"
)
end
object :config_queries do
@desc "Get the instance config"
field :config, :config do

View File

@@ -70,6 +70,12 @@ defmodule Mobilizon.GraphQL.Schema.Events.ParticipantType do
value(:rejected, description: "The participant has been rejected from this event")
end
enum :export_format_enum do
value(:csv, description: "CSV format")
value(:pdf, description: "PDF format")
value(:ods, description: "ODS format")
end
@desc "Represents a deleted participant"
object :deleted_participant do
field(:id, :id, description: "The participant ID")
@@ -111,5 +117,20 @@ defmodule Mobilizon.GraphQL.Schema.Events.ParticipantType do
arg(:confirmation_token, non_null(:string), description: "The participation token")
resolve(&Participant.confirm_participation_from_token/3)
end
@desc "Export the event participants as a file"
field :export_event_participants, :string do
arg(:event_id, non_null(:id),
description: "The ID from the event for which to export participants"
)
arg(:roles, list_of(:participant_role_enum),
default_value: [],
description: "The participant roles to include"
)
arg(:format, :export_format_enum, description: "The format in which to return the file")
resolve(&Participant.export_event_participants/3)
end
end
end

View File

@@ -47,6 +47,7 @@ defmodule Mobilizon do
# workers
Guardian.DB.Token.SweeperServer,
ActivityPub.Federator,
Mobilizon.PythonWorker,
cachex_spec(:feed, 2500, 60, 60, &Feed.create_cache/1),
cachex_spec(:ics, 2500, 60, 60, &ICalendar.create_cache/1),
cachex_spec(

View File

@@ -6,6 +6,7 @@ defmodule Mobilizon.Config do
alias Mobilizon.Actors
alias Mobilizon.Service.GitStatus
require Logger
import Mobilizon.Service.Export.Participants.Common, only: [enabled_formats: 0]
@type mobilizon_config :: [
name: String.t(),
@@ -302,6 +303,13 @@ defmodule Mobilizon.Config do
def instance_event_creation_enabled?,
do: :mobilizon |> Application.get_env(:events) |> Keyword.get(:creation)
@spec instance_export_formats :: %{event_participants: list(String.t())}
def instance_export_formats do
%{
event_participants: enabled_formats()
}
end
@spec anonymous_actor_id :: integer
def anonymous_actor_id, do: get_cached_value(:anonymous_actor_id)
@spec relay_actor_id :: integer

View File

@@ -796,7 +796,7 @@ defmodule Mobilizon.Events do
end
end
@spec get_participant_by_confirmation_token(String.t()) :: Participant.t()
@spec get_participant_by_confirmation_token(String.t()) :: Participant.t() | nil
def get_participant_by_confirmation_token(confirmation_token) do
Participant
|> where([p], fragment("? ->>'confirmation_token' = ?", p.metadata, ^confirmation_token))
@@ -857,9 +857,8 @@ defmodule Mobilizon.Events do
limit \\ nil
) do
id
|> list_participants_for_event_query()
|> filter_role(roles)
|> order_by(asc: :role)
|> participants_for_event_query(roles)
|> preload([:actor, :event])
|> Page.build_page(page, limit)
end
@@ -1604,11 +1603,8 @@ defmodule Mobilizon.Events do
@spec list_participants_for_event_query(String.t()) :: Ecto.Query.t()
defp list_participants_for_event_query(event_id) do
from(
p in Participant,
where: p.event_id == ^event_id,
preload: [:actor, :event]
)
Participant
|> where([p], p.event_id == ^event_id)
end
@spec list_participant_actors_for_event_query(String.t()) :: Ecto.Query.t()
@@ -1621,6 +1617,21 @@ defmodule Mobilizon.Events do
)
end
@spec participants_for_event_query(String.t(), list(atom())) :: Ecto.Query.t()
def participants_for_event_query(id, roles \\ []) do
id
|> list_participants_for_event_query()
|> filter_role(roles)
|> order_by(asc: :role)
end
def participant_for_event_export_query(id, roles) do
id
|> participants_for_event_query(roles)
|> join(:inner, [p], a in Actor, on: p.actor_id == a.id)
|> select([p, a], {p, a})
end
@doc """
List emails for local users (including anonymous ones) participating to an event

57
lib/mobilizon/export.ex Normal file
View File

@@ -0,0 +1,57 @@
defmodule Mobilizon.Export do
@moduledoc """
Manage exported files
"""
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query, only: [where: 3]
alias Mobilizon.Storage.Repo
@type t :: %__MODULE__{
file_path: String.t(),
file_name: String.t() | nil,
file_size: integer() | nil,
type: String.t(),
reference: String.t(),
format: String.t()
}
@required_attrs [:file_path, :type, :reference, :format]
@optional_attrs [:file_size, :file_name]
@attrs @required_attrs ++ @optional_attrs
schema "exports" do
field(:file_path, :string)
field(:file_size, :integer)
field(:file_name, :string)
field(:type, :string)
field(:reference, :string)
field(:format, :string)
timestamps()
end
@doc false
def changeset(export, attrs) do
export
|> cast(attrs, @attrs)
|> validate_required(@required_attrs)
end
@spec get_export(String.t(), String.t(), String.t()) :: t() | nil
def get_export(file_path, type, format) do
__MODULE__
|> where([e], e.file_path == ^file_path and e.type == ^type and e.format == ^format)
|> Repo.one()
end
@spec outdated(String.t(), String.t(), integer()) :: list(t())
def outdated(type, format, expiration) do
expiration_date = DateTime.add(DateTime.utc_now(), -expiration)
__MODULE__
|> where([e], e.type == ^type and e.format == ^format and e.updated_at < ^expiration_date)
|> Repo.all()
end
end

View File

@@ -34,6 +34,7 @@ defmodule Mobilizon.Users.PushSubscription do
|> unique_constraint([:digest, :user_id], name: :user_push_subscriptions_user_id_digest_index)
end
@spec compute_digest(map()) :: String.t()
defp compute_digest(attrs) do
data =
Jason.encode!(%{endpoint: attrs.endpoint, keys: %{auth: attrs.auth, p256dh: attrs.p256dh}})

View File

@@ -129,7 +129,7 @@ defmodule Mobilizon.Users.User do
end
@doc false
@spec registration_changeset(t, map) :: Ecto.Changeset.t()
@spec registration_changeset(t | Ecto.Schema.t(), map) :: Ecto.Changeset.t()
def registration_changeset(%__MODULE__{} = user, attrs) do
user
|> changeset(attrs)
@@ -147,7 +147,7 @@ defmodule Mobilizon.Users.User do
end
@doc false
@spec auth_provider_changeset(t, map) :: Ecto.Changeset.t()
@spec auth_provider_changeset(t | Ecto.Schema.t(), map) :: Ecto.Changeset.t()
def auth_provider_changeset(%__MODULE__{} = user, attrs) do
user
|> changeset(attrs)
@@ -156,13 +156,13 @@ defmodule Mobilizon.Users.User do
end
@doc false
@spec send_password_reset_changeset(t, map) :: Ecto.Changeset.t()
@spec send_password_reset_changeset(t | Ecto.Schema.t(), map) :: Ecto.Changeset.t()
def send_password_reset_changeset(%__MODULE__{} = user, attrs) do
cast(user, attrs, [:reset_password_token, :reset_password_sent_at])
end
@doc false
@spec password_reset_changeset(t, map) :: Ecto.Changeset.t()
@spec password_reset_changeset(t | Ecto.Schema.t(), map) :: Ecto.Changeset.t()
def password_reset_changeset(%__MODULE__{} = user, attrs) do
password_change_changeset(user, attrs, @password_reset_required_attrs)
end

View File

@@ -281,9 +281,9 @@ defmodule Mobilizon.Users do
@doc """
Returns the list of users.
"""
@spec list_users(String.t(), integer | nil, integer | nil, atom | nil, atom | nil) ::
@spec list_users(String.t(), integer | nil, integer | nil, atom, atom) ::
Page.t(User.t())
def list_users(email \\ "", page \\ nil, limit \\ nil, sort \\ nil, direction \\ nil)
def list_users(email, page, limit \\ nil, sort, direction)
def list_users("", page, limit, sort, direction) do
User
@@ -452,7 +452,7 @@ defmodule Mobilizon.Users do
"""
@spec create_push_subscription(map()) ::
{:ok, PushSubscription.t()} | {:error, Ecto.Changeset.t()}
def create_push_subscription(attrs \\ %{}) do
def create_push_subscription(attrs) do
%PushSubscription{}
|> PushSubscription.changeset(attrs)
|> Repo.insert()

View File

@@ -0,0 +1,120 @@
defmodule Mobilizon.Service.Export.Participants.Common do
@moduledoc """
Common functions for managing participants export
"""
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.Participant
alias Mobilizon.Events.Participant.Metadata
alias Mobilizon.Export
alias Mobilizon.Storage.Repo
import Mobilizon.Web.Gettext, only: [gettext: 1]
@spec save_upload(String.t(), String.t(), String.t(), String.t(), String.t()) ::
{:ok, Export.t()} | {:error, atom() | Ecto.Changeset.t()}
def save_upload(full_path, file_path, reference, file_name, format) do
with {:ok, %File.Stat{size: file_size}} <- File.stat(full_path) do
%Export{}
|> Export.changeset(%{
file_size: file_size,
file_name: file_name,
file_path: file_path,
format: format,
reference: reference,
type: "event_participants"
})
|> Repo.insert()
end
end
@doc """
Match a participant role to it's translated version
"""
@spec translate_role(Mobilizon.Events.ParticipantRole.t()) :: String.t()
def translate_role(role) do
case role do
:not_approved ->
gettext("Not approved")
:not_confirmed ->
gettext("Not confirmed")
:rejected ->
gettext("Rejected")
:participant ->
gettext("Participant")
:moderator ->
gettext("Moderator")
:administrator ->
gettext("Administrator")
:creator ->
gettext("Creator")
end
end
@spec columns :: list(String.t())
def columns do
[gettext("Participant name"), gettext("Participant status"), gettext("Participant message")]
end
# One hour
@expiration 60 * 60
@doc """
Clean outdated files in export folder
"""
@spec clean_exports(String.t(), String.t(), integer()) :: :ok
def clean_exports(format, upload_path, expiration \\ @expiration) do
"event_participants"
|> Export.outdated(format, expiration)
|> Enum.each(&remove_export(&1, upload_path))
end
defp remove_export(%Export{file_path: filename} = export, upload_path) do
full_path = upload_path <> filename
File.rm(full_path)
Repo.delete!(export)
end
@spec to_list({Participant.t(), Actor.t()}) :: list(String.t())
def to_list(
{%Participant{role: role, metadata: metadata},
%Actor{domain: nil, preferred_username: "anonymous"}}
) do
[gettext("Anonymous participant"), translate_role(role), convert_metadata(metadata)]
end
def to_list({%Participant{role: role, metadata: metadata}, %Actor{} = actor}) do
[Actor.display_name_and_username(actor), translate_role(role), convert_metadata(metadata)]
end
@spec convert_metadata(Metadata.t() | nil) :: String.t()
defp convert_metadata(%Metadata{message: message}) when is_binary(message) do
message
end
defp convert_metadata(_), do: ""
@spec export_modules :: list(module())
def export_modules do
export_config = Application.get_env(:mobilizon, :exports)
Keyword.get(export_config, :formats, [])
end
@spec enabled_formats :: list(String.t())
def enabled_formats do
export_modules()
|> Enum.map(& &1.extension())
end
@spec export_enabled?(module()) :: boolean
def export_enabled?(type) do
export_config = Application.get_env(:mobilizon, :exports)
formats = Keyword.get(export_config, :formats, [])
type in formats
end
end

View File

@@ -0,0 +1,100 @@
defmodule Mobilizon.Service.Export.Participants.CSV do
@moduledoc """
Export a list of participants to CSV
"""
alias Mobilizon.Events
alias Mobilizon.Events.Event
alias Mobilizon.Export
alias Mobilizon.Storage.Repo
alias Mobilizon.Web.Gettext
import Mobilizon.Web.Gettext, only: [gettext: 2]
import Mobilizon.Service.Export.Participants.Common,
only: [save_upload: 5, columns: 0, to_list: 1, clean_exports: 2, export_enabled?: 1]
@upload_path "uploads/exports/csv/"
@extension "csv"
def extension do
@extension
end
@spec export(Event.t(), Keyword.t()) ::
{:ok, String.t()} | {:error, :failed_to_save_upload | :export_dependency_not_installed}
def export(%Event{id: event_id} = event, options \\ []) do
if ready?() do
filename = "#{ShortUUID.encode!(Ecto.UUID.generate())}.csv"
full_path = @upload_path <> filename
file = File.open!(full_path, [:write, :utf8])
case Repo.transaction(
fn ->
event_id
|> Events.participant_for_event_export_query(Keyword.get(options, :roles, []))
|> Repo.stream()
|> Stream.map(&to_list/1)
|> NimbleCSV.RFC4180.dump_to_iodata()
|> (fn stream -> Stream.concat([Enum.join(columns(), ","), "\n"], stream) end).()
|> Stream.each(fn line -> IO.write(file, line) end)
|> Stream.run()
with {:error, err} <- save_csv_upload(full_path, filename, event) do
Repo.rollback(err)
end
end,
timeout: :infinity
) do
{:error, _err} ->
File.rm!(full_path)
{:error, :failed_to_save_upload}
{:ok, _ok} ->
{:ok, filename}
end
else
{:error, :export_dependency_not_installed}
end
end
@spec save_csv_upload(String.t(), String.t(), Event.t()) ::
{:ok, Export.t()} | {:error, atom() | Ecto.Changeset.t()}
defp save_csv_upload(full_path, filename, %Event{id: event_id, title: title}) do
Gettext.gettext_comment(
"File name template for exported list of participants. Should NOT contain spaces. Make sure the output is going to be something standardized that is acceptable as a file name on most systems."
)
save_upload(
full_path,
filename,
to_string(event_id),
gettext("%{event}_participants", event: title) <> ".csv",
"csv"
)
end
@doc """
Clean outdated files in export folder
"""
@spec clean_exports :: :ok
def clean_exports do
clean_exports("csv", @upload_path)
end
@spec dependencies_ok? :: boolean
def dependencies_ok? do
true
end
@spec enabled? :: boolean
def enabled? do
export_enabled?(__MODULE__)
end
@spec ready? :: boolean
def ready? do
enabled?() && dependencies_ok?()
end
end

View File

@@ -0,0 +1,106 @@
defmodule Mobilizon.Service.Export.Participants.ODS do
@moduledoc """
Export a list of participants to ODS
"""
alias Mobilizon.Events
alias Mobilizon.Events.Event
alias Mobilizon.Export
alias Mobilizon.PythonWorker
alias Mobilizon.Storage.Repo
alias Mobilizon.Web.Gettext, as: GettextBackend
import Mobilizon.Web.Gettext, only: [gettext: 2]
import Mobilizon.Service.Export.Participants.Common,
only: [save_upload: 5, to_list: 1, clean_exports: 2, columns: 0, export_enabled?: 1]
@upload_path "uploads/exports/ods/"
@extension "ods"
def extension do
@extension
end
@spec export(Event.t(), Keyword.t()) ::
{:ok, String.t()} | {:error, :failed_to_save_upload | :export_dependency_not_installed}
def export(%Event{id: event_id} = event, options \\ []) do
if ready?() do
filename = "#{ShortUUID.encode!(Ecto.UUID.generate())}.ods"
full_path = @upload_path <> filename
case Repo.transaction(
fn ->
content =
event_id
|> Events.participant_for_event_export_query(Keyword.get(options, :roles, []))
|> Repo.all()
|> Enum.map(&to_list/1)
|> (fn data -> Enum.concat([columns()], data) end).()
|> generate_ods()
File.write!(full_path, content)
with {:error, err} <- save_ods_upload(full_path, filename, event) do
Repo.rollback(err)
end
end,
timeout: :infinity
) do
{:error, _err} ->
File.rm!(full_path)
{:error, :failed_to_save_upload}
{:ok, _ok} ->
{:ok, filename}
end
else
{:error, :export_dependency_not_installed}
end
end
defp generate_ods(data) do
data
|> Jason.encode!()
|> PythonWorker.generate_ods()
end
@spec save_ods_upload(String.t(), String.t(), Event.t()) ::
{:ok, Export.t()} | {:error, atom() | Ecto.Changeset.t()}
defp save_ods_upload(full_path, filename, %Event{id: event_id, title: title}) do
GettextBackend.gettext_comment(
"File name template for exported list of participants. Should NOT contain spaces. Make sure the output is going to be something standardized that is acceptable as a file name on most systems."
)
save_upload(
full_path,
filename,
to_string(event_id),
gettext("%{event}_participants", event: title) <> ".ods",
"ods"
)
end
@doc """
Clean outdated files in export folder
"""
@spec clean_exports :: :ok
def clean_exports do
clean_exports("ods", @upload_path)
end
@spec dependencies_ok? :: boolean
def dependencies_ok? do
PythonWorker.has_module("pyexcel_ods3")
end
@spec enabled? :: boolean
def enabled? do
export_enabled?(__MODULE__)
end
@spec ready? :: boolean
def ready? do
enabled?() && dependencies_ok?()
end
end

View File

@@ -0,0 +1,120 @@
defmodule Mobilizon.Service.Export.Participants.PDF do
@moduledoc """
Export a list of participants to PDF
"""
alias Mobilizon.Events
alias Mobilizon.Events.Event
alias Mobilizon.Export
alias Mobilizon.PythonWorker
alias Mobilizon.Storage.Repo
alias Mobilizon.Web.ExportView
alias Mobilizon.Web.Gettext, as: GettextBackend
alias Phoenix.HTML.Safe
import Mobilizon.Web.Gettext, only: [gettext: 2]
import Mobilizon.Service.Export.Participants.Common,
only: [save_upload: 5, columns: 0, to_list: 1, clean_exports: 2, export_enabled?: 1]
@upload_path "uploads/exports/pdf/"
@extension "pdf"
def extension do
@extension
end
@spec export(Event.t(), Keyword.t()) ::
{:ok, String.t()} | {:error, :failed_to_save_upload | :export_dependency_not_installed}
def export(%Event{id: event_id} = event, options \\ []) do
if ready?() do
filename = "#{ShortUUID.encode!(Ecto.UUID.generate())}.pdf"
full_path = @upload_path <> filename
case Repo.transaction(
fn ->
content =
event_id
|> Events.participant_for_event_export_query(Keyword.get(options, :roles, []))
|> Repo.all()
|> Enum.map(&to_list/1)
|> render_template(event, Keyword.get(options, :locale, "en"))
|> generate_pdf()
File.write!(full_path, content)
with {:error, err} <- save_pdf_upload(full_path, filename, event) do
Repo.rollback(err)
end
end,
timeout: :infinity
) do
{:error, _err} ->
File.rm!(full_path)
{:error, :failed_to_save_upload}
{:ok, _ok} ->
{:ok, filename}
end
else
{:error, :export_dependency_not_installed}
end
end
@spec render_template(list(), Event.t(), String.t()) :: String.t()
defp render_template(data, %Event{} = event, locale) do
Gettext.put_locale(locale)
ExportView.render("event_participants.html",
data: data,
columns: columns(),
event: event,
locale: locale
)
|> Safe.to_iodata()
|> IO.iodata_to_binary()
end
defp generate_pdf(html) do
PythonWorker.generate_pdf(html)
end
@spec save_pdf_upload(String.t(), String.t(), Event.t()) ::
{:ok, Export.t()} | {:error, atom() | Ecto.Changeset.t()}
defp save_pdf_upload(full_path, filename, %Event{id: event_id, title: title}) do
GettextBackend.gettext_comment(
"File name template for exported list of participants. Should NOT contain spaces. Make sure the output is going to be something standardized that is acceptable as a file name on most systems."
)
save_upload(
full_path,
filename,
to_string(event_id),
gettext("%{event}_participants", event: title) <> ".pdf",
"pdf"
)
end
@doc """
Clean outdated files in export folder
"""
@spec clean_exports :: :ok
def clean_exports do
clean_exports("pdf", @upload_path)
end
@spec dependencies_ok? :: boolean
def dependencies_ok? do
PythonWorker.has_module("weasyprint")
end
@spec enabled? :: boolean
def enabled? do
export_enabled?(__MODULE__)
end
@spec ready? :: boolean
def ready? do
enabled?() && dependencies_ok?()
end
end

View File

@@ -0,0 +1,28 @@
defmodule Mobilizon.PythonPort do
@moduledoc """
Port to use Python modules from Elixir
"""
use Export.Python
@doc """
## Parameters
- path: directory to include in python path
"""
@spec python_instance(String.t()) :: pid
def python_instance(path) do
python = "/usr/bin/python3"
{:ok, pid} = Python.start(python: python, python_path: path)
pid
end
@doc """
Call python function using MFA format
"""
@spec call_python(pid, binary, binary, list) :: any
def call_python(pid, module, function, arguments \\ []) do
Python.call(pid, module, function, arguments)
end
end

View File

@@ -0,0 +1,65 @@
defmodule Mobilizon.PythonWorker do
@moduledoc """
Genserver to handle an instance of Python handling the calls to `Mobilizon.PythonPort`.
"""
use GenServer
use Export.Python
alias Mobilizon.PythonPort
@spec start_link(any) :: :ignore | {:error, any} | {:ok, pid}
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@spec init(any) :: {:ok, %{python_pid: pid}}
def init(_) do
path = Path.join([:code.priv_dir(:mobilizon), "python"])
pid = PythonPort.python_instance(path)
{:ok, %{python_pid: pid}}
end
def terminate(_reason, %{python_pid: pid}) do
Python.stop(pid)
end
@spec generate_pdf(String.t()) :: any
def generate_pdf(html) do
GenServer.call(__MODULE__, %{html: html, format: :pdf})
end
@spec generate_ods(String.t()) :: any
def generate_ods(data) do
GenServer.call(__MODULE__, %{data: data, format: :ods})
end
@spec has_module(String.t()) :: any
def has_module(module) do
GenServer.call(__MODULE__, %{module: module})
end
@spec handle_call(
%{html: String.t(), format: :pdf} | %{data: String.t(), format: :ods},
any(),
map()
) :: {:reply, String.t(), map()}
def handle_call(%{html: html, format: :pdf}, _from, %{python_pid: pid} = state) do
res = PythonPort.call_python(pid, "pdf", "generate", [html])
{:reply, res, state}
end
def handle_call(%{data: data, format: :ods}, _from, %{python_pid: pid} = state) do
res = PythonPort.call_python(pid, "ods", "generate", [data])
{:reply, res, state}
end
def handle_call(%{module: module}, _from, %{python_pid: pid} = state) do
res = PythonPort.call_python(pid, "module", "has_package", [module])
{:reply, res, state}
end
end

View File

@@ -0,0 +1,14 @@
defmodule Mobilizon.Service.Workers.ExportCleanerWorker do
@moduledoc """
Worker to clean exports
"""
use Oban.Worker, queue: "background"
import Mobilizon.Service.Export.Participants.Common, only: [export_modules: 0]
@impl Oban.Worker
@spec perform(Oban.Job.t()) :: :ok
def perform(%Job{}) do
Enum.each(export_modules(), & &1.clean_exports())
end
end

View File

@@ -53,28 +53,30 @@ defmodule Mobilizon.Web.Auth.Guardian do
end
end
@spec on_verify(any(), any(), any()) :: {:ok, any()}
@spec on_verify(any(), any(), any()) :: {:ok, map()} | {:error, :token_not_found}
def on_verify(claims, token, _options) do
with {:ok, _} <- Guardian.DB.on_verify(claims, token) do
{:ok, claims}
end
end
@spec on_revoke(any(), any(), any()) :: {:ok, any()}
@spec on_revoke(any(), any(), any()) :: {:ok, map()} | {:error, :could_not_revoke_token}
def on_revoke(claims, token, _options) do
with {:ok, _} <- Guardian.DB.on_revoke(claims, token) do
{:ok, claims}
end
end
@spec on_refresh({any(), any()}, {any(), any()}, any()) :: {:ok, {any(), any()}, {any(), any()}}
@spec on_refresh({any(), any()}, {any(), any()}, any()) ::
{:ok, {String.t(), map()}, {String.t(), map()}} | {:error, any()}
def on_refresh({old_token, old_claims}, {new_token, new_claims}, _options) do
with {:ok, _, _} <- Guardian.DB.on_refresh({old_token, old_claims}, {new_token, new_claims}) do
{:ok, {old_token, old_claims}, {new_token, new_claims}}
end
end
@spec on_exchange(any(), any(), any()) :: {:ok, {any(), any()}, {any(), any()}}
@spec on_exchange(any(), any(), any()) ::
{:ok, {String.t(), map()}, {String.t(), map()}} | {:error, any()}
def on_exchange(old_stuff, new_stuff, options), do: on_refresh(old_stuff, new_stuff, options)
# def build_claims(claims, _resource, opts) do

View File

@@ -0,0 +1,35 @@
defmodule Mobilizon.Web.ExportController do
@moduledoc """
Controller to serve exported files
"""
use Mobilizon.Web, :controller
plug(:put_layout, false)
action_fallback(Mobilizon.Web.FallbackController)
alias Mobilizon.Export
import Mobilizon.Service.Export.Participants.Common, only: [enabled_formats: 0]
import Mobilizon.Web.Gettext, only: [dgettext: 3]
# sobelow_skip ["Traversal.SendDownload"]
@spec export(Plug.Conn.t(), map) :: {:error, :not_found} | Plug.Conn.t()
def export(conn, %{"format" => format, "file" => file}) do
if format in enabled_formats() do
case Export.get_export(file, "event_participants", format) do
%Export{file_name: file_name, file_path: file_path} ->
local_path = "uploads/exports/#{format}/#{file_path}"
# We're using encode: false to disable escaping the filename with URI.encode_www_form/1
# but it may introduce an security issue if the event title wasn't properly sanitized
# https://github.com/phoenixframework/phoenix/pull/3344
# https://owasp.org/www-community/attacks/HTTP_Response_Splitting
send_download(conn, {:file, local_path}, filename: file_name, encode: false)
nil ->
{:error, :not_found}
end
else
{:error,
dgettext("errors", "Export to format %{format} is not enabled on this instance",
format: format
)}
end
end
end

View File

@@ -77,17 +77,6 @@ defmodule Mobilizon.Web.Endpoint do
plug(Plug.MethodOverride)
plug(Plug.Head)
# The session will be stored in the cookie and signed,
# this means its contents can be read but not tampered with.
# Set :encryption_salt if you would also like to encrypt it.
plug(
Plug.Session,
store: :cookie,
key: "_mobilizon_key",
signing_salt: "F9CCTF22"
)
plug(Mobilizon.Web.Router)
@spec websocket_url :: String.t()

View File

@@ -61,6 +61,9 @@ defmodule Mobilizon.Web.Router do
plug(:accepts, ["atom", "ics", "html"])
end
pipeline :exports do
end
pipeline :browser do
plug(Plug.Static, at: "/", from: "priv/static")
@@ -78,6 +81,11 @@ defmodule Mobilizon.Web.Router do
pipeline :remote_media do
end
scope "/exports", Mobilizon.Web do
pipe_through(:browser)
get("/:format/:file", ExportController, :export)
end
scope "/api" do
pipe_through(:graphql)

View File

@@ -0,0 +1,154 @@
<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1">
<style>
table {
border: 1px solid #bdbdbd;
border-collapse: collapse;
width: 100%; }
th,
td,
table caption {
padding: 0.75rem;
text-align: left;
text-align: start;
/* 1 */ }
[dir="rtl"] th,
[dir="rtl"] td,
[dir="rtl"] table caption {
text-align: right;
text-align: start;
/* 1 */ }
td {
vertical-align: text-top;
}
th {
vertical-align: bottom; }
th[scope="col"] {
background-color: #024488;
color: #fff; }
dl {
display: flex;
flex-flow: row wrap;
}
dt {
flex-basis: 20%;
padding: 2px 4px;
text-align: right;
}
dd {
flex-basis: 70%;
flex-grow: 1;
margin: 0;
padding: 2px 4px;
}
dl dt {
font-weight: bold; }
dl dd + dt {
margin-top: 0.5em; }
dl dt + dd,
dl dd + dd {
margin-top: 0.25em; }
tr:nth-child(even) {
background-color: rgba(255, 255, 255, 0.25);
}
tr:nth-child(odd) {
background-color: rgba(0, 0, 0, 0.05);
}
th {
background-color: rgba(0, 0, 0, 0.5);
}
@media print {
body {
/*font-size: 6pt;*/
color: #000;
background-color: #fff;
background-image: none;
}
body,
main {
margin: 0;
padding: 0;
background-color: #fff;
border: none;
}
table {
page-break-inside: avoid;
}
div {
overflow: visible;
}
th {
color: #000;
background-color: #fff;
border-bottom: 1pt solid #000;
}
tr {
border-top: 1pt solid #000;
}
}
@media print and (max-width: 5in) {
caption {
color: #000;
background-color: #fff;
border-bottom: 1pt solid #000;
}
table {
page-break-inside: auto;
}
tr {
page-break-inside: avoid;
}
}
</style>
<%# <title><%= gettext("Participants for %{event}") ></title> %>
</head>
<body>
<h1><%= gettext("Participants for %{event}", event: @event.title) %></h1>
<dl>
<dt><%= gettext("Begins on") %></dt>
<dd><%= datetime_to_string(@event.begins_on, @locale, :long) %></dd>
<%= if @event.ends_on do %>
<dt><%= gettext("Ends on") %></dt>
<dd><%= datetime_to_string(@event.ends_on, @locale, :long) %></dd>
<% end %>
<%= if @event.physical_address do %>
<dt><%= gettext("Location") %></dt>
<dd><%= render_address(@event.physical_address) %></dd>
<% end %>
<dt><%= gettext("Number of participants") %></dt>
<dd><%= @event.participant_stats.participant + @event.participant_stats.moderator + @event.participant_stats.administrator + @event.participant_stats.creator %></dd>
</dl>
<table>
<thead>
<tr>
<%= for column <- @columns do %>
<th><%= column %></th>
<% end %>
</tr>
</thead>
<tbody>
<%= for line <- @data do %>
<tr>
<%= for cell <- line do %>
<td><%= cell %></td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
</body>
</html>

View File

@@ -0,0 +1,17 @@
defmodule Mobilizon.Web.ExportView do
use Mobilizon.Web, :view
alias Mobilizon.Service.Address
alias Mobilizon.Service.DateTime, as: DateTimeRenderer
import Mobilizon.Web.Gettext
defdelegate datetime_to_string(datetime, locale \\ "en", format \\ :medium),
to: DateTimeRenderer
defdelegate datetime_to_time_string(datetime, locale \\ "en", format \\ :short),
to: DateTimeRenderer
defdelegate datetime_tz_convert(datetime, timezone), to: DateTimeRenderer
defdelegate datetime_relative(datetime, locale \\ "en"), to: DateTimeRenderer
defdelegate render_address(address), to: Address
end