hamlab/mobilizon-frontend
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Introduce event language detection

Browse Source 
Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel
2021-08-19 20:43:35 +02:00
parent 7c9b76765f
commit d577b07c6e
16 changed files with 206 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/federation/activity_pub/types/comments.ex
View File

@@ -10,6 +10,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Comments do
alias Mobilizon.Federation.ActivityStream.Convertible
alias Mobilizon.GraphQL.API.Utils, as: APIUtils
alias Mobilizon.Service.Activity.Comment, as: CommentActivity
alias Mobilizon.Service.LanguageDetection
alias Mobilizon.Share
alias Mobilizon.Tombstone
alias Mobilizon.Web.Endpoint
@@ -127,6 +128,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Comments do
),
tags <- ConverterUtils.fetch_tags(tags),
mentions <- Map.get(args, :mentions, []) ++ ConverterUtils.fetch_mentions(mentions),
lang <- Map.get(args, :language, "und"),
args <-
Map.merge(args, %{
actor_id: Map.get(args, :actor_id),
@@ -141,7 +143,8 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Comments do
if(is_nil(in_reply_to_comment),
do: nil,
else: Comment.get_thread_id(in_reply_to_comment)
)
),
language: if(lang == "und", do: LanguageDetection.detect(:comment, args), else: lang)
}) do
args
end

5
lib/federation/activity_pub/types/events.ex
View File

@@ -12,6 +12,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Events do
alias Mobilizon.GraphQL.API.Utils, as: APIUtils
alias Mobilizon.Service.Activity.Event, as: EventActivity
alias Mobilizon.Service.Formatter.HTML
alias Mobilizon.Service.LanguageDetection
alias Mobilizon.Service.Notifications.Scheduler
alias Mobilizon.Share
alias Mobilizon.Tombstone
@@ -234,6 +235,10 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Events do
args
|> Map.put(:options, options)
|> Map.put_new(:language, "und")
|> Map.update!(:language, fn lang ->
if lang == "und", do: LanguageDetection.detect(:event, args), else: lang
end)
|> Map.update(:tags, [], &ConverterUtils.fetch_tags/1)
|> Map.update(:contacts, [], &ConverterUtils.fetch_actors/1)
end

14
lib/federation/activity_pub/types/posts.ex
View File

@@ -8,6 +8,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Posts do
alias Mobilizon.Federation.ActivityStream.Convertible
alias Mobilizon.Posts.Post
alias Mobilizon.Service.Activity.Post, as: PostsActivity
alias Mobilizon.Service.LanguageDetection
require Logger
import Mobilizon.Federation.ActivityPub.Utils, only: [make_create_data: 2, make_update_data: 2]
@@ -17,7 +18,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Posts do
@impl Entity
def create(args, additional) do
with args <- Map.update(args, :tags, [], &ConverterUtils.fetch_tags/1),
with args <- prepare_args(args),
{:ok, %Post{attributed_to_id: group_id, author_id: creator_id} = post} <-
Posts.create_post(args),
{:ok, _} <- PostsActivity.insert_activity(post, subject: "post_created"),
@@ -37,7 +38,7 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Posts do
@impl Entity
def update(%Post{} = post, args, additional) do
with args <- Map.update(args, :tags, [], &ConverterUtils.fetch_tags/1),
with args <- prepare_args(args),
{:ok, %Post{attributed_to_id: group_id, author_id: creator_id} = post} <-
Posts.update_post(post, args),
{:ok, _} <- PostsActivity.insert_activity(post, subject: "post_updated"),
@@ -99,4 +100,13 @@ defmodule Mobilizon.Federation.ActivityPub.Types.Posts do
delete: :moderator
}
end
defp prepare_args(args) do
args
|> Map.update(:tags, [], &ConverterUtils.fetch_tags/1)
|> Map.put_new(:language, "und")
|> Map.update!(:language, fn lang ->
if lang == "und", do: LanguageDetection.detect(:post, args), else: lang
end)
end
end

4
lib/federation/activity_pub/utils.ex
View File

@@ -36,6 +36,7 @@ defmodule Mobilizon.Federation.ActivityPub.Utils do
"https://www.w3.org/ns/activitystreams",
"https://litepub.social/context.jsonld",
%{
"@language" => "und",
"sc" => "http://schema.org#",
"ical" => "http://www.w3.org/2002/12/cal/ical#",
"pt" => "https://joinpeertube.org/ns#",
@@ -91,7 +92,8 @@ defmodule Mobilizon.Federation.ActivityPub.Utils do
},
"PropertyValue" => "sc:PropertyValue",
"value" => "sc:value",
"propertyID" => "sc:propertyID"
"propertyID" => "sc:propertyID",
"inLanguage" => "sc:inLanguage"
}
]
}

6
lib/federation/activity_stream/converter/event.ex
View File

@@ -83,7 +83,8 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
mentions: mentions,
physical_address_id: address_id,
updated_at: object["updated"],
publish_at: object["published"]
publish_at: object["published"],
language: object["inLanguage"]
}
else
{:ok, %Actor{suspended: true}} ->
@@ -128,7 +129,8 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Event do
"draft" => event.draft,
"ical:status" => event.status |> to_string |> String.upcase(),
"id" => event.url,
"url" => event.url
"url" => event.url,
"inLanguage" => event.language
}
|> maybe_add_physical_address(event)
|> maybe_add_event_picture(event)

4
lib/graphql/schema/discussions/comment.ex
View File

@@ -54,6 +54,8 @@ defmodule Mobilizon.GraphQL.Schema.Discussions.CommentType do
field(:is_announcement, non_null(:boolean),
description: "Whether this comment needs to be announced to participants"
)
field(:language, non_null(:string), description: "The comment language")
end
@desc "The list of visibility options for a comment"
@@ -89,6 +91,7 @@ defmodule Mobilizon.GraphQL.Schema.Discussions.CommentType do
arg(:text, non_null(:string), description: "The comment's body")
arg(:event_id, non_null(:id), description: "The event under which this comment is")
arg(:in_reply_to_comment_id, :id, description: "The comment ID this one replies to")
arg(:language, :string, description: "The comment language", default_value: "und")
arg(:is_announcement, :boolean, description: "Should this comment be announced to everyone?")
@@ -99,6 +102,7 @@ defmodule Mobilizon.GraphQL.Schema.Discussions.CommentType do
field :update_comment, type: :comment do
arg(:text, non_null(:string), description: "The comment updated body")
arg(:comment_id, non_null(:id), description: "The comment ID")
arg(:language, :string, description: "The comment language", default_value: "und")
arg(:is_announcement, :boolean, description: "Should this comment be announced to everyone?")

3
lib/graphql/schema/event.ex
View File

@@ -104,6 +104,7 @@ defmodule Mobilizon.GraphQL.Schema.EventType do
field(:inserted_at, :datetime, description: "When the event was created")
field(:options, :event_options, description: "The event options")
field(:metadata, list_of(:event_metadata), description: "A key-value list of metadata")
field(:language, non_null(:string), description: "The event language")
end
@desc "The list of visibility options for an event"
@@ -401,6 +402,7 @@ defmodule Mobilizon.GraphQL.Schema.EventType do
)
arg(:contacts, list_of(:contact), default_value: [], description: "The events contacts")
arg(:language, :string, description: "The event language", default_value: "und")
resolve(&Event.create_event/3)
end
@@ -444,6 +446,7 @@ defmodule Mobilizon.GraphQL.Schema.EventType do
arg(:metadata, list_of(:event_metadata_input), description: "The event metadata")
arg(:draft, :boolean, description: "Whether or not the event is a draft")
arg(:contacts, list_of(:contact), default_value: [], description: "The events contacts")
arg(:language, :string, description: "The event language", default_value: "und")
resolve(&Event.update_event/3)
end

3
lib/graphql/schema/post.ex
View File

@@ -20,6 +20,7 @@ defmodule Mobilizon.GraphQL.Schema.PostType do
field(:publish_at, :datetime, description: "When the post was published")
field(:inserted_at, :datetime, description: "The post's creation date")
field(:updated_at, :datetime, description: "The post's last update date")
field(:language, non_null(:string), description: "The post language")
field(:tags, list_of(:tag),
resolve: &Tag.list_tags_for_post/3,
@@ -71,6 +72,7 @@ defmodule Mobilizon.GraphQL.Schema.PostType do
arg(:draft, :boolean, default_value: false, description: "Whether the post is a draft")
arg(:visibility, :post_visibility, description: "The post's visibility")
arg(:publish_at, :datetime, description: "The post's publish date")
arg(:language, :string, description: "The post language", default_value: "und")
arg(:tags, list_of(:string),
default_value: [],
@@ -93,6 +95,7 @@ defmodule Mobilizon.GraphQL.Schema.PostType do
arg(:attributed_to_id, :id, description: "The group the post is attributed to")
arg(:draft, :boolean, description: "Whether the post is a draft")
arg(:visibility, :post_visibility, description: "The post's visibility")
arg(:language, :string, description: "The post language", default_value: "und")
arg(:publish_at, :datetime,
description: "The time when the posts is going to be or has been published"

7
lib/mobilizon/discussions/comment.ex
View File

@@ -30,7 +30,8 @@ defmodule Mobilizon.Discussions.Comment do
mentions: [Mention.t()],
media: [Media.t()],
in_reply_to_comment: t,
origin_comment: t
origin_comment: t,
language: String.t()
}
# When deleting an event we only nihilify everything
@@ -46,7 +47,8 @@ defmodule Mobilizon.Discussions.Comment do
:deleted_at,
:local,
:is_announcement,
:discussion_id
:discussion_id,
:language
]
@attrs @required_attrs ++ @optional_attrs
@@ -60,6 +62,7 @@ defmodule Mobilizon.Discussions.Comment do
field(:deleted_at, :utc_datetime)
field(:published_at, :utc_datetime)
field(:is_announcement, :boolean, default: false)
field(:language, :string, default: "und")
belongs_to(:actor, Actor, foreign_key: :actor_id)
belongs_to(:attributed_to, Actor, foreign_key: :attributed_to_id)

7
lib/mobilizon/events/event.ex
View File

@@ -62,7 +62,8 @@ defmodule Mobilizon.Events.Event do
mentions: [Mention.t()],
tags: [Tag.t()],
participants: [Actor.t()],
contacts: [Actor.t()]
contacts: [Actor.t()],
language: String.t()
}
@update_required_attrs [:title, :begins_on, :organizer_actor_id]
@@ -83,7 +84,8 @@ defmodule Mobilizon.Events.Event do
:phone_address,
:picture_id,
:physical_address_id,
:attributed_to_id
:attributed_to_id,
:language
]
@attrs @required_attrs ++ @optional_attrs
@@ -106,6 +108,7 @@ defmodule Mobilizon.Events.Event do
field(:online_address, :string)
field(:phone_address, :string)
field(:category, :string)
field(:language, :string, default: "und")
embeds_one(:options, EventOptions, on_replace: :delete)
embeds_one(:participant_stats, EventParticipantStats, on_replace: :update)

6
lib/mobilizon/posts/post.ex
View File

@@ -43,7 +43,8 @@ defmodule Mobilizon.Posts.Post do
attributed_to: Actor.t(),
picture: Media.t(),
media: [Media.t()],
tags: [Tag.t()]
tags: [Tag.t()],
language: String.t()
}
@primary_key {:id, Ecto.UUID, autogenerate: true}
@@ -57,6 +58,7 @@ defmodule Mobilizon.Posts.Post do
field(:url, :string)
field(:publish_at, :utc_datetime)
field(:visibility, PostVisibility, default: :public)
field(:language, :string, default: "und")
belongs_to(:author, Actor)
belongs_to(:attributed_to, Actor)
belongs_to(:picture, Media, on_replace: :update)
@@ -76,7 +78,7 @@ defmodule Mobilizon.Posts.Post do
:author_id,
:attributed_to_id
]
@optional_attrs [:picture_id, :local, :publish_at, :visibility]
@optional_attrs [:picture_id, :local, :publish_at, :visibility, :language]
@attrs @required_attrs ++ @optional_attrs
@doc false

84
lib/service/language_detection/language_detection.ex Normal file
View File

@@ -0,0 +1,84 @@
defmodule Mobilizon.Service.LanguageDetection do
@moduledoc """
Detect the language of the event
"""
alias Mobilizon.Service.Formatter.HTML
@und "und"
@paasaa_languages Paasaa.Data.languages()
|> Map.values()
|> List.flatten()
|> Enum.map(fn {lang, _val} ->
lang
end)
@allow_listed_locales Mobilizon.Cldr.known_locale_names()
@type entity_type :: :event | :comment | :post
@spec detect(entity_type(), map()) :: String.t()
def detect(:event, %{title: title} = args) do
description = Map.get(args, :description)
if is_nil(description) or description == "" do
title
|> Paasaa.detect(whitelist: allow_listed_languages())
|> normalize()
else
sanitized_description = HTML.strip_tags_and_insert_spaces(description)
"#{title}\n\n#{sanitized_description}"
|> Paasaa.detect(whitelist: allow_listed_languages())
|> normalize()
end
end
def detect(:comment, %{text: text}) do
text
|> HTML.strip_tags_and_insert_spaces()
|> Paasaa.detect(whitelist: allow_listed_languages())
|> normalize()
end
def detect(:post, %{title: title} = args) do
body = Map.get(args, :body)
if is_nil(body) or body == "" do
title
|> Paasaa.detect(whitelist: allow_listed_languages())
|> normalize()
else
sanitized_body = HTML.strip_tags_and_insert_spaces(body)
"#{title}\n\n#{sanitized_body}"
|> Paasaa.detect(whitelist: allow_listed_languages())
|> normalize()
end
end
def detect(_, _), do: @und
@spec normalize(String.t()) :: String.t()
def normalize(""), do: @und
def normalize(language) do
case Cldr.AcceptLanguage.parse(language, Mobilizon.Cldr) do
{:ok, [{_, tag}]} ->
tag.language
_ ->
@und
end
end
def allow_listed_languages do
@paasaa_languages
|> Enum.map(fn lang ->
{__MODULE__.normalize(lang), lang}
end)
|> Enum.into(%{})
|> Map.take(@allow_listed_locales)
|> Map.values()
end
end