Track usage of media files and add a job to clean them

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

lib/mix/tasks/mobilizon/common.ex

@@ -47,12 +47,14 @@ defmodule Mix.Tasks.Mobilizon.Common do
       else: shell_prompt(message, "Continue?") in ~w(Yn Y y)
   end
 
+  @spec shell_info(String.t()) :: :ok
   def shell_info(message) do
     if mix_shell?(),
       do: Mix.shell().info(message),
       else: IO.puts(message)
   end
 
+  @spec shell_error(String.t()) :: :ok
   def shell_error(message) do
     if mix_shell?(),
       do: Mix.shell().error(message),

lib/mix/tasks/mobilizon/maintenance.ex

@@ -0,0 +1,23 @@
+defmodule Mix.Tasks.Mobilizon.Maintenance do
+  @moduledoc """
+  Tasks to maintain mobilizon
+  """
+
+  use Mix.Task
+
+  alias Mix.Tasks
+  import Mix.Tasks.Mobilizon.Common
+
+  @shortdoc "List common Mobilizon maintenance tasks"
+
+  @impl Mix.Task
+  def run(_) do
+    shell_info("\nAvailable tasks:")
+
+    if mix_shell?() do
+      Tasks.Help.run(["--search", "mobilizon.maintenance."])
+    else
+      show_subtasks_for_module(__MODULE__)
+    end
+  end
+end

lib/mix/tasks/mobilizon/maintenance/fix_unattached_media_in_body.ex

@@ -0,0 +1,107 @@
+defmodule Mix.Tasks.Mobilizon.Maintenance.FixUnattachedMediaInBody do
+  @moduledoc """
+  Task to reattach media files that were added in event, post or comment bodies without being attached to their entities.
+
+  This task should only be run once.
+  """
+  use Mix.Task
+
+  import Mix.Tasks.Mobilizon.Common
+  alias Mobilizon.{Discussions, Events, Medias, Posts}
+  alias Mobilizon.Discussions.Comment
+  alias Mobilizon.Events.Event
+  alias Mobilizon.Posts.Post
+  alias Mobilizon.Storage.Repo
+  require Logger
+
+  @preferred_cli_env "prod"
+
+  # TODO: Remove me in Mobilizon 1.2
+
+  @shortdoc "Reattaches inline media from events and posts"
+  def run([]) do
+    start_mobilizon()
+
+    shell_info("Going to extract pictures from events")
+    extract_inline_pictures_from_bodies(Event)
+    shell_info("Going to extract pictures from posts")
+    extract_inline_pictures_from_bodies(Post)
+    shell_info("Going to extract pictures from comments")
+    extract_inline_pictures_from_bodies(Comment)
+  end
+
+  defp extract_inline_pictures_from_bodies(entity) do
+    Repo.transaction(
+      fn ->
+        entity
+        |> Repo.stream()
+        |> Stream.map(&extract_pictures(&1))
+        |> Stream.map(fn {entity, pics} -> save_entity(entity, pics) end)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
+  end
+
+  defp extract_pictures(entity) do
+    extracted_pictures = entity |> get_body() |> parse_body() |> get_media_entities_from_urls()
+
+    attached_picture = entity |> get_picture() |> get_media_entity_from_media_id()
+    attached_pictures = [attached_picture] |> Enum.filter(& &1)
+
+    {entity, extracted_pictures ++ attached_pictures}
+  end
+
+  defp get_body(%Event{description: description}), do: description
+  defp get_body(%Post{body: body}), do: body
+  defp get_body(%Comment{text: text}), do: text
+
+  defp get_picture(%Event{picture_id: picture_id}), do: picture_id
+  defp get_picture(%Post{picture_id: picture_id}), do: picture_id
+  defp get_picture(%Comment{}), do: nil
+
+  defp parse_body(nil), do: []
+
+  defp parse_body(body) do
+    with res <- Regex.scan(~r/<img\s[^>]*?src\s*=\s*['\"]([^'\"]*?)['\"][^>]*?>/, body),
+         res <- Enum.map(res, fn [_, res] -> res end) do
+      res
+    end
+  end
+
+  defp get_media_entities_from_urls(media_urls) do
+    media_urls
+    |> Enum.map(fn media_url ->
+      # We prefer orphan media, but fallback on already attached media just in case
+      Medias.get_unattached_media_by_url(media_url) || Medias.get_media_by_url(media_url)
+    end)
+    |> Enum.filter(& &1)
+  end
+
+  defp get_media_entity_from_media_id(nil), do: nil
+
+  defp get_media_entity_from_media_id(media_id) do
+    Medias.get_media(media_id)
+  end
+
+  defp save_entity(%Event{} = _event, []), do: :ok
+
+  defp save_entity(%Event{} = event, media) do
+    event = Repo.preload(event, [:contacts, :media])
+    Events.update_event(event, %{media: media})
+  end
+
+  defp save_entity(%Post{} = _post, []), do: :ok
+
+  defp save_entity(%Post{} = post, media) do
+    post = Repo.preload(post, [:media])
+    Posts.update_post(post, %{media: media})
+  end
+
+  defp save_entity(%Comment{} = _comment, []), do: :ok
+
+  defp save_entity(%Comment{} = comment, media) do
+    comment = Repo.preload(comment, [:media])
+    Discussions.update_comment(comment, %{media: media})
+  end
+end

lib/mix/tasks/mobilizon/media.ex

@@ -0,0 +1,23 @@
+defmodule Mix.Tasks.Mobilizon.Media do
+  @moduledoc """
+  Tasks to manage media
+  """
+
+  use Mix.Task
+
+  alias Mix.Tasks
+  import Mix.Tasks.Mobilizon.Common
+
+  @shortdoc "Manages Mobilizon media"
+
+  @impl Mix.Task
+  def run(_) do
+    shell_info("\nAvailable tasks:")
+
+    if mix_shell?() do
+      Tasks.Help.run(["--search", "mobilizon.media."])
+    else
+      show_subtasks_for_module(__MODULE__)
+    end
+  end
+end

lib/mix/tasks/mobilizon/media/clean_orphan.ex

@@ -0,0 +1,87 @@
+defmodule Mix.Tasks.Mobilizon.Media.CleanOrphan do
+  @moduledoc """
+  Task to accept an instance follow request
+  """
+  use Mix.Task
+  import Mix.Tasks.Mobilizon.Common
+  alias Mobilizon.Service.CleanOrphanMedia
+
+  @shortdoc "Clean orphan media"
+
+  @grace_period Mobilizon.Config.get([:instance, :orphan_upload_grace_period_hours], 48)
+
+  @impl Mix.Task
+  def run(options) do
+    {options, [], []} =
+      OptionParser.parse(
+        options,
+        strict: [
+          dry_run: :boolean,
+          days: :integer,
+          verbose: :boolean
+        ],
+        aliases: [
+          d: :days,
+          v: :verbose
+        ]
+      )
+
+    dry_run = Keyword.get(options, :dry_run, false)
+    grace_period = Keyword.get(options, :days)
+    grace_period = if is_nil(grace_period), do: @grace_period, else: grace_period * 24
+    verbose = Keyword.get(options, :verbose, false)
+
+    start_mobilizon()
+
+    case CleanOrphanMedia.clean(dry_run: dry_run, grace_period: grace_period) do
+      {:ok, medias} ->
+        if length(medias) > 0 do
+          if dry_run or verbose do
+            details(medias, dry_run, verbose)
+          end
+
+          result(dry_run, length(medias))
+        else
+          empty_result(dry_run)
+        end
+
+        :ok
+
+      _err ->
+        shell_error("Error while cleaning orphan media files")
+    end
+  end
+
+  @spec details(list(Media.t()), boolean(), boolean()) :: :ok
+  defp details(medias, dry_run, verbose) do
+    cond do
+      dry_run ->
+        shell_info("List of files that would have been deleted")
+
+      verbose ->
+        shell_info("List of files that have been deleted")
+    end
+
+    Enum.each(medias, fn media ->
+      shell_info("ID: #{media.id}, Actor: #{media.actor_id}, URL: #{media.file.url}")
+    end)
+  end
+
+  @spec result(boolean(), boolean()) :: :ok
+  defp result(dry_run, nb_medias) do
+    if dry_run do
+      shell_info("#{nb_medias} files would have been deleted")
+    else
+      shell_info("#{nb_medias} files have been deleted")
+    end
+  end
+
+  @spec empty_result(boolean()) :: :ok
+  defp empty_result(dry_run) do
+    if dry_run do
+      shell_info("No files would have been deleted")
+    else
+      shell_info("No files were deleted")
+    end
+  end
+end