refactor(pagi): use offical scrivener_ecto (#421)

* refactor(pagi): use offical scrivener_ecto

* refactor(pagi): fix matcher

* refactor(pagi): cleanup & naming adjust

* refactor(pagi): cleanup & naming adjust

Author: mydearxym

cover/excoveralls.json

@@ -134,7 +134,7 @@ defmodule GroupherServer.Accounts.Delegate.Achievements do
       |> where([e], e.user_id == ^user.id)
       |> join(:inner, [e], c in assoc(e, :community))
       |> select([e, c], c)
-      |> ORM.paginater(page: page, size: size)
+      |> ORM.paginator(page: page, size: size)
       |> done()
     end
   end

lib/groupher_server/accounts/delegates/achievements.ex

@@ -82,7 +82,7 @@ defmodule GroupherServer.Accounts.Delegate.CollectFolder do
       end)
 
     Repo.preload(folder.collects, article_preload)
-    |> ORM.embeds_paginater(filter)
+    |> ORM.embeds_paginator(filter)
     |> ORM.extract_articles()
     |> done()
   end
@@ -241,7 +241,7 @@ defmodule GroupherServer.Accounts.Delegate.CollectFolder do
     |> filter_thread_ifneed(filter)
     # delete thread in filter for now, otherwise it will crash querybuilder, because thread not exsit on CollectFolder
     |> QueryBuilder.filter_pack(filter |> Map.delete(:thread))
-    |> ORM.paginater(page: page, size: size)
+    |> ORM.paginator(page: page, size: size)
     |> done()
   end
 

lib/groupher_server/accounts/delegates/collect_folder.ex

@@ -159,7 +159,7 @@ defmodule GroupherServer.Accounts.Delegate.Fans do
     queryable
     |> select([uf, u], u)
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> done()
   end
 

lib/groupher_server/accounts/delegates/fans.ex

@@ -155,7 +155,7 @@ defmodule GroupherServer.Accounts.Delegate.Profile do
     |> join(:inner, [c], cc in assoc(c, :community))
     |> select([c, cc], cc)
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> sort_communities(user)
     |> done()
   end

lib/groupher_server/accounts/delegates/profile.ex

@@ -17,7 +17,7 @@ defmodule GroupherServer.Accounts.Delegate.Search do
   def search_users(%{name: name} = _args) do
     User
     |> where([c], ilike(c.nickname, ^"%#{name}%"))
-    |> ORM.paginater(page: 1, size: @search_items_count)
+    |> ORM.paginator(page: 1, size: @search_items_count)
     |> done()
   end
 end

lib/groupher_server/accounts/delegates/search.ex

@@ -41,7 +41,7 @@ defmodule GroupherServer.Accounts.Delegate.UpvotedArticles do
     query
     |> where(^where_query)
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> ORM.extract_articles()
     |> done()
   end

lib/groupher_server/accounts/delegates/upvoted_articles.ex

@@ -25,7 +25,7 @@ defmodule GroupherServer.Billing.Delegate.CURD do
     with {:ok, user} <- ORM.find(User, user_id) do
       BillRecord
       |> where([r], r.user_id == ^user.id)
-      |> ORM.paginater(page: page, size: size)
+      |> ORM.paginator(page: page, size: size)
       |> done()
     end
   end

lib/groupher_server/billing/delegates/curd.ex

@@ -201,13 +201,13 @@ defmodule GroupherServer.CMS.Delegate.AbuseReport do
 
     query
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> reports_formater(thread)
     |> done()
   end
 
   defp do_paged_reports(query, %{page: page, size: size}) do
-    query |> ORM.paginater(~m(page size)a) |> done()
+    query |> ORM.paginator(~m(page size)a) |> done()
   end
 
   defp create_report(type, content_id, reason, attr, %User{} = user) do

lib/groupher_server/cms/delegates/abuse_report.ex

@@ -100,7 +100,7 @@ defmodule GroupherServer.CMS.Delegate.ArticleCURD do
       info.model
       |> QueryBuilder.domain_query(filter)
       |> QueryBuilder.filter_pack(Map.merge(filter, %{mark_delete: false}))
-      |> ORM.paginater(~m(page size)a)
+      |> ORM.paginator(~m(page size)a)
       |> add_pin_articles_ifneed(info.model, filter)
       |> done()
     end
@@ -126,7 +126,7 @@ defmodule GroupherServer.CMS.Delegate.ArticleCURD do
       |> where([article, author], author.user_id == ^user.id)
       |> select([article, author], article)
       |> QueryBuilder.filter_pack(filter)
-      |> ORM.paginater(~m(page size)a)
+      |> ORM.paginator(~m(page size)a)
       |> mark_viewer_emotion_states(user)
       |> mark_viewer_has_states(user)
       |> done()

lib/groupher_server/cms/delegates/article_curd.ex

@@ -147,15 +147,15 @@ defmodule GroupherServer.CMS.Delegate.ArticleTag do
   def paged_article_tags(%{page: page, size: size} = filter) do
     ArticleTag
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> done()
   end
 
   # if no page info given, load 100 tags by default
   def paged_article_tags(filter) do
     ArticleTag
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(%{page: 1, size: 100})
+    |> ORM.paginator(%{page: 1, size: 100})
     |> done()
   end
 

lib/groupher_server/cms/delegates/article_tag.ex

@@ -38,7 +38,7 @@ defmodule GroupherServer.CMS.Delegate.CitedArtiment do
     CitedArtiment
     |> where([c], c.cited_by_id == ^cited_by_id and c.cited_by_type == ^cited_by_type)
     |> QueryBuilder.filter_pack(Map.merge(filter, %{sort: :asc_inserted}))
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> extract_contents
     |> done
   end

lib/groupher_server/cms/delegates/cited_artiment.ex

@@ -62,7 +62,7 @@ defmodule GroupherServer.CMS.Delegate.CommentCurd do
     |> join(:inner, [comment], author in assoc(comment, :author))
     |> where([comment, author], author.id == ^user_id)
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> ORM.extract_and_assign_article()
     |> done()
   end
@@ -81,7 +81,7 @@ defmodule GroupherServer.CMS.Delegate.CommentCurd do
     |> where([comment, author], comment.thread == ^thread)
     |> where([comment, author], author.id == ^user_id)
     |> QueryBuilder.filter_pack(filter)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> ORM.extract_and_assign_article()
     |> done()
   end
@@ -116,10 +116,12 @@ defmodule GroupherServer.CMS.Delegate.CommentCurd do
       |> QueryBuilder.filter_pack(Map.merge(filters, %{sort: :desc_inserted}))
       |> join(:inner, [c], a in assoc(c, :author))
       |> distinct([c, a], a.id)
+      # group_by
       |> group_by([c, a], a.id)
       |> group_by([c, a], c.inserted_at)
+      |> group_by([c, a], c.id)
       |> select([c, a], a)
-      |> ORM.paginater(~m(page size)a)
+      |> ORM.paginator(~m(page size)a)
       |> done()
     end
   end
@@ -331,7 +333,7 @@ defmodule GroupherServer.CMS.Delegate.CommentCurd do
       |> where(^thread_query)
       |> where(^where_query)
       |> QueryBuilder.filter_pack(Map.merge(filters, %{sort: sort}))
-      |> ORM.paginater(~m(page size)a)
+      |> ORM.paginator(~m(page size)a)
       |> add_pinned_comments_ifneed(thread, article_id, filters)
       |> mark_viewer_emotion_states(user, :comment)
       |> mark_viewer_has_upvoted(user)
@@ -348,7 +350,7 @@ defmodule GroupherServer.CMS.Delegate.CommentCurd do
     query
     |> where(^where_query)
     |> QueryBuilder.filter_pack(filters)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> mark_viewer_emotion_states(user, :comment)
     |> mark_viewer_has_upvoted(user)
     |> done()

lib/groupher_server/cms/delegates/comment_curd.ex

@@ -233,7 +233,7 @@ defmodule GroupherServer.CMS.Delegate.CommunityCURD do
       result =
         ArticleTag
         |> where([t], t.community_id == ^community.id)
-        |> ORM.paginater(page: 1, size: 1)
+        |> ORM.paginator(page: 1, size: 1)
 
       {:ok, result.total_count}
     end
@@ -262,7 +262,7 @@ defmodule GroupherServer.CMS.Delegate.CommunityCURD do
     queryable
     |> where([c], c.community_id == ^id)
     |> QueryBuilder.load_inner_users(filters)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> done()
   end
 
@@ -277,7 +277,7 @@ defmodule GroupherServer.CMS.Delegate.CommunityCURD do
     |> join(:inner, [member], u in assoc(member, :user))
     |> select([member, c, u], u)
     |> QueryBuilder.filter_pack(filters)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> done()
   end
 end

lib/groupher_server/cms/delegates/community_curd.ex

@@ -156,7 +156,7 @@ defmodule GroupherServer.CMS.Delegate.Helper do
       queryable
       |> where([u], field(u, ^info.foreign_key) == ^article_id)
       |> QueryBuilder.load_inner_users(filter)
-      |> ORM.paginater(~m(page size)a)
+      |> ORM.paginator(~m(page size)a)
       |> done()
     end
   end

lib/groupher_server/cms/delegates/helper.ex

@@ -18,7 +18,7 @@ defmodule GroupherServer.CMS.Delegate.Search do
   def search_communities(title) do
     Community
     |> where([c], ilike(c.title, ^"%#{title}%") or ilike(c.raw, ^"%#{title}%"))
-    |> ORM.paginater(page: 1, size: @search_items_count)
+    |> ORM.paginator(page: 1, size: @search_items_count)
     |> done()
   end
 
@@ -30,7 +30,7 @@ defmodule GroupherServer.CMS.Delegate.Search do
       info.model
       # |> where([c], ilike(c.title, ^"%#{title}%") or ilike(c.digest, ^"%#{title}%"))
       |> where([c], ilike(c.title, ^"%#{title}%"))
-      |> ORM.paginater(page: 1, size: @search_items_count)
+      |> ORM.paginator(page: 1, size: @search_items_count)
       |> done()
     end
   end

lib/groupher_server/cms/delegates/search.ex

@@ -233,7 +233,7 @@ defmodule GroupherServer.CMS.Delegate.Seeds do
 
     CMS.Thread
     |> where([t], t.raw in ^thread_titles)
-    |> ORM.paginater(page: 1, size: 30)
+    |> ORM.paginator(page: 1, size: 30)
     |> done()
   end
 
@@ -248,7 +248,7 @@ defmodule GroupherServer.CMS.Delegate.Seeds do
 
     CMS.Thread
     |> where([t], t.raw in @city_threads)
-    |> ORM.paginater(page: 1, size: 10)
+    |> ORM.paginator(page: 1, size: 10)
     |> done()
   end
 
@@ -266,7 +266,7 @@ defmodule GroupherServer.CMS.Delegate.Seeds do
 
     CMS.Thread
     |> where([t], t.raw in @home_threads)
-    |> ORM.paginater(page: 1, size: 10)
+    |> ORM.paginator(page: 1, size: 10)
     |> done()
   end
 

lib/groupher_server/cms/delegates/seeds.ex

@@ -74,7 +74,7 @@ defmodule GroupherServer.Delivery.Delegate.Mention do
 
     Mention
     |> where([m], m.to_user_id == ^user.id and m.read == ^read)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> extract_mentions
     |> done()
   end

lib/groupher_server/delivery/delegates/mention.ex

@@ -84,7 +84,7 @@ defmodule GroupherServer.Delivery.Delegate.Notification do
 
     Notification
     |> where([n], n.user_id == ^user.id and n.read == ^read)
-    |> ORM.paginater(~m(page size)a)
+    |> ORM.paginator(~m(page size)a)
     |> cut_from_users_ifneed
     |> done
   end

lib/groupher_server/delivery/delegates/notification.ex

@@ -18,7 +18,7 @@ defmodule GroupherServer.Statistics.Delegate.Geo do
   def list_cities_info do
     UserGeoInfo
     |> where([g], g.value > 0)
-    |> ORM.paginater(page: 1, size: 300)
+    |> ORM.paginator(page: 1, size: 300)
     |> done()
   end
 end

lib/groupher_server/statistics/delegates/geo.ex

@@ -15,19 +15,21 @@ defmodule Helper.ORM do
   @article_threads get_config(:article, :threads)
 
   @doc """
-  a wrap for paginate request
+  offset-limit based pagination
+  total_count is a personal-taste naming convert
   """
-  def paginater(queryable, page: page, size: size) do
-    queryable |> Repo.paginate(page: page, page_size: size)
-  end
+  def paginator(queryable, page: page, size: size), do: do_pagi(queryable, page, size)
+  def paginator(queryable, ~m(page size)a), do: do_pagi(queryable, page, size)
 
-  def paginater(queryable, ~m(page size)a) do
-    queryable |> Repo.paginate(page: page, page_size: size)
+  defp do_pagi(queryable, page, size) do
+    result = queryable |> Repo.paginate(page: page, page_size: size)
+    total_count = result.total_entries
+    result |> Map.put(:total_count, total_count) |> Map.drop([:total_entries])
   end
 
   # NOTE: should have limit length for list, otherwise it will cause mem issues
   @doc "simu paginator in normal list, used for embeds_many etc"
-  def embeds_paginater(list, %{page: page, size: size} = _filter) when is_list(list) do
+  def embeds_paginator(list, %{page: page, size: size} = _filter) when is_list(list) do
     chunked_list = Enum.chunk_every(list, size)
 
     entries = chunked_list |> Enum.at(page - 1)
@@ -84,7 +86,7 @@ defmodule Helper.ORM do
   def find_all(queryable, %{page: page, size: size} = filter) do
     queryable
     |> QueryBuilder.filter_pack(filter)
-    |> paginater(page: page, size: size)
+    |> paginator(page: page, size: size)
     |> done()
   end
 

lib/helper/orm.ex

@@ -74,9 +74,9 @@ defmodule GroupherServer.Mixfile do
       {:poison, "~> 3.1"},
       # for fake data in test env
       {:faker, "~> 0.9"},
-      {:scrivener_ecto,
-       git: "https://github.com/mastani-stack/scrivener_ecto", branch: "dev", override: true},
-      # {:scrivener_ecto, "~> 2.0.0"},
+      {:scrivener_ecto, "~> 2.7.0"},
+      # cursor based pagination
+      {:paginator, "~> 1.0.4"},
       {:guardian, "~> 2.0"},
       {:timex, "~> 3.7.5"},
       {:dataloader, "~> 1.0.7"},

mix.exs

@@ -61,6 +61,7 @@
   "nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"},
   "open_graph": {:hex, :open_graph, "0.0.4", "0790882d5735286abc8391b37d2707613ecd74ebdc5340be664d3df345cd8cae", [:mix], [{:httpoison, "~> 1.5", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "a22a8ddaf9f881af38d519ced463627c37a8dea49a35cc27efdb9f2b839b9049"},
+  "paginator": {:hex, :paginator, "1.0.4", "471e8e59e7a08541c5b7d2ea7166308c8e05ceb412fb444f3bdf320f5d793260", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "9d79d21c0192d8d1fc562e5c99b5ebf7e0bb443c671e256b41fd709170730b91"},
   "pangu": {:hex, :pangu, "0.1.0", "9f9e06418212017bb076034865462e92effed4e5b18c063d4448c186f12a6f0a", [:mix], [], "hexpm", "2634cc2463421757aca0a76665de83940d4fda12f8ed316ae929bb29f64d06c5"},
   "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
   "phoenix": {:hex, :phoenix, "1.5.9", "a6368d36cfd59d917b37c44386e01315bc89f7609a10a45a22f47c007edf2597", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7e4bce20a67c012f1fbb0af90e5da49fa7bf0d34e3a067795703b74aef75427d"},
@@ -80,7 +81,7 @@
   "recase": {:hex, :recase, "0.7.0", "3f2f719f0886c7a3b7fe469058ec539cb7bbe0023604ae3bce920e186305e5ae", [:mix], [], "hexpm", "36f5756a9f552f4a94b54a695870e32f4e72d5fad9c25e61bc4a3151c08a4e0c"},
   "rihanna": {:hex, :rihanna, "1.3.5", "5f5e6c5b1e514978a29a6791f338f4bb963401959fc212bd18d4a2c92d79a7a4", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.13.3", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "fa1918c2ab63c8ada9a23ad6fe03cd181378739f0ff10741b45d0bcb50003c74"},
   "scrivener": {:hex, :scrivener, "2.5.0", "e1f78c62b6806d91cc9c4778deef1ea4e80aa9fadfce2c16831afe0468cc8a2c", [:mix], [], "hexpm", "c3e484da6bb7084b5a24c7e38a8ca09310d5fbf5241db05f625fb8af557ef667"},
-  "scrivener_ecto": {:git, "https://github.com/mastani-stack/scrivener_ecto", "a6ebbbe9585e9829219ed0f73a3afcb4edde3558", [branch: "dev"]},
+  "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"},
   "sentry": {:hex, :sentry, "7.1.0", "546729ea0be4a3f593b456fe77a2cf5537e390fbe87c191424557dae8c2bd760", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "6e439c2b93a3d1e049eeaa7910d9ef4ff255b107b2192c25fe50e5f2cd57d2d3"},
   "short_maps": {:hex, :short_maps, "0.1.2", "a7c2bfd91179cdbdfe90e74a023992335d116982fa672612c74776b2e9257a7b", [:mix], [], "hexpm", "8147a645e2f518c857178f10361e98c4488bdbaa4d96a280cb13c1d3c0ce7142"},
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},

mix.lock

@@ -373,6 +373,7 @@ defmodule GroupherServer.Test.CMS.Comments.MeetupComment do
   end
 
   describe "paged article comments" do
+    @tag :wip
     test "can load paged comments participants of a article", ~m(user meetup)a do
       total_count = 30
       page_size = 10

test/groupher_server/cms/comments/meetup_comment_test.exs

@@ -68,8 +68,6 @@ defmodule GroupherServer.Test.Mutation.Sink.BlogSink do
       assert not blog.meta.is_sinked
     end
 
-    :wip2
-
     test "unauth user undo sink a blog fails", ~m(guest_conn community blog)a do
       variables = %{id: blog.id, communityId: community.id}
 

test/groupher_server_web/mutation/cms/sink/blog_sink_test.exs

@@ -68,8 +68,6 @@ defmodule GroupherServer.Test.Mutation.Sink.DrinkSink do
       assert not drink.meta.is_sinked
     end
 
-    :wip2
-
     test "unauth user undo sink a drink fails", ~m(guest_conn community drink)a do
       variables = %{id: drink.id, communityId: community.id}
 

test/groupher_server_web/mutation/cms/sink/drink_sink_test.exs

@@ -68,8 +68,6 @@ defmodule GroupherServer.Test.Mutation.Sink.GuideSink do
       assert not guide.meta.is_sinked
     end
 
-    :wip2
-
     test "unauth user undo sink a guide fails", ~m(guest_conn community guide)a do
       variables = %{id: guide.id, communityId: community.id}