fix(serializer): make list encoding linear instead of quadratic

lib/jsonapi/serializer.ex

@@ -50,10 +50,9 @@ defmodule JSONAPI.Serializer do
   def encode_data(_view, nil, _conn, _query_includes, _options), do: {[], nil}
 
   def encode_data(view, data, conn, query_includes, options) when is_list(data) do
-    Enum.map_reduce(data, [], fn d, acc ->
-      {to_include, encoded_data} = encode_data(view, d, conn, query_includes, options)
-      {to_include, acc ++ [encoded_data]}
-    end)
+    data
+    |> Enum.map(&encode_data(view, &1, conn, query_includes, options))
+    |> Enum.unzip()
   end
 
   def encode_data(view, data, conn, query_includes, options) do

mix.exs

@@ -49,11 +49,11 @@ defmodule JSONAPI.Mixfile do
       {:plug, "~> 1.10"},
       {:jason, "~> 1.0", optional: true},
       {:ex_doc, "~> 0.20", only: ~w[dev test]a},
-      {:earmark, ">= 0.0.0", only: :dev},
       {:credo, "~> 1.4", only: [:dev, :test], runtime: false},
       {:phoenix, "~> 1.3", only: :test},
       {:dialyxir, "~> 1.1.0", only: [:dev, :test], runtime: false},
-      {:git_ops, "~> 2.2", only: ~w[dev test]a}
+      {:git_ops, "~> 2.2", only: ~w[dev test]a},
+      {:stream_data, "~> 1.1", only: :test}
     ]
   end
 

mix.lock

@@ -3,25 +3,25 @@
   "castore": {:hex, :castore, "1.0.11", "4bbd584741601eb658007339ea730b082cc61f3554cf2e8f39bf693a11b49073", [:mix], [], "hexpm", "e03990b4db988df56262852f20de0f659871c35154691427a5047f4967a16a62"},
   "credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"},
   "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"},
-  "earmark": {:hex, :earmark, "1.4.47", "7e7596b84fe4ebeb8751e14cbaeaf4d7a0237708f2ce43630cfd9065551f94ca", [:mix], [], "hexpm", "3e96bebea2c2d95f3b346a7ff22285bc68a99fbabdad9b655aa9c6be06c698f8"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.42", "f23d856f41919f17cd06a493923a722d87a2d684f143a1e663c04a2b93100682", [:mix], [], "hexpm", "6915b6ca369b5f7346636a2f41c6a6d78b5af419d61a611079189233358b8b8b"},
   "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"},
   "ex_doc": {:hex, :ex_doc, "0.36.1", "4197d034f93e0b89ec79fac56e226107824adcce8d2dd0a26f5ed3a95efc36b1", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d7d26a7cf965dacadcd48f9fa7b5953d7d0cfa3b44fa7a65514427da44eafd89"},
   "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"},
   "git_cli": {:hex, :git_cli, "0.3.0", "a5422f9b95c99483385b976f5d43f7e8233283a47cda13533d7c16131cb14df5", [:mix], [], "hexpm", "78cb952f4c86a41f4d3511f1d3ecb28edb268e3a7df278de2faa1bd4672eaf9b"},
   "git_ops": {:hex, :git_ops, "2.6.3", "38c6e381b8281b86e2911fa39bea4eab2d171c86d7428786566891efb73b68c3", [:mix], [{:git_cli, "~> 0.2", [hex: :git_cli, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a81cb6c6a2a026a4d48cb9a2e1dfca203f9283a3a70aa0c7bc171970c44f23f8"},
-  "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
+  "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
   "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},
   "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"},
   "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"},
-  "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
+  "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
-  "phoenix": {:hex, :phoenix, "1.7.18", "5310c21443514be44ed93c422e15870aef254cf1b3619e4f91538e7529d2b2e4", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "1797fcc82108442a66f2c77a643a62980f342bfeb63d6c9a515ab8294870004e"},
-  "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
+  "phoenix": {:hex, :phoenix, "1.7.24", "4cb76aed6d3f03878893769020e97c4394ee95b62b2b2d6313c20f66d7d37baa", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "a283a9d91517116166244fdd8ed8b405c142774dc39b4a4047d179cecca9c09f"},
+  "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.3.0", "03916bfbc31a5121945b3cfffe5aec647a5c97fe1dc172a319b94428562359c9", [:mix], [], "hexpm", "eec7be6e9cf02e2551d389b558402d6c637cd3973796326e7ba4bb03c6b2e91d"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},
-  "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"},
-  "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
-  "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
+  "plug": {:hex, :plug, "1.17.4", "5b9972f4ea7e71b9d235a91eadb2b2079941ef2b4547f9ee6a87c68d50ddca4a", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "39645c7e8cf5bd812d9df702d8c954e86e42c98791a49ea296c6433f69579a94"},
+  "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"},
+  "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"},
+  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
-  "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"},
+  "websock_adapter": {:hex, :websock_adapter, "0.5.9", "43dc3ba6d89ef5dec5b1d0a39698436a1e856d000d84bf31a3149862b01a287f", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5534d5c9adad3c18a0f58a9371220d75a803bf0b9a3d87e6fe072faaeed76a08"},
 }

test/jsonapi/serializer_property_test.exs

@@ -0,0 +1,120 @@
+defmodule JSONAPI.SerializerPropertyTest do
+  @moduledoc """
+  Properties of `JSONAPI.Serializer.serialize/5` for collections.
+
+  The list clause of `encode_data/5` must behave exactly like serializing
+  each element on its own and concatenating: same resources, same order,
+  same `included`. These pin that down for arbitrary lists, including the
+  degenerate cases (empty list, duplicate resources, duplicate includes)
+  that a hand-written example test tends to miss.
+  """
+
+  # Not async: the views read process-global Application env
+  # (:remove_links, :field_transformation, :namespace, ...) that other test
+  # modules mutate with put_env/delete_env.
+  use ExUnit.Case, async: false
+  use ExUnitProperties
+
+  alias JSONAPI.Serializer
+
+  @max_runs 25
+
+  defmodule UserView do
+    use JSONAPI.View, type: "user"
+
+    def fields, do: [:username]
+  end
+
+  defmodule CommentView do
+    use JSONAPI.View, type: "comment"
+
+    def fields, do: [:text]
+    def relationships, do: [user: {JSONAPI.SerializerPropertyTest.UserView, :include}]
+  end
+
+  defmodule PostView do
+    use JSONAPI.View, type: "post"
+
+    def fields, do: [:title, :body]
+    def meta(data, _conn), do: %{title_length: String.length(data.title)}
+
+    def relationships do
+      [
+        author: {JSONAPI.SerializerPropertyTest.UserView, :include},
+        comments: {JSONAPI.SerializerPropertyTest.CommentView, :include}
+      ]
+    end
+  end
+
+  # Small id space on purpose so that lists routinely contain repeated
+  # resources; `included` must still be deduplicated in first-seen order.
+  defp id, do: integer(1..20)
+
+  # Users and comments are drawn from small fixed pools rather than generated
+  # with random attributes, so that structurally identical includes recur and
+  # the dedup path in `flatten_included/1` is actually exercised.
+  @users for i <- 1..5, do: %{id: i, username: "user#{i}"}
+  @comments for i <- 1..8, u <- @users, do: %{id: i, text: "comment #{i}", user: u}
+
+  defp user, do: member_of(@users)
+  defp comment, do: member_of(@comments)
+
+  defp post do
+    gen all(
+          id <- id(),
+          title <- string(:printable, max_length: 20),
+          body <- string(:printable, max_length: 40),
+          author <- one_of([constant(nil), user()]),
+          comments <- list_of(comment(), max_length: 4)
+        ) do
+      %{id: id, title: title, body: body, author: author, comments: comments}
+    end
+  end
+
+  defp conn, do: Plug.Conn.fetch_query_params(%Plug.Conn{host: "example.com"})
+
+  defp expected_included(posts, conn) do
+    posts
+    |> Enum.map(&Serializer.serialize(PostView, &1, conn)[:included])
+    |> Serializer.flatten_included()
+  end
+
+  property "serializing a list yields each element's resource, in order" do
+    check all(posts <- list_of(post(), max_length: 30), max_runs: @max_runs) do
+      conn = conn()
+      encoded = Serializer.serialize(PostView, posts, conn)
+
+      expected = Enum.map(posts, &Serializer.serialize(PostView, &1, conn)[:data])
+
+      assert encoded[:data] == expected
+    end
+  end
+
+  property "included for a list is the deduplicated, in-order union of each element's included" do
+    check all(posts <- list_of(post(), max_length: 30), max_runs: @max_runs) do
+      conn = conn()
+      encoded = Serializer.serialize(PostView, posts, conn)
+
+      assert encoded[:included] == expected_included(posts, conn)
+    end
+  end
+
+  property "serializing a list is invariant to how it is split into chunks" do
+    check all(
+            posts <- list_of(post(), min_length: 1, max_length: 30),
+            chunk <- integer(1..30),
+            max_runs: @max_runs
+          ) do
+      conn = conn()
+      whole = Serializer.serialize(PostView, posts, conn)
+
+      chunked =
+        Enum.map(Enum.chunk_every(posts, chunk), &Serializer.serialize(PostView, &1, conn))
+
+      assert whole[:data] == Enum.flat_map(chunked, & &1[:data])
+
+      assert whole[:included] ==
+               chunked |> Enum.map(& &1[:included]) |> Serializer.flatten_included()
+    end
+  end
+end

test/jsonapi/serializer_test.exs

@@ -216,39 +216,45 @@ defmodule JSONAPI.SerializerTest do
     assert Enum.count(encoded[:included]) == 4
   end
 
-  test "serialize handles a list" do
-    data = %{
-      id: 1,
-      text: "Hello",
-      body: "Hello world",
-      author: %{id: 2, username: "jason"},
-      best_comments: [
-        %{id: 5, text: "greatest comment ever", user: %{id: 4, username: "jack"}},
-        %{id: 6, text: "not so great", user: %{id: 2, username: "jason"}}
-      ]
-    }
-
-    data_list = [data, data, data]
+  test "serialize handles a list, preserving order and deduplicating includes" do
+    data_list =
+      Enum.map(1..50, fn i ->
+        %{
+          id: i,
+          text: "post #{i}",
+          body: "body #{i}",
+          author: %{id: 100 + i, username: "u#{i}"},
+          best_comments: [
+            %{id: 200 + i, text: "comment #{i}", user: %{id: 4, username: "jack"}}
+          ]
+        }
+      end)
 
     conn = Plug.Conn.fetch_query_params(%Plug.Conn{})
 
     encoded = Serializer.serialize(PostView, data_list, conn)
 
-    assert Enum.count(encoded[:data]) == 3
+    assert Enum.count(encoded[:data]) == 50
 
-    Enum.each(encoded[:data], fn enc ->
+    Enum.zip(encoded[:data], data_list)
+    |> Enum.each(fn {enc, data} ->
       assert enc[:id] == PostView.id(data)
       assert enc[:type] == PostView.type()
-
-      attributes = enc[:attributes]
-      assert attributes[:text] == data[:text]
-      assert attributes[:body] == data[:body]
-
+      assert enc[:attributes][:text] == data[:text]
+      assert enc[:attributes][:body] == data[:body]
       assert enc[:links][:self] == PostView.url_for(data, conn)
       assert map_size(enc[:relationships]) == 2
     end)
 
-    assert Enum.count(encoded[:included]) == 4
+    # 50 authors + 50 comments + the one shared comment author (included once).
+    assert Enum.count(encoded[:included]) == 101
+
+    included_ids = fn type ->
+      encoded[:included] |> Enum.filter(&(&1[:type] == type)) |> Enum.map(& &1[:id])
+    end
+
+    assert included_ids.("comment") == Enum.map(1..50, &"#{200 + &1}")
+    assert included_ids.("user") == ["101", "4" | Enum.map(2..50, &"#{100 + &1}")]
   end
 
   test "serialize handles an empty relationship" do