BUG: Fix json_normalize throwing TypeError when record_path has a sequence of dicts #22706

doc/source/whatsnew/v0.24.0.rst

@@ -1387,7 +1387,6 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form
 - :func:`read_sas()` will correctly parse sas7bdat files with data page types having also bit 7 set (so page type is 128 + 256 = 384) (:issue:`16615`)
 - Bug in :meth:`detect_client_encoding` where potential ``IOError`` goes unhandled when importing in a mod_wsgi process due to restricted access to stdout. (:issue:`21552`)
 - Bug in :func:`to_html()` with ``index=False`` misses truncation indicators (...) on truncated DataFrame (:issue:`15019`, :issue:`22783`)
-- Bug in :func:`DataFrame.to_string()` that broke column alignment when ``index=False`` and width of first column's values is greater than the width of first column's header (:issue:`16839`, :issue:`13032`)
 - Bug in :func:`DataFrame.to_string()` that caused representations of :class:`DataFrame` to not take up the whole window (:issue:`22984`)
 - Bug in :func:`DataFrame.to_csv` where a single level MultiIndex incorrectly wrote a tuple. Now just the value of the index is written (:issue:`19589`).
 - Bug in :meth:`HDFStore.append` when appending a :class:`DataFrame` with an empty string column and ``min_itemsize`` < 8 (:issue:`12242`)
@@ -1399,6 +1398,7 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form
 - Bug in :meth:`read_excel()` in which extraneous header names were extracted, even though none were specified (:issue:`11733`)
 - Bug in :meth:`read_excel()` in which ``index_col=None`` was not being respected and parsing index columns anyway (:issue:`20480`)
 - Bug in :meth:`read_excel()` in which ``usecols`` was not being validated for proper column names when passed in as a string (:issue:`20480`)
+- Bug in :func:`json_normalize` that caused it to raise ``TypeError`` when two consecutive elements of ``record_path`` are dicts (:issue:`22706`)
 
 Plotting
 ^^^^^^^^

pandas/io/json/normalize.py

@@ -225,6 +225,44 @@ def _pull_field(js, spec):
         sep = str(sep)
     meta_keys = [sep.join(val) for val in meta]
 
+    def _extract(obj, field, seen_meta, level):
+        """
+        Extract field from obj and update `records`, `lengths`, `meta_vals`.
+
+        Parameters
+        ----------
+        obj : dict
+            Unserialized dict
+        field : str
+            The field to extract from obj
+        seen_meta : dict
+            The dict of meta values that have been visited
+        level: int
+            The current level
+        """
+        recs = _pull_field(obj, field)
+
+        # For repeating the metadata later
+        lengths.append(len(recs))
+
+        for val, key in zip(meta, meta_keys):
+            if level >= len(val):
+                meta_val = seen_meta[key]
+            else:
+                try:
+                    meta_val = _pull_field(obj, val[level:])
+                except KeyError as e:
+                    if errors == 'ignore':
+                        meta_val = np.nan
+                    else:
+                        raise KeyError("Try running with "
+                                       "errors='ignore' as key "
+                                       "{err} is not always present"
+                                       .format(err=e))
+            meta_vals[key].append(meta_val)
+
+        records.extend(recs)
+
     def _recursive_extract(data, path, seen_meta, level=0):
         if len(path) > 1:
             for obj in data:
@@ -234,30 +272,11 @@ def _recursive_extract(data, path, seen_meta, level=0):
 
                 _recursive_extract(obj[path[0]], path[1:],
                                    seen_meta, level=level + 1)
-        else:
+        elif isinstance(data, list):
             for obj in data:
-                recs = _pull_field(obj, path[0])
-
-                # For repeating the metadata later
-                lengths.append(len(recs))
-
-                for val, key in zip(meta, meta_keys):
-                    if level + 1 > len(val):
-                        meta_val = seen_meta[key]
-                    else:
-                        try:
-                            meta_val = _pull_field(obj, val[level:])
-                        except KeyError as e:
-                            if errors == 'ignore':
-                                meta_val = np.nan
-                            else:
-                                raise KeyError("Try running with "
-                                               "errors='ignore' as key "
-                                               "{err} is not always present"
-                                               .format(err=e))
-                    meta_vals[key].append(meta_val)
-
-                records.extend(recs)
+                _extract(obj, path[0], seen_meta, level)
+        else:
+            _extract(data, path[0], seen_meta, level)
 
     _recursive_extract(data, record_path, {}, level=0)
 

pandas/tests/io/json/test_normalize.py

@@ -129,6 +129,21 @@ def test_value_array_record_prefix(self):
         expected = DataFrame([[1], [2]], columns=['Prefix.0'])
         tm.assert_frame_equal(result, expected)
 
+    def test_nested_object_record_path(self):
+        # GH 22706
+        data = {'state': 'Florida',
+                'info': {
+                    'governor': 'Rick Scott',
+                    'counties': [{'name': 'Dade', 'population': 12345},
+                                 {'name': 'Broward', 'population': 40000},
+                                 {'name': 'Palm Beach', 'population': 60000}]}}
+        result = json_normalize(data, record_path=["info", "counties"])
+        expected = DataFrame([['Dade', 12345],
+                              ['Broward', 40000],
+                              ['Palm Beach', 60000]],
+                             columns=['name', 'population'])
+        tm.assert_frame_equal(result, expected)
+
     def test_more_deeply_nested(self, deep_nested):
 
         result = json_normalize(deep_nested, ['states', 'cities'],