BUG: Fix json_normalize throwing TypeError when record_path has a sequence of dicts along its path #22706

Author: vuminhle

doc/source/whatsnew/v0.23.5.txt

@@ -52,3 +52,5 @@ Bug Fixes
 **I/O**
 
 - Bug in :func:`read_csv` that caused it to raise ``OverflowError`` when trying to use 'inf' as ``na_value`` with integer index column (:issue:`17128`)
+- Bug in :func:`json_normalize` that caused it to raise ``TypeError`` when ``record_path`` has a sequence of dicts along its path (:issue:`22706`)
+-

pandas/io/json/normalize.py

@@ -224,6 +224,30 @@ def _pull_field(js, spec):
         sep = str(sep)
     meta_keys = [sep.join(val) for val in meta]
 
+    def _extract(obj, path, seen_meta, level):
+        recs = _pull_field(obj, path[0])
+
+        # For repeating the metadata later
+        lengths.append(len(recs))
+
+        for val, key in zip(meta, meta_keys):
+            if level + 1 > len(val):
+                meta_val = seen_meta[key]
+            else:
+                try:
+                    meta_val = _pull_field(obj, val[level:])
+                except KeyError as e:
+                    if errors == 'ignore':
+                        meta_val = np.nan
+                    else:
+                        raise KeyError("Try running with "
+                                       "errors='ignore' as key "
+                                       "{err} is not always present"
+                                       .format(err=e))
+            meta_vals[key].append(meta_val)
+
+        records.extend(recs)
+
     def _recursive_extract(data, path, seen_meta, level=0):
         if len(path) > 1:
             for obj in data:
@@ -233,31 +257,11 @@ def _recursive_extract(data, path, seen_meta, level=0):
 
                 _recursive_extract(obj[path[0]], path[1:],
                                    seen_meta, level=level + 1)
-        else:
+        elif isinstance(data, list):
             for obj in data:
-                recs = _pull_field(obj, path[0])
-
-                # For repeating the metadata later
-                lengths.append(len(recs))
-
-                for val, key in zip(meta, meta_keys):
-                    if level + 1 > len(val):
-                        meta_val = seen_meta[key]
-                    else:
-                        try:
-                            meta_val = _pull_field(obj, val[level:])
-                        except KeyError as e:
-                            if errors == 'ignore':
-                                meta_val = np.nan
-                            else:
-                                raise \
-                                    KeyError("Try running with "
-                                             "errors='ignore' as key "
-                                             "{err} is not always present"
-                                             .format(err=e))
-                    meta_vals[key].append(meta_val)
-
-                records.extend(recs)
+                _extract(obj, path, seen_meta, level)
+        else:
+            _extract(data, path, seen_meta, level)
 
     _recursive_extract(data, record_path, {}, level=0)
 

pandas/tests/io/json/test_normalize.py

@@ -129,6 +129,21 @@ def test_value_array_record_prefix(self):
         expected = DataFrame([[1], [2]], columns=['Prefix.0'])
         tm.assert_frame_equal(result, expected)
 
+    def test_nested_object_record_path(self):
+        # GH 22706
+        data = {'state': 'Florida',
+                'info': {
+                    'governor': 'Rick Scott',
+                    'counties': [{'name': 'Dade', 'population': 12345},
+                                 {'name': 'Broward', 'population': 40000},
+                                 {'name': 'Palm Beach', 'population': 60000}]}}
+        result = json_normalize(data, record_path=["info", "counties"])
+        expected = DataFrame([['Dade', 12345],
+                              ['Broward', 40000],
+                              ['Palm Beach', 60000]],
+                             columns=['name', 'population'])
+        tm.assert_frame_equal(result, expected)
+
     def test_more_deeply_nested(self, deep_nested):
 
         result = json_normalize(deep_nested, ['states', 'cities'],