File tree 1 file changed +5
-6
lines changed
1 file changed +5
-6
lines changed Original file line number Diff line number Diff line change @@ -117,8 +117,9 @@ def __init__(
117
117
self ._items = {}
118
118
self .dirs = None
119
119
self .fs = fsspec .filesystem ("file" ) if fs is None else fs
120
- with self .fs .open ("/" .join ([self .root , ".zmetadata" ]), "rb" ) as f :
121
- self ._items [".zmetadata" ] = f .read ()
120
+ self ._items [".zmetadata" ] = self .fs .cat_file (
121
+ "/" .join ([self .root , ".zmetadata" ])
122
+ )
122
123
met = json .loads (self ._items [".zmetadata" ])
123
124
self .record_size = met ["record_size" ]
124
125
self .zmetadata = met ["metadata" ]
@@ -131,10 +132,8 @@ def __init__(
131
132
def open_refs (field , record ):
132
133
"""cached parquet file loader"""
133
134
path = self .url .format (field = field , record = record )
134
- with self .fs .open (path ) as f :
135
- # TODO: since all we do is iterate, is arrow without pandas
136
- # better here?
137
- df = self .pd .read_parquet (f , engine = "fastparquet" )
135
+ data = io .BytesIO (self .fs .cat_file (path ))
136
+ df = self .pd .read_parquet (data , engine = "fastparquet" )
138
137
refs = {c : df [c ].values for c in df .columns }
139
138
return refs
140
139
You can’t perform that action at this time.
0 commit comments