Skip to content

Commit 3aea012

Browse files
get chunked mysql rows as lists of tuples
1 parent ee9da4a commit 3aea012

File tree

3 files changed

+18
-0
lines changed

3 files changed

+18
-0
lines changed

doc/source/whatsnew/v0.18.0.txt

+1
Original file line numberDiff line numberDiff line change
@@ -238,3 +238,4 @@ Bug Fixes
238238
- Bug in ``df.replace`` while replacing value in mixed dtype ``Dataframe`` (:issue:`11698`)
239239

240240
- Bug in ``read_excel`` failing to read any non-empty sheets when empty sheets exist and ``sheetname=None`` (:issue:`11711`)
241+
- Bug in ``read_sql`` with pymysql connections failing to return chunked data (:issue:`11522`)

pandas/io/sql.py

+2
Original file line numberDiff line numberDiff line change
@@ -1556,6 +1556,8 @@ def _query_iterator(cursor, chunksize, columns, index_col=None,
15561556

15571557
while True:
15581558
data = cursor.fetchmany(chunksize)
1559+
if type(data) == tuple:
1560+
data = list(data)
15591561
if not data:
15601562
cursor.close()
15611563
break

pandas/io/tests/test_sql.py

+15
Original file line numberDiff line numberDiff line change
@@ -2477,6 +2477,21 @@ def test_write_row_by_row(self):
24772477
result.index = frame.index
24782478
tm.assert_frame_equal(result, frame)
24792479

2480+
def test_chunksize_read_type(self):
2481+
_skip_if_no_pymysql()
2482+
frame = tm.makeTimeDataFrame()
2483+
frame.index.name = "index"
2484+
drop_sql = "DROP TABLE IF EXISTS test"
2485+
cur = self.conn.cursor()
2486+
cur.execute(drop_sql)
2487+
sql.to_sql(frame, name='test', con=self.conn, flavor='mysql')
2488+
query = "select * from test"
2489+
chunksize = 5
2490+
chunk_gen = pd.read_sql_query(sql=query, con=self.conn,
2491+
chunksize=chunksize, index_col="index")
2492+
chunk_df = next(chunk_gen)
2493+
tm.assert_frame_equal(frame[:chunksize], chunk_df)
2494+
24802495
def test_execute(self):
24812496
_skip_if_no_pymysql()
24822497
frame = tm.makeTimeDataFrame()

0 commit comments

Comments
 (0)