500 Unknown Error with no details when trying to download a report from dfareporting v4

[robinstauntoncollins]

Environment details

• OS type and version: Windows 10
• Python version: 3.7.9
• pip version: 22.0.3
• google-api-python-client version: 2.52.0

Steps to reproduce

1. Add profileId, reportId and credentials to code snippet below and run
2. Observe 500 error

Code example

import os
import json
import time
from io import FileIO

from googleapiclient import http
from google.oauth2 import service_account
from googleapiclient.discovery import build

report = {
    "criteria": {
        "dateRange": {
            "relativeDateRange": None,
            "startDate": "2022-07-04",
            "endDate": "2022-07-04",
        }
    }
}

REPORT_ID = ''  # Report Id (int)
PROFILE_ID = ''  # Profile Id (int)


def get_report(service):
    """Runs a report and waits for it to be ready"""
    report_response: dict = (
        service.reports().run(profileId=PROFILE_ID, reportId=REPORT_ID).execute()
    )
    print(json.dumps(report_response, indent=4))
    file_id = report_response["id"]
    report_response: dict = (
        service.reports()
        .files()
        .get(profileId=PROFILE_ID, reportId=REPORT_ID, fileId=file_id)
        .execute()
    )
    time.sleep(5)

    while report_response["status"] != "REPORT_AVAILABLE":
        report_response: dict = (
            service.reports()
            .files()
            .get(profileId=PROFILE_ID, reportId=REPORT_ID, fileId=file_id)
            .execute()
        )
        time.sleep(5)
    print(json.dumps(report_response, indent=4))
    print(f"API URL to retrieve file: {report_response.get('urls').get('apiUrl')}")
    return report_response


def download_report(service, file_path, report_file):
    """Retrieves a ready report file"""
    out_file = FileIO(str(file_path), mode="wb")
    request = service.files().get_media(
        reportId=report_file["reportId"],
        fileId=report_file["id"],
    )

    downloader = http.MediaIoBaseDownload(out_file, request, chunksize=32 * 1024 * 1024)
    download_finished = False

    while download_finished is False:
        _, download_finished = downloader.next_chunk()

    print(
        f"File {report_file['id']} downloaded to {os.path.realpath(str(out_file.name))}"
    )
    return str(out_file.name)


if __name__ == "__main__":
    # Insert code to load service account credentials

    oauth_credentials = service_account.Credentials.from_service_account_info(
        campaign_manager_secrets
    )
    service = build(
        "dfareporting",
        "v3.5",
        credentials=oauth_credentials,
        cache_discovery=True,
    )

    report_file = get_report(service)
    file_path = download_report(service, "./campaign_manager_report.csv", report_file)
    print(f"File was downloaded: {file_path}")

Output of code snippet

{
    "kind": "dfareporting#file",
    "etag": "\"08fTcNjAfGpA5iZbj4pqeKSxfZQ\"",
    "reportId": "<removed-for-snippet>",
    "id": "<removed-for-snippet>",
    "lastModifiedTime": "1657293406000",
    "status": "PROCESSING",
    "fileName": "<removed-for-snippet>",
    "dateRange": {
        "kind": "dfareporting#dateRange",
        "startDate": "2022-07-07",
        "endDate": "2022-07-07"
    }
}
{
    "kind": "dfareporting#file",
    "etag": "\"lHjvsbj0FTJKOmBHFJQYhcw7xaY\"",
    "reportId": "<removed-for-snippet>",
    "id": "<removed-for-snippet>",
    "lastModifiedTime": "1657293407000",
    "status": "REPORT_AVAILABLE",
    "fileName": "<removed-for-snippet>",
    "format": "CSV",
    "dateRange": {
        "kind": "dfareporting#dateRange",
        "startDate": "2022-07-07",
        "endDate": "2022-07-07"
    },
    "urls": {
        "browserUrl": "https://www.google.com/reporting/downloadFile?id=<removed-for-snippet>:<removed-for-snippet>",
        "apiUrl": "https://www.googleapis.com/dfareporting/v4/reports/<removed-for-snippet>/files/<removed-for-snippet>?alt=media"
    }
}
API URL to retrieve file: https://www.googleapis.com/dfareporting/v4/reports/<removed-for-snippet>/files/<removed-for-snippet>?alt=media

Traceback (most recent call last):
  File "campaign_manager_test.py", line 93, in <module>
    file_path = download_report(service, "./campaign_manager_report.csv", report_file)
  File "campaign_manager_test.py", line 64, in download_report
    _, download_finished = downloader.next_chunk()
  File ".venv\lib\site-packages\googleapiclient\_helpers.py", line 130, in positional_wrapper
    return wrapped(*args, **kwargs)
  File ".venv\lib\site-packages\googleapiclient\http.py", line 780, in next_chunk
    raise HttpError(resp, content, uri=self._uri)
googleapiclient.errors.HttpError: <HttpError 500 when requesting https://dfareporting.googleapis.com/dfareporting/v4/reports/<removed-for-snippet>/files/<removed-for-snippet>?alt=media returned "Unknown Error.". Details: "[{}]">

Stack trace

Traceback (most recent call last):
  File "campaign_manager_test.py", line 93, in <module>
    file_path = download_report(service, "./campaign_manager_report.csv", report_file)
  File "campaign_manager_test.py", line 64, in download_report
    _, download_finished = downloader.next_chunk()
  File ".venv\lib\site-packages\googleapiclient\_helpers.py", line 130, in positional_wrapper
    return wrapped(*args, **kwargs)
  File ".venv\lib\site-packages\googleapiclient\http.py", line 780, in next_chunk
    raise HttpError(resp, content, uri=self._uri)
googleapiclient.errors.HttpError: <HttpError 500 when requesting https://dfareporting.googleapis.com/dfareporting/v4/reports/<removed-for-snippet>/files/<removed-for-snippet>?alt=media returned "Unknown Error.". Details: "[{}]">

This started happening suddently, with no code changes as far as I can tell, using v3.5. Upgraded to v4 to see if it was related to some deprecation but same result.

A 500 error with no error message. Could it have something to do with the API download URL link in the get report response being:

https://www.googleapis.com/dfareporting/v4/reports/REPORT_ID/files/FILE_ID?alt=media

But the python library is trying to retrieve it from:

https://dfareporting.googleapis.com/dfareporting/v4/reports/REPORT_ID/files/FILE_ID?alt=media

[krishnadw2]

Hi @robinstauntoncollins ,

Did you got solution for this error?, i am also facing same issue.

Thanks

[robinstauntoncollins]

@krishnadw2 Nope. But still early days.

[robinstauntoncollins]

I believe that this error occurs when trying to access the report (or retrieve the status of the report) before it has completed and reached the REPORT_AVAILABLE status. The report I was trying to access was not yet completed. But this doesn't explain why I get a HttpError with a 500 and an 'Unknown Error' message instead of a json result with the status of the file as 'PROCESSING' or 'QUEUED'?

[krishnadw2]

I don't thin this is due to status of the report. I have a daily job which runs at same time and it was running fine till July 5th.
From July5th something changed ,throwing httperror 500.

[robinstauntoncollins]

I don't think it's because of the status of the report. Just that's where I encountered it. I have since also encountered 500 errors when trying to download the report with REPORT_AVAILABLE status.

[suryadeepan]

My job is failing at the exact point. Did you find any solution for this?

[robinstauntoncollins]

No solution. But one workaround is to use exponential backoff retries for example with tenacity. Seems to work eventually. Also, if you're downloading files you can pass an integer to the next_chunk method with for the amount of times to retry.