Skip to content

Updated get.py with ability to verify extracted files and skip if ok #8496

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Updated get.py with ability to verify extracted files and skip if ok #8496

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bedd35d
Updated get.py with ability to verify extracted files and skip if ok
PilnyTomas Aug 7, 2023
323913b
Updates
PilnyTomas Aug 10, 2023
bb30916
Minor fixes
PilnyTomas Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 182 additions & 34 deletions tools/get.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
import tarfile
import zipfile
import re
import time
from datetime import timedelta
import functools

# Initialize start_time globally
start_time = -1

if sys.version_info[0] == 3:
from urllib.request import urlretrieve
Expand Down Expand Up @@ -58,51 +64,161 @@ def mkdir_p(path):
if exc.errno != errno.EEXIST or not os.path.isdir(path):
raise

def report_progress(count, blockSize, totalSize):
def format_time(seconds):
minutes, seconds = divmod(seconds, 60)
return "{:02}:{:05.2f}".format(int(minutes), seconds)

def report_progress(block_count, block_size, total_size, start_time):
downloaded_size = block_count * block_size
time_elapsed = time.time() - start_time
current_speed = downloaded_size / (time_elapsed)

if sys.stdout.isatty():
if totalSize > 0:
percent = int(count*blockSize*100/totalSize)
percent = min(100, percent)
sys.stdout.write("\r%d%%" % percent)
if total_size > 0:
percent_complete = min((downloaded_size / total_size) * 100, 100)
sys.stdout.write(f"\rDownloading... {percent_complete:.2f}% - {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s")
else:
sofar = (count*blockSize) / 1024
if sofar >= 1000:
sofar /= 1024
sys.stdout.write("\r%dMB " % (sofar))
else:
sys.stdout.write("\r%dKB" % (sofar))
sys.stdout.write(f"\rDownloading... {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s")
sys.stdout.flush()

def print_verification_progress(total_files, i, t1):
if sys.stdout.isatty():
sys.stdout.write(f'\rElapsed time {format_time(time.time() - t1)}')
sys.stdout.flush()

def unpack(filename, destination):
def verify_files(filename, destination, rename_to):
# Set the path of the extracted directory
extracted_dir_path = destination
t1 = time.time()
if filename.endswith(".zip"):
try:
with zipfile.ZipFile(filename, 'r') as archive:
first_dir = archive.namelist()[0].split('/')[0]
total_files = len(archive.namelist())
for i, zipped_file in enumerate(archive.namelist(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
#print(f'\nMissing {zipped_file} on location: {extracted_dir_path}')
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i , t1)
except zipfile.BadZipFile:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
elif filename.endswith(".tar.gz"):
try:
with tarfile.open(filename, 'r:gz') as archive:
first_dir = archive.getnames()[0].split('/')[0]
total_files = len(archive.getnames())
for i, zipped_file in enumerate(archive.getnames(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
#print(f'\nMissing {zipped_file} on location: {extracted_dir_path}')
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i , t1)
except tarfile.ReadError:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
elif filename.endswith(".tar.xz"):
try:
with tarfile.open(filename, 'r:xz') as archive:
first_dir = archive.getnames()[0].split('/')[0]
total_files = len(archive.getnames())
for i, zipped_file in enumerate(archive.getnames(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
#print(f'\nMissing {zipped_file} on location: {extracted_dir_path}')
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i , t1)
except tarfile.ReadError:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
else:
raise NotImplementedError('Unsupported archive type')

#if(verbose):
#print(f"\nVerification passed; completed in {format_time(time.time() - t1)}")
return True


def unpack(filename, destination, force_extract):
dirname = ''
print('Extracting {0} ...'.format(os.path.basename(filename)))
sys.stdout.flush()
file_is_corrupted=False
if(not force_extract):
print(f' > Verify archive... ', end="", flush=True)

try:
if filename.endswith('tar.gz'):
if tarfile.is_tarfile(filename):
tfile = tarfile.open(filename, 'r:gz')
dirname = tfile.getnames()[0]
else:
print('File corrupted!')
file_is_corrupted=True
elif filename.endswith('tar.xz'):
if tarfile.is_tarfile(filename):
tfile = tarfile.open(filename, 'r:xz')
dirname = tfile.getnames()[0]
else:
print('File corrupted!')
file_is_corrupted=True
elif filename.endswith('zip'):
if zipfile.is_zipfile(filename):
zfile = zipfile.ZipFile(filename)
dirname = zfile.namelist()[0]
else:
print('File corrupted!')
file_is_corrupted=True
else:
raise NotImplementedError('Unsupported archive type')
except EOFError:
print(f'File corrupted or incomplete!')
file_is_corrupted=True

if(file_is_corrupted):
corrupted_filename = filename + ".corrupted"
os.rename(filename, corrupted_filename)
if(verbose):
print(f'Renaming corrupted archive to {corrupted_filename}')
return False

# A little trick to rename tool directories so they don't contain version number
rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-')
if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'):
rename_to = 'esp32-arduino-libs'

if not force_extract:
if(verify_files(filename, destination, rename_to)):
print(" Files ok. Skipping Extraction")
return True
else:
print(" Extracting archive...")
else:
print(" Forcing extraction")

if filename.endswith('tar.gz'):
tfile = tarfile.open(filename, 'r:gz')
tfile.extractall(destination)
dirname = tfile.getnames()[0]
elif filename.endswith('tar.xz'):
tfile = tarfile.open(filename, 'r:xz')
tfile.extractall(destination)
dirname = tfile.getnames()[0]
elif filename.endswith('zip'):
zfile = zipfile.ZipFile(filename)
zfile.extractall(destination)
dirname = zfile.namelist()[0]
else:
raise NotImplementedError('Unsupported archive type')

# a little trick to rename tool directories so they don't contain version number
rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-')
if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'):
rename_to = 'esp32-arduino-libs'
if rename_to != dirname:
print('Renaming {0} to {1} ...'.format(dirname, rename_to))
if os.path.isdir(rename_to):
shutil.rmtree(rename_to)
shutil.move(dirname, rename_to)

def download_file_with_progress(url,filename):
return True

def download_file_with_progress(url,filename, start_time):
import ssl
import contextlib
ctx = ssl.create_default_context()
Expand All @@ -117,16 +233,16 @@ def download_file_with_progress(url,filename):
with open(filename,'wb') as out_file:
out_file.write(block)
block_count += 1
report_progress(block_count, block_size, total_size)
report_progress(block_count, block_size, total_size, start_time)
while True:
block = fp.read(block_size)
if not block:
break
out_file.write(block)
block_count += 1
report_progress(block_count, block_size, total_size)
report_progress(block_count, block_size, total_size, start_time)
else:
raise Exception ('nonexisting file or connection error')
raise Exception('Non-existing file or connection error')

def download_file(url,filename):
import ssl
Expand All @@ -146,15 +262,19 @@ def download_file(url,filename):
break
out_file.write(block)
else:
raise Exception ('nonexisting file or connection error')
raise Exception ('Non-existing file or connection error')

def get_tool(tool):
def get_tool(tool, force_download, force_extract):
sys_name = platform.system()
archive_name = tool['archiveFileName']
local_path = dist_dir + archive_name
url = tool['url']
if not os.path.isfile(local_path):
print('Downloading ' + archive_name + ' ...')
start_time = time.time()
if not os.path.isfile(local_path) or force_download:
if verbose:
print('Downloading \'' + archive_name + '\' to \'' + local_path + '\'')
else:
print('Downloading \'' + archive_name + '\' ...')
sys.stdout.flush()
if 'CYGWIN_NT' in sys_name:
import ssl
Expand All @@ -175,13 +295,13 @@ def get_tool(tool):
try:
urlretrieve(url, local_path, report_progress)
except:
download_file_with_progress(url, local_path)
sys.stdout.write("\rDone \n")
download_file_with_progress(url, local_path, start_time)
sys.stdout.write(" - Done\n")
sys.stdout.flush()
else:
print('Tool {0} already downloaded'.format(archive_name))
sys.stdout.flush()
unpack(local_path, '.')
return unpack(local_path, '.', force_extract)

def load_tools_list(filename, platform):
tools_info = json.load(open(filename))['packages'][0]['tools']
Expand Down Expand Up @@ -224,10 +344,34 @@ def identify_platform():
print('System: %s, Bits: %d, Info: %s' % (sys_name, bits, sys_platform))
return arduino_platform_names[sys_name][bits]

def print_help():
print("TODO help")

if __name__ == '__main__':
is_test = (len(sys.argv) > 1 and sys.argv[1] == '-h')
option_print_help = "-h" in sys.argv
verbose = "-v" in sys.argv
force_download = "-d" in sys.argv
force_extract = "-e" in sys.argv
force_all = "-f" in sys.argv
is_test = "-t" in sys.argv

print(f'Debug: options values: option_print_help={option_print_help}; verbose={verbose}; force_download={force_download}; force_extract={force_extract}; force_all={force_all}; is_test={is_test};')
if option_print_help:
print_help()
sys.exit()

if is_test and (force_download or force_extract or force_all):
print('Cannot combine test (-t) and forced execution (-d | -e | -f)')
print_help()
sys.exit()

if is_test:
print('Test run!')

if force_all:
force_download = True
force_extract = True

identified_platform = identify_platform()
print('Platform: {0}'.format(identified_platform))
tools_to_download = load_tools_list(current_dir + '/../package/package_esp32_index.template.json', identified_platform)
Expand All @@ -236,5 +380,9 @@ def identify_platform():
if is_test:
print('Would install: {0}'.format(tool['archiveFileName']))
else:
get_tool(tool)
if(not get_tool(tool, force_download, force_extract)):
if(verbose):
print(f"Tool {tool['archiveFileName']} was corrupted. Re-downloading...\n")
if(not get_tool(tool, True, force_extract)): # Corrupted file was renamed, will not be found and will be re-downloaded
print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n")
print('Platform Tools Installed')