Skip to content

Commit 32d8d06

Browse files
ahlincahlinc
authored
feat: Radically redesign the build plan form (#646)
1 parent 1fe3e4a commit 32d8d06

File tree

3 files changed

+159
-135
lines changed

3 files changed

+159
-135
lines changed

‎package.py

Lines changed: 150 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -679,9 +679,10 @@ def plan(self, source_path, query):
679679

680680
source_paths = []
681681
build_plan = []
682+
build_step = []
682683

683684
def step(*x):
684-
build_plan.append(x)
685+
build_step.append(x)
685686

686687
def hash(path):
687688
source_paths.append(path)
@@ -754,30 +755,30 @@ def commands_step(path, commands):
754755

755756
if path:
756757
path = os.path.normpath(path)
758+
step("set:workdir", path)
759+
757760
batch = []
758761
for c in commands:
759762
if isinstance(c, str):
760763
if c.startswith(":zip"):
761764
if path:
762765
hash(path)
763766
if batch:
764-
step("sh", path, "\n".join(batch))
767+
step("sh", "\n".join(batch))
765768
batch.clear()
766769
c = shlex.split(c)
767-
if len(c) == 3:
770+
n = len(c)
771+
if n == 3:
768772
_, _path, prefix = c
769773
prefix = prefix.strip()
770-
_path = os.path.normpath(os.path.join(path, _path))
774+
_path = os.path.normpath(_path)
771775
step("zip:embedded", _path, prefix)
772-
elif len(c) == 2:
776+
elif n == 2:
773777
_, _path = c
774-
prefix = None
775778
_path = os.path.normpath(_path)
776-
step("zip:embedded", _path, prefix)
777-
elif len(c) == 1:
778-
prefix = None
779-
_path = None
780-
step("zip:embedded", _path, prefix)
779+
step("zip:embedded", _path)
780+
elif n == 1:
781+
step("zip:embedded")
781782
else:
782783
raise ValueError(
783784
":zip invalid call signature, use: "
@@ -786,11 +787,9 @@ def commands_step(path, commands):
786787
else:
787788
batch.append(c)
788789
if batch:
789-
step("sh", path, "\n".join(batch))
790+
step("sh", "\n".join(batch))
790791
batch.clear()
791792

792-
step("reset:workdir")
793-
794793
for claim in claims:
795794
if isinstance(claim, str):
796795
path = claim
@@ -877,12 +876,13 @@ def commands_step(path, commands):
877876
hash(path_from_pattern)
878877
else:
879878
hash(path)
880-
881-
if patterns:
882-
step("clear:filter")
883879
else:
884880
raise ValueError("Unsupported source_path item: {}".format(claim))
885881

882+
if build_step:
883+
build_plan.append(build_step)
884+
build_step = []
885+
886886
self._source_paths = source_paths
887887
return build_plan
888888

@@ -895,125 +895,145 @@ def execute(self, build_plan, zip_stream, query):
895895
sh_work_dir = None
896896
pf = None
897897

898-
for action in build_plan:
899-
cmd = action[0]
900-
if cmd.startswith("zip"):
901-
ts = 0 if cmd == "zip:embedded" else None
902-
source_path, prefix = action[1:]
903-
if not sh_work_dir:
904-
sh_work_dir = tf_work_dir
905-
log.debug("WORKDIR: %s", sh_work_dir)
906-
if source_path:
907-
if not os.path.isabs(source_path):
908-
source_path = os.path.normpath(
909-
os.path.join(sh_work_dir, source_path)
910-
)
911-
else:
912-
source_path = sh_work_dir
913-
if os.path.isdir(source_path):
914-
if pf:
915-
self._zip_write_with_filter(
916-
zs, pf, source_path, prefix, timestamp=ts
917-
)
898+
for step in build_plan:
899+
# init step
900+
sh_work_dir = tf_work_dir
901+
if pf:
902+
pf.reset()
903+
pf = None
904+
905+
log.debug("STEPDIR: %s", sh_work_dir)
906+
907+
# execute step actions
908+
for action in step:
909+
cmd = action[0]
910+
if cmd.startswith("zip"):
911+
ts = 0 if cmd == "zip:embedded" else None
912+
913+
source_path, prefix = None, None
914+
n = len(action)
915+
if n == 2:
916+
source_path = action[1]
917+
elif n == 3:
918+
source_path, prefix = action[1:]
919+
920+
if source_path:
921+
if not os.path.isabs(source_path):
922+
source_path = os.path.normpath(
923+
os.path.join(sh_work_dir, source_path)
924+
)
918925
else:
919-
zs.write_dirs(source_path, prefix=prefix, timestamp=ts)
920-
else:
921-
zs.write_file(source_path, prefix=prefix, timestamp=ts)
922-
elif cmd == "pip":
923-
runtime, pip_requirements, prefix, tmp_dir = action[1:]
924-
with install_pip_requirements(query, pip_requirements, tmp_dir) as rd:
925-
if rd:
926+
source_path = sh_work_dir
927+
if os.path.isdir(source_path):
926928
if pf:
927-
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
928-
else:
929-
# XXX: timestamp=0 - what actually do with it?
930-
zs.write_dirs(rd, prefix=prefix, timestamp=0)
931-
elif cmd == "poetry":
932-
(
933-
runtime,
934-
path,
935-
poetry_export_extra_args,
936-
prefix,
937-
) = action[1:]
938-
log.info("poetry_export_extra_args: %s", poetry_export_extra_args)
939-
with install_poetry_dependencies(
940-
query, path, poetry_export_extra_args
941-
) as rd:
942-
if rd:
943-
if pf:
944-
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
945-
else:
946-
# XXX: timestamp=0 - what actually do with it?
947-
zs.write_dirs(rd, prefix=prefix, timestamp=0)
948-
elif cmd == "npm":
949-
runtime, npm_requirements, prefix, tmp_dir = action[1:]
950-
with install_npm_requirements(query, npm_requirements, tmp_dir) as rd:
951-
if rd:
952-
if pf:
953-
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
929+
self._zip_write_with_filter(
930+
zs, pf, source_path, prefix, timestamp=ts
931+
)
954932
else:
955-
# XXX: timestamp=0 - what actually do with it?
956-
zs.write_dirs(rd, prefix=prefix, timestamp=0)
957-
elif cmd == "sh":
958-
with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file:
959-
path, script = action[1:]
960-
961-
if not path:
962-
path = tf_work_dir
963-
if not os.path.isabs(path):
964-
path = os.path.normpath(os.path.join(tf_work_dir, path))
965-
966-
if log.isEnabledFor(DEBUG2):
967-
log.debug("exec shell script ...")
968-
for line in script.splitlines():
969-
sh_log.debug(line)
970-
971-
script = "\n".join(
972-
(
973-
script,
974-
# NOTE: Execute `pwd` to determine the subprocess shell's
975-
# working directory after having executed all other commands.
976-
"retcode=$?",
977-
f"pwd >{temp_file.name}",
978-
"exit $retcode",
933+
zs.write_dirs(source_path, prefix=prefix, timestamp=ts)
934+
else:
935+
zs.write_file(source_path, prefix=prefix, timestamp=ts)
936+
elif cmd == "pip":
937+
runtime, pip_requirements, prefix, tmp_dir = action[1:]
938+
with install_pip_requirements(
939+
query, pip_requirements, tmp_dir
940+
) as rd:
941+
if rd:
942+
if pf:
943+
self._zip_write_with_filter(
944+
zs, pf, rd, prefix, timestamp=0
945+
)
946+
else:
947+
# XXX: timestamp=0 - what actually do with it?
948+
zs.write_dirs(rd, prefix=prefix, timestamp=0)
949+
elif cmd == "poetry":
950+
(
951+
runtime,
952+
path,
953+
poetry_export_extra_args,
954+
prefix,
955+
) = action[1:]
956+
log.info("poetry_export_extra_args: %s", poetry_export_extra_args)
957+
with install_poetry_dependencies(
958+
query, path, poetry_export_extra_args
959+
) as rd:
960+
if rd:
961+
if pf:
962+
self._zip_write_with_filter(
963+
zs, pf, rd, prefix, timestamp=0
964+
)
965+
else:
966+
# XXX: timestamp=0 - what actually do with it?
967+
zs.write_dirs(rd, prefix=prefix, timestamp=0)
968+
elif cmd == "npm":
969+
runtime, npm_requirements, prefix, tmp_dir = action[1:]
970+
with install_npm_requirements(
971+
query, npm_requirements, tmp_dir
972+
) as rd:
973+
if rd:
974+
if pf:
975+
self._zip_write_with_filter(
976+
zs, pf, rd, prefix, timestamp=0
977+
)
978+
else:
979+
# XXX: timestamp=0 - what actually do with it?
980+
zs.write_dirs(rd, prefix=prefix, timestamp=0)
981+
elif cmd == "sh":
982+
with tempfile.NamedTemporaryFile(
983+
mode="w+t", delete=True
984+
) as temp_file:
985+
script = action[1]
986+
987+
if log.isEnabledFor(DEBUG2):
988+
log.debug("exec shell script ...")
989+
for line in script.splitlines():
990+
sh_log.debug(line)
991+
992+
script = "\n".join(
993+
(
994+
script,
995+
# NOTE: Execute `pwd` to determine the subprocess shell's
996+
# working directory after having executed all other commands.
997+
"retcode=$?",
998+
f"pwd >{temp_file.name}",
999+
"exit $retcode",
1000+
)
9791001
)
980-
)
9811002

982-
p = subprocess.Popen(
983-
script,
984-
shell=True,
985-
stdout=subprocess.PIPE,
986-
stderr=subprocess.PIPE,
987-
cwd=path,
988-
)
1003+
p = subprocess.Popen(
1004+
script,
1005+
shell=True,
1006+
stdout=subprocess.PIPE,
1007+
stderr=subprocess.PIPE,
1008+
cwd=sh_work_dir,
1009+
)
9891010

990-
call_stdout, call_stderr = p.communicate()
991-
exit_code = p.returncode
992-
log.debug("exit_code: %s", exit_code)
993-
if exit_code != 0:
994-
raise RuntimeError(
995-
"Script did not run successfully, exit code {}: {} - {}".format(
996-
exit_code,
997-
call_stdout.decode("utf-8").strip(),
998-
call_stderr.decode("utf-8").strip(),
1011+
call_stdout, call_stderr = p.communicate()
1012+
exit_code = p.returncode
1013+
log.debug("exit_code: %s", exit_code)
1014+
if exit_code != 0:
1015+
raise RuntimeError(
1016+
"Script did not run successfully, exit code {}: {} - {}".format(
1017+
exit_code,
1018+
call_stdout.decode("utf-8").strip(),
1019+
call_stderr.decode("utf-8").strip(),
1020+
)
9991021
)
1000-
)
10011022

1002-
temp_file.seek(0)
1003-
# NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop
1004-
sh_work_dir = temp_file.read().strip()
1023+
temp_file.seek(0)
1024+
# NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop
1025+
sh_work_dir = temp_file.read().strip()
1026+
log.debug("WORKDIR: %s", sh_work_dir)
1027+
1028+
elif cmd == "set:workdir":
1029+
path = action[1]
1030+
sh_work_dir = os.path.normpath(os.path.join(tf_work_dir, path))
10051031
log.debug("WORKDIR: %s", sh_work_dir)
10061032

1007-
elif cmd == "reset:workdir":
1008-
sh_work_dir = tf_work_dir
1009-
log.debug("WORKDIR: %s", sh_work_dir)
1010-
elif cmd == "set:filter":
1011-
patterns = action[1]
1012-
pf = ZipContentFilter(args=self._args)
1013-
pf.compile(patterns)
1014-
elif cmd == "clear:filter":
1015-
pf.reset()
1016-
pf = None
1033+
elif cmd == "set:filter":
1034+
patterns = action[1]
1035+
pf = ZipContentFilter(args=self._args)
1036+
pf.compile(patterns)
10171037

10181038
@staticmethod
10191039
def _zip_write_with_filter(
@@ -1616,11 +1636,11 @@ def prepare_command(args):
16161636
content_hash = content_hash.hexdigest()
16171637

16181638
# Generate a unique filename based on the hash.
1619-
filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash))
1639+
zip_filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash))
16201640

16211641
# Compute timestamp trigger
16221642
was_missing = False
1623-
filename_path = os.path.join(os.getcwd(), filename)
1643+
filename_path = os.path.join(os.getcwd(), zip_filename)
16241644
if recreate_missing_package:
16251645
if os.path.exists(filename_path):
16261646
st = os.stat(filename_path)
@@ -1633,7 +1653,7 @@ def prepare_command(args):
16331653

16341654
# Replace variables in the build command with calculated values.
16351655
build_data = {
1636-
"filename": filename,
1656+
"filename": zip_filename,
16371657
"runtime": runtime,
16381658
"artifacts_dir": artifacts_dir,
16391659
"build_plan": build_plan,
@@ -1653,7 +1673,7 @@ def prepare_command(args):
16531673
# Output the result to Terraform.
16541674
json.dump(
16551675
{
1656-
"filename": filename,
1676+
"filename": zip_filename,
16571677
"build_plan": build_plan,
16581678
"build_plan_filename": build_plan_filename,
16591679
"timestamp": str(timestamp),

‎tests/test_package_toml.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def test_build_manager_failing_command():
2626
bpm = BuildPlanManager(args=Mock())
2727
with raises(Exception):
2828
bpm.execute(
29-
build_plan=[["sh", "/tmp", "NOTACOMMAND"]],
29+
build_plan=[[["sh", "/tmp", "NOTACOMMAND"]]],
3030
zip_stream=None,
3131
query=None,
3232
)

0 commit comments

Comments
 (0)