Skip to content

Commit 5681e31

Browse files
committed
Process all languages in a single step
1 parent f8e6a02 commit 5681e31

File tree

5 files changed

+84
-101
lines changed

5 files changed

+84
-101
lines changed

.github/workflows/docbuild-and-upload.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
run: python -m pytest web/
5555

5656
- name: Build website
57-
run: python web/pandas_web.py web/pandas --target-path=web/build --translations
57+
run: python web/pandas_web.py web/pandas --target-path=web/build
5858

5959
- name: Build documentation
6060
run: doc/make.py --warnings-are-errors

web/pandas/_templates/layout.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030

3131
<div class="collapse navbar-collapse" id="nav-content">
3232
<ul class="navbar-nav ms-auto">
33-
{% for item in navbar %}
33+
{% for item in navbar[selected_language] %}
3434
{% if not item.has_subitems %}
3535
<li class="nav-item">
3636
<a class="nav-link" href="{% if not item.target.startswith("http") %}{{ base_url }}{% endif %}{{ item.target }}">{{ item.name }}</a>

web/pandas/config.yml

+1
Original file line numberDiff line numberDiff line change
@@ -176,5 +176,6 @@ translations:
176176
folder: translations
177177
source_path: pandas-translations-main/web/pandas/
178178
default_language: 'en'
179+
default_prefix: ''
179180
ignore:
180181
- docs/

web/pandas_translations.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ def copy_translations(source_path: str, target_path: str, languages: list[str])
8282

8383

8484
def process_translations(
85-
config_fname: str, source_path: str, process_translations: bool
85+
config_fname: str, source_path: str
8686
) -> tuple[list[str], list[str]]:
8787
"""
8888
Process the translations by downloading and extracting them from
@@ -95,15 +95,15 @@ def process_translations(
9595
translations_path, config["translations"]["source_path"]
9696
)
9797
default_language = config["translations"]["default_language"]
98+
9899
sys.stderr.write("\nDownloading and extracting translations...\n\n")
99100
download_and_extract_translations(config["translations"]["url"], translations_path)
100-
languages = [default_language]
101+
101102
translated_languages = get_languages(translations_source_path)
102103
remove_translations(source_path, translated_languages)
103-
if process_translations:
104-
languages = [default_language] + translated_languages
105104

106-
sys.stderr.write("\nCopying translations...\n")
107-
copy_translations(translations_source_path, source_path, translated_languages)
105+
languages = [default_language] + translated_languages
106+
sys.stderr.write("\nCopying translations...\n")
107+
copy_translations(translations_source_path, source_path, translated_languages)
108108

109109
return translated_languages, languages

web/pandas_web.py

+75-93
Original file line numberDiff line numberDiff line change
@@ -81,22 +81,17 @@ def navbar_add_info(context: dict, skip: bool = True) -> dict:
8181
``has_subitems`` that tells which one of them every element is. It
8282
also adds a ``slug`` field to be used as a CSS id.
8383
"""
84-
if skip:
85-
return context
86-
87-
lang = context["selected_language"]
8884
ignore = context["translations"]["ignore"]
89-
default_language = context["translations"]["default_language"]
90-
for i, item in enumerate(context["navbar"]):
91-
if item["target"] in ignore:
92-
if lang != default_language:
93-
item["target"] = "../" + item["target"]
94-
95-
context["navbar"][i] = dict(
96-
item,
97-
has_subitems=isinstance(item["target"], list),
98-
slug=item["name"].replace(" ", "-").lower(),
99-
)
85+
for language in context["languages"]:
86+
for i, item in enumerate(context["navbar"][language]):
87+
if item["target"] in ignore:
88+
item["target"] = f"../{item['target']}"
89+
90+
context["navbar"][language][i] = dict(
91+
item,
92+
has_subitems=isinstance(item["target"], list),
93+
slug=item["name"].replace(" ", "-").lower(),
94+
)
10095
return context
10196

10297
@staticmethod
@@ -396,7 +391,9 @@ def get_callable(obj_as_str: str) -> object:
396391
return obj
397392

398393

399-
def get_context(config_fname: str, **kwargs: dict) -> dict:
394+
def get_context(
395+
config_fname: str, navbar_fname: str, languages: list[str], **kwargs: dict
396+
) -> dict:
400397
"""
401398
Load the config yaml as the base context, and enrich it with the
402399
information added by the context preprocessors defined in the file.
@@ -405,6 +402,22 @@ def get_context(config_fname: str, **kwargs: dict) -> dict:
405402
context = yaml.safe_load(f)
406403

407404
context["source_path"] = os.path.dirname(config_fname)
405+
406+
navbar = {}
407+
context["languages"] = languages
408+
default_language = context["translations"]["default_language"]
409+
default_prefix = context["translations"]["default_prefix"]
410+
for language in languages:
411+
prefix = default_prefix if language == default_language else language
412+
navbar_path = os.path.join(context["source_path"], prefix, navbar_fname)
413+
414+
with open(navbar_path, encoding="utf-8") as f:
415+
navbar_lang = yaml.safe_load(f)
416+
417+
navbar[language] = navbar_lang["navbar"]
418+
419+
context["navbar"] = navbar
420+
408421
context.update(kwargs)
409422
preprocessors = (
410423
get_callable(context_prep)
@@ -418,43 +431,16 @@ def get_context(config_fname: str, **kwargs: dict) -> dict:
418431
return context
419432

420433

421-
def update_navbar_context(context: dict) -> dict:
422-
"""
423-
Update the context with the navbar information for each language.
424-
"""
425-
language = context["selected_language"]
426-
lang_prefix = (
427-
language if language != context["translations"]["default_language"] else ""
428-
)
429-
navbar_path = os.path.join(context["source_path"], lang_prefix, "navbar.yml")
430-
with open(navbar_path) as f:
431-
navbar = yaml.safe_load(f)
432-
433-
context.update(navbar)
434-
return Preprocessors.navbar_add_info(context, skip=False)
435-
436-
437434
def get_source_files(
438-
source_path: str, language: str, languages: list[str]
435+
source_path: str, languages: list[str]
439436
) -> typing.Generator[str, None, None]:
440437
"""
441438
Generate the list of files present in the source directory.
442439
"""
443-
paths = []
444-
all_languages = languages[:]
445-
all_languages.remove(language)
446440
for root, dirs, fnames in os.walk(source_path):
447441
root_rel_path = os.path.relpath(root, source_path)
448442
for fname in fnames:
449-
path = os.path.join(root_rel_path, fname)
450-
for language in all_languages:
451-
if path.startswith(language + "/"):
452-
break
453-
else:
454-
paths.append(path)
455-
456-
for path in paths:
457-
yield path
443+
yield os.path.join(root_rel_path, fname)
458444

459445

460446
def extend_base_template(content: str, base_template: str) -> str:
@@ -472,7 +458,6 @@ def extend_base_template(content: str, base_template: str) -> str:
472458
def main(
473459
source_path: str,
474460
target_path: str,
475-
process_translations: bool = False,
476461
) -> int:
477462
"""
478463
Copy every file in the source directory to the target directory.
@@ -481,8 +466,6 @@ def main(
481466
before copying them. ``.md`` files are transformed to HTML.
482467
"""
483468
base_folder = os.path.dirname(__file__)
484-
base_source_path = source_path
485-
base_target_path = target_path
486469

487470
shutil.rmtree(target_path, ignore_errors=True)
488471
os.makedirs(target_path, exist_ok=True)
@@ -491,12 +474,13 @@ def main(
491474
sys.path.append(base_folder)
492475
trans = importlib.import_module("pandas_translations")
493476
translated_languages, languages = trans.process_translations(
494-
"config.yml", source_path, process_translations
477+
"config.yml", source_path
495478
)
496479

497480
sys.stderr.write("Generating context...\n")
498481
context = get_context(
499482
os.path.join(source_path, "config.yml"),
483+
navbar_fname="navbar.yml",
500484
target_path=target_path,
501485
languages=languages,
502486
)
@@ -505,52 +489,51 @@ def main(
505489
templates_path = os.path.join(source_path, context["main"]["templates_path"])
506490
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))
507491

508-
for language in languages:
509-
context["selected_language"] = language
510-
context = update_navbar_context(context)
511-
sys.stderr.write(f"\nProcessing language: {language}...\n\n")
512-
513-
if language != context["translations"]["default_language"]:
514-
target_path = os.path.join(base_target_path, language)
515-
source_path = os.path.join(base_source_path, language)
492+
default_language = context["translations"]["default_language"]
493+
for fname in get_source_files(source_path, translated_languages):
494+
selected_language = context["translations"]["default_language"]
495+
for language in translated_languages:
496+
if fname.startswith(language + "/"):
497+
selected_language = language
498+
break
516499

517-
for fname in get_source_files(source_path, language, languages):
518-
if os.path.normpath(fname) in context["main"]["ignore"]:
519-
continue
500+
context["selected_language"] = selected_language
501+
if os.path.normpath(fname) in context["main"]["ignore"]:
502+
continue
503+
504+
sys.stderr.write(f"Processing {fname}\n")
505+
dirname = os.path.dirname(fname)
506+
os.makedirs(os.path.join(target_path, dirname), exist_ok=True)
507+
508+
extension = os.path.splitext(fname)[-1]
509+
if extension in (".html", ".md"):
510+
with open(os.path.join(source_path, fname), encoding="utf-8") as f:
511+
content = f.read()
512+
if extension == ".md":
513+
body = markdown.markdown(
514+
content, extensions=context["main"]["markdown_extensions"]
515+
)
516+
# Apply Bootstrap's table formatting manually
517+
# Python-Markdown doesn't let us config table attributes by hand
518+
body = body.replace("<table>", '<table class="table table-bordered">')
519+
content = extend_base_template(body, context["main"]["base_template"])
520520

521-
sys.stderr.write(f"Processing {fname}\n")
522-
dirname = os.path.dirname(fname)
523-
os.makedirs(os.path.join(target_path, dirname), exist_ok=True)
524-
525-
extension = os.path.splitext(fname)[-1]
526-
if extension in (".html", ".md"):
527-
with open(os.path.join(source_path, fname), encoding="utf-8") as f:
528-
content = f.read()
529-
if extension == ".md":
530-
body = markdown.markdown(
531-
content, extensions=context["main"]["markdown_extensions"]
532-
)
533-
# Apply Bootstrap's table formatting manually
534-
# Python-Markdown doesn't let us config table attributes by hand
535-
body = body.replace(
536-
"<table>", '<table class="table table-bordered">'
537-
)
538-
content = extend_base_template(
539-
body, context["main"]["base_template"]
540-
)
521+
context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/"))
522+
if selected_language != default_language:
541523
context["base_url"] = "".join(
542-
["../"] * os.path.normpath(fname).count("/")
543-
)
544-
content = jinja_env.from_string(content).render(**context)
545-
fname_html = os.path.splitext(fname)[0] + ".html"
546-
with open(
547-
os.path.join(target_path, fname_html), "w", encoding="utf-8"
548-
) as f:
549-
f.write(content)
550-
else:
551-
shutil.copy(
552-
os.path.join(source_path, fname), os.path.join(target_path, dirname)
524+
["../"] * (os.path.normpath(fname).count("/") - 1)
553525
)
526+
527+
content = jinja_env.from_string(content).render(**context)
528+
fname_html = os.path.splitext(fname)[0] + ".html"
529+
with open(
530+
os.path.join(target_path, fname_html), "w", encoding="utf-8"
531+
) as f:
532+
f.write(content)
533+
else:
534+
shutil.copy(
535+
os.path.join(source_path, fname), os.path.join(target_path, dirname)
536+
)
554537
return 0
555538

556539

@@ -562,6 +545,5 @@ def main(
562545
parser.add_argument(
563546
"--target-path", default="build", help="directory where to write the output"
564547
)
565-
parser.add_argument("-t", "--translations", action="store_true")
566548
args = parser.parse_args()
567-
sys.exit(main(args.source_path, args.target_path, args.translations))
549+
sys.exit(main(args.source_path, args.target_path))

0 commit comments

Comments
 (0)