bigscience-workshop · stephenbach · Jul 8, 2022 · May 12, 2022 · May 12, 2022 · May 12, 2022
diff --git a/promptsource/app.py b/promptsource/app.py
@@ -11,10 +11,9 @@
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import DjangoLexer
-from templates import INCLUDED_USERS
 
 from promptsource.session import _get_state
-from promptsource.templates import DatasetTemplates, Template, TemplateCollection
+from promptsource.templates import INCLUDED_USERS, LANGUAGES, METRICS, DatasetTemplates, Template, TemplateCollection
 from promptsource.utils import (
     get_dataset,
     get_dataset_confs,
@@ -41,6 +40,17 @@ def get_infos(all_infos, d_name):
     all_infos[d_name] = get_dataset_infos(d_name)
 
 
+def format_language(tag):
+    """
+    Formats a language tag for display in the UI.
+
+    For example, if the tag is "en", then the function returns "en (English)"
+    :param tag: language tag
+    :return: formatted language name
+    """
+    return tag + " (" + LANGUAGES[tag] + ")"
+
+
 # add an argument for read-only
 # At the moment, streamlit does not handle python script arguments gracefully.
 # Thus, for read-only mode, you have to type one of the below two:
@@ -403,6 +413,11 @@ def show_text(t, width=WIDTH, with_markdown=False):
                     st.text(template.metadata.choices_in_prompt)
                     st.markdown("##### Metrics")
                     st.text(", ".join(template.metadata.metrics) if template.metadata.metrics else None)
+                    st.markdown("##### Languages")
+                    if template.metadata.languages:
+                        st.text(", ".join([format_language(tag) for tag in template.metadata.languages]))
+                    else:
+                        st.text(None)
                     st.markdown("##### Answer Choices")
                     if template.get_answer_choices_expr() is not None:
                         show_jinja(template.get_answer_choices_expr())
@@ -539,35 +554,24 @@ def show_text(t, width=WIDTH, with_markdown=False):
                                 help="Prompt explicitly lists choices in the template for the output.",
                             )
 
-                            # Metrics from here:
-                            # https://github.com/google-research/text-to-text-transfer-transformer/blob/4b580f23968c2139be7fb1cd53b22c7a7f686cdf/t5/evaluation/metrics.py
-                            metrics_choices = [
-                                "BLEU",
-                                "ROUGE",
-                                "Squad",
-                                "Trivia QA",
-                                "Accuracy",
-                                "Pearson Correlation",
-                                "Spearman Correlation",
-                                "MultiRC",
-                                "AUC",
-                                "COQA F1",
-                                "Edit Distance",
-                            ]
-                            # Add mean reciprocal rank
-                            metrics_choices.append("Mean Reciprocal Rank")
-                            # Add generic other
-                            metrics_choices.append("Other")
-                            # Sort alphabetically
-                            metrics_choices = sorted(metrics_choices)
                             state.metadata.metrics = st.multiselect(
                                 "Metrics",
-                                metrics_choices,
+                                sorted(METRICS),
                                 default=template.metadata.metrics,
                                 help="Select all metrics that are commonly used (or should "
                                 "be used if a new task) to evaluate this prompt.",
                             )
 
+                            state.metadata.languages = st.multiselect(
+                                "Languages",
+                                sorted(LANGUAGES.keys()),
+                                default=template.metadata.languages,
+                                format_func=format_language,
+                                help="Select all languages used in this prompt. "
+                                "This annotation is independent from the language(s) "
+                                "of the dataset.",
+                            )
+
                             # Answer choices
                             if template.get_answer_choices_expr() is not None:
                                 answer_choices = template.get_answer_choices_expr()

diff --git a/promptsource/templates.py b/promptsource/templates.py
@@ -28,6 +28,154 @@
 # filter_english_datasets (regardless of their metadata)
 INCLUDED_USERS = {"Zaid", "craffel"}
 
+# These are the metrics with which templates can be tagged
+METRICS = {
+    "BLEU",
+    "ROUGE",
+    "Squad",
+    "Trivia QA",
+    "Accuracy",
+    "Pearson Correlation",
+    "Spearman Correlation",
+    "MultiRC",
+    "AUC",
+    "COQA F1",
+    "Edit Distance",
+    "Mean Reciprocal Rank",
+    "Other",
+}
+
+# These are the languages with which templates can be tagged. Keys are IETF
+# primary language subtags, which are the actual tags we use. Values are English
+# names shown in the UI for convenience.
+LANGUAGES = {
+    "af": "Afrikaans",
+    "am": "Amharic",
+    "ar": "Arabic",
+    "arn": "Mapudungun",
+    "as": "Assamese",
+    "az": "Azeri",
+    "ba": "Bashkir",
+    "be": "Belarusian",
+    "bg": "Bulgarian",
+    "bn": "Bengali",
+    "bo": "Tibetan",
+    "br": "Breton",
+    "bs": "Bosnian",
+    "ca": "Catalan",
+    "co": "Corsican",
+    "cs": "Czech",
+    "cy": "Welsh",
+    "da": "Danish",
+    "de": "German",
+    "dsb": "Lower Sorbian",
+    "dv": "Divehi",
+    "el": "Greek",
+    "en": "English",
+    "es": "Spanish",
+    "et": "Estonian",
+    "eu": "Basque",
+    "fa": "Persian",
+    "fi": "Finnish",
+    "fil": "Filipino",
+    "fo": "Faroese",
+    "fr": "French",
+    "fy": "Frisian",
+    "ga": "Irish",
+    "gd": "Scottish Gaelic",
+    "gl": "Galician",
+    "gsw": "Alsatian",
+    "gu": "Gujarati",
+    "ha": "Hausa",
+    "he": "Hebrew",
+    "hi": "Hindi",
+    "hr": "Croatian",
+    "hsb": "Upper Sorbian",
+    "hu": "Hungarian",
+    "hy": "Armenian",
+    "id": "Indonesian",
+    "ig": "Igbo",
+    "ii": "Yi",
+    "is": "Icelandic",
+    "it": "Italian",
+    "iu": "Inuktitut",
+    "ja": "Japanese",
+    "ka": "Georgian",
+    "kk": "Kazakh",
+    "kl": "Greenlandic",
+    "km": "Khmer",
+    "kn": "Kannada",
+    "ko": "Korean",
+    "kok": "Konkani",
+    "ky": "Kyrgyz",
+    "lb": "Luxembourgish",
+    "lo": "Lao",
+    "lt": "Lithuanian",
+    "lv": "Latvian",
+    "mi": "Maori",
+    "mk": "Macedonian",
+    "ml": "Malayalam",
+    "mn": "Mongolian",
+    "moh": "Mohawk",
+    "mr": "Marathi",
+    "ms": "Malay",
+    "mt": "Maltese",
+    "my": "Burmese",
+    "nb": "Norwegian (Bokmål)",
+    "ne": "Nepali",
+    "nl": "Dutch",
+    "nn": "Norwegian (Nynorsk)",
+    "no": "Norwegian",
+    "nso": "Sesotho",
+    "oc": "Occitan",
+    "or": "Oriya",
+    "pa": "Punjabi",
+    "pl": "Polish",
+    "prs": "Dari",
+    "ps": "Pashto",
+    "pt": "Portuguese",
+    "quc": "K'iche",
+    "quz": "Quechua",
+    "rm": "Romansh",
+    "ro": "Romanian",
+    "ru": "Russian",
+    "rw": "Kinyarwanda",
+    "sa": "Sanskrit",
+    "sah": "Yakut",
+    "se": "Sami (Northern)",
+    "si": "Sinhala",
+    "sk": "Slovak",
+    "sl": "Slovenian",
+    "sma": "Sami (Southern)",
+    "smj": "Sami (Lule)",
+    "smn": "Sami (Inari)",
+    "sms": "Sami (Skolt)",
+    "sq": "Albanian",
+    "sr": "Serbian",
+    "sv": "Swedish",
+    "sw": "Kiswahili",
+    "syr": "Syriac",
+    "ta": "Tamil",
+    "te": "Telugu",
+    "tg": "Tajik",
+    "th": "Thai",
+    "tk": "Turkmen",
+    "tn": "Setswana",
+    "tr": "Turkish",
+    "tt": "Tatar",
+    "tzm": "Tamazight",
+    "ug": "Uyghur",
+    "uk": "Ukrainian",
+    "ur": "Urdu",
+    "uz": "Uzbek",
+    "vi": "Vietnamese",
+    "wo": "Wolof",
+    "xh": "isiXhosa",
+    "yo": "Yoruba",
+    "zh": "Chinese",
+    "zu": "isiZulu",
+}
+
 
 def highlight(input):
     return "<span style='color: #F08080'>" + input + "</span>"
@@ -220,6 +368,7 @@ def __init__(
             original_task: Optional[bool] = None,
             choices_in_prompt: Optional[bool] = None,
             metrics: Optional[List[str]] = None,
+            languages: Optional[List[str]] = None,
         ):
             """
             Initializes template metadata.
@@ -233,10 +382,12 @@ def __init__(
             :param choices_in_prompt: If True, the answer choices are included in the templates such that models
                 see those choices in the input. Only applicable to classification tasks.
             :param metrics: List of strings denoting metrics to use for evaluation
+            :param metrics: List of strings denoting languages used in the prompt (not the associated dataset!)
             """
             self.original_task = original_task
             self.choices_in_prompt = choices_in_prompt
             self.metrics = metrics
+            self.languages = languages
 
 
 class TemplateCollection:
@@ -496,6 +647,7 @@ def get_templates_data_frame():
         "original_task": [],
         "choices_in_prompt": [],
         "metrics": [],
+        "languages": [],
         "answer_choices": [],
         "jinja": [],
     }
@@ -514,6 +666,7 @@ def get_templates_data_frame():
             data["original_task"].append(template.metadata.original_task)
             data["choices_in_prompt"].append(template.metadata.choices_in_prompt)
             data["metrics"].append(template.metadata.metrics)
+            data["languages"].append(template.metadata.languages)
             data["answer_choices"].append(template.get_answer_choices_expr())
             data["jinja"].append(template.jinja)
 

diff --git a/promptsource/templates/Zaid/coqa_expanded/templates.yaml b/promptsource/templates/Zaid/coqa_expanded/templates.yaml
@@ -10,6 +10,8 @@ templates:
       \ %}\n{{answer[\"input_text\"]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -24,6 +26,8 @@ templates:
       input_text\"]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -38,6 +42,8 @@ templates:
       input_text\"]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -52,6 +58,8 @@ templates:
       input_text\"]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -82,6 +90,8 @@ templates:
       {% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Squad
       original_task: false
@@ -96,6 +106,8 @@ templates:
       ]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -108,6 +120,8 @@ templates:
       \ -1 %}\n{{answer[\"input_text\"]}}\n{% else %}\nunknown\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true

diff --git a/promptsource/templates/Zaid/quac_expanded/templates.yaml b/promptsource/templates/Zaid/quac_expanded/templates.yaml
@@ -8,6 +8,8 @@ templates:
       Q: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -19,6 +21,8 @@ templates:
     jinja: "{{context}}\n\nQ: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -33,6 +37,8 @@ templates:
       \ {{context}}\n\nQ: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -46,6 +52,8 @@ templates:
       \nQ: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -59,6 +67,8 @@ templates:
       \nQ: {{question}} \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true
@@ -72,6 +82,8 @@ templates:
       \ \nA: ||| {{answer[\"texts\"][0]}}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
+      languages:
+      - en
       metrics:
       - Other
       original_task: true