Add HuffPost (#750)

* add username to INCLUDED_USERS list

* add 5 prompts

* change prompt name

* add choices_in_prompt and fix minor issues

* add labels and fix a prompt

* title case answer choices

* fix grammatical error

* add other metrics

* remove file

* fix issues

* Update templates.py

* Update templates.py

* set choices_in_prompt to false

* add answer_choices in the prompts

* fix minor issues

* make prompts more natural

Author: KhalidAlt

promptsource/.app.py.swp

@@ -26,7 +26,7 @@
 
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
-INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps"}
+INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt"}
 
 
 def highlight(input):

promptsource/templates.py

@@ -0,0 +1,98 @@
+dataset: khalidalt/HuffPost
+templates:
+  19d2449b-1a52-4079-aaec-301517ee6ee1: !Template
+    answer_choices: Politics|||  Wellness|||  Entertainment|||  Travel|||  Style &
+      Beauty|||  Parenting|||  Healthy Living|||  Queer Voices|||  Food & Drink|||  Business|||  Comedy|||  Sports|||  Black
+      Voices|||  Home & Living|||  Parents|||  The Worldpost|||  Weddings|||  Women|||  Impact|||  Divorce|||  Crime|||  Media|||  Weird
+      News|||  Green|||  Worldpost|||  Religion|||  Style|||  Science|||  World News|||  Taste|||  Tech|||  Money|||  Arts|||  Fifty|||  Good
+      News|||  Arts & Culture|||  Environment|||  College|||  Latino Voices|||  Culture
+      & Arts|||  Education
+    id: 19d2449b-1a52-4079-aaec-301517ee6ee1
+    jinja: "Given these headline and description, what category can you infer this\
+      \ piece of news belongs to? Headline: {{headline}}, Short description: {{short_description}}\
+      \ \n||| {{answer_choices[label]}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: infer_from
+    reference: ''
+  21973a70-f643-4172-a5f3-624aaf0d339a: !Template
+    answer_choices: Politics|||  Wellness|||  Entertainment|||  Travel|||  Style &
+      Beauty|||  Parenting|||  Healthy Living|||  Queer Voices|||  Food & Drink|||  Business|||  Comedy|||  Sports|||  Black
+      Voices|||  Home & Living|||  Parents|||  The Worldpost|||  Weddings|||  Women|||  Impact|||  Divorce|||  Crime|||  Media|||  Weird
+      News|||  Green|||  Worldpost|||  Religion|||  Style|||  Science|||  World News|||  Taste|||  Tech|||  Money|||  Arts|||  Fifty|||  Good
+      News|||  Arts & Culture|||  Environment|||  College|||  Latino Voices|||  Culture
+      & Arts|||  Education
+    id: 21973a70-f643-4172-a5f3-624aaf0d339a
+    jinja: "Given the following news headline and short description: \n{{headline}}\n\
+      {{short_description}} \nWhat is a correct category? Please choose from the list\
+      \ of options below: \n\u2022 {{answer_choices | join(\",\") |replace(\",\",\"\
+      \\n\u2022 \") }} \n||| {{answer_choices[label]}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: class
+    reference: ''
+  3a782f3e-7a67-4e46-ad46-57d4491a91c5: !Template
+    answer_choices: Politics|||  Wellness|||  Entertainment|||  Travel|||  Style &
+      Beauty|||  Parenting|||  Healthy Living|||  Queer Voices|||  Food & Drink|||  Business|||  Comedy|||  Sports|||  Black
+      Voices|||  Home & Living|||  Parents|||  The Worldpost|||  Weddings|||  Women|||  Impact|||  Divorce|||  Crime|||  Media|||  Weird
+      News|||  Green|||  Worldpost|||  Religion|||  Style|||  Science|||  World News|||  Taste|||  Tech|||  Money|||  Arts|||  Fifty|||  Good
+      News|||  Arts & Culture|||  Environment|||  College|||  Latino Voices|||  Culture
+      & Arts|||  Education
+    id: 3a782f3e-7a67-4e46-ad46-57d4491a91c5
+    jinja: "Given the following list of news categories: \n- {{answer_choices | join(\"\
+      ,\") |replace(\",\",\"\\n- \") }}\nClassify the following news to the correct\
+      \ category: \n{{headline}}\n{{short_description}} \n||| {{answer_choices[label]}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: Classify_category
+    reference: ''
+  ab757ddf-0c94-49b4-89a6-e7ead4ab5a58: !Template
+    answer_choices: Politics|||  Wellness|||  Entertainment|||  Travel|||  Style &
+      Beauty|||  Parenting|||  Healthy Living|||  Queer Voices|||  Food & Drink|||  Business|||  Comedy|||  Sports|||  Black
+      Voices|||  Home & Living|||  Parents|||  The Worldpost|||  Weddings|||  Women|||  Impact|||  Divorce|||  Crime|||  Media|||  Weird
+      News|||  Green|||  Worldpost|||  Religion|||  Style|||  Science|||  World News|||  Taste|||  Tech|||  Money|||  Arts|||  Fifty|||  Good
+      News|||  Arts & Culture|||  Environment|||  College|||  Latino Voices|||  Culture
+      & Arts|||  Education
+    id: ab757ddf-0c94-49b4-89a6-e7ead4ab5a58
+    jinja: "{{headline}}\n{{date}}\n{{short_description}}\nWhat is a correct category\
+      \ for this news from the following list of options:  \n- {{answer_choices |\
+      \ join(\",\") |replace(\",\",\"\\n- \") }} |||\n{{answer_choices[label]}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: correct_category
+    reference: ''
+  deac9d78-f784-47f6-8cac-3cc6bbdf9f1a: !Template
+    answer_choices: Politics|||  Wellness|||  Entertainment|||  Travel|||  Style &
+      Beauty|||  Parenting|||  Healthy Living|||  Queer Voices|||  Food & Drink|||  Business|||  Comedy|||  Sports|||  Black
+      Voices|||  Home & Living|||  Parents|||  The Worldpost|||  Weddings|||  Women|||  Impact|||  Divorce|||  Crime|||  Media|||  Weird
+      News|||  Green|||  Worldpost|||  Religion|||  Style|||  Science|||  World News|||  Taste|||  Tech|||  Money|||  Arts|||  Fifty|||  Good
+      News|||  Arts & Culture|||  Environment|||  College|||  Latino Voices|||  Culture
+      & Arts|||  Education
+    id: deac9d78-f784-47f6-8cac-3cc6bbdf9f1a
+    jinja: "Given the following article headline and short passage of the article,\
+      \ what is the category of this news report? \nArticle headline: {{headline}}\n\
+      Short passage: \"{{short_description}}\" \n ||| {{answer_choices[label]}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: category_of
+    reference: ''