Merge pull request #4701 from akshay-99/i18n-host-translations-online

FES + i18n: Load translations from a CDN

Author: stalgiag

Gruntfile.js

@@ -189,7 +189,7 @@ module.exports = grunt => {
           'src/**/*.vert',
           'src/**/*.glsl'
         ],
-        tasks: ['browserify'],
+        tasks: ['browserify:dev'],
         options: {
           livereload: true
         }

contributor_docs/internationalization.md

@@ -82,7 +82,12 @@ The easiest way to do this is to add your language code (like "de" for German, "
 
 This will generate you a fresh translations file in `translations/{LANGUAGE_CODE}/`! Now you can begin populating it with your fresh translations! 🥖
 
-You'll also need to add an entry for it in `translations/index.js`. You can follow the pattern used in that file for `en` and `es`.
+You'll also need to add an entry for it in [`translations/index.js`](../translations/index.js) and [`translations/dev.js`](../translations/dev.js). You can follow the pattern used in that file for `en` and `es`.
+
+### Testing changes
+The bulk of translations are not included in the final library, but are hosted online and are automatically downloaded by p5.js when it needs them. Updates to these only happen whenever a new version of p5.js is released. 
+
+However, if you want to see your changes (or any other changes which aren't released yet), you can simply run `npm run dev` which will build p5.js configured to use the translation files present locally on your computer, instead of the ones on the internet.
 
 ### Further Reading
 

package.json

@@ -4,7 +4,7 @@
   "scripts": {
     "grunt": "grunt",
     "build": "grunt build",
-    "dev": "grunt browserify connect:yui watch:quick",
+    "dev": "grunt browserify:dev connect:yui watch:quick",
     "docs": "grunt yui",
     "docs:dev": "grunt yui:dev",
     "test": "grunt",
@@ -94,7 +94,8 @@
     "lib/p5.min.js",
     "lib/p5.js",
     "lib/addons/p5.sound.js",
-    "lib/addons/p5.sound.min.js"
+    "lib/addons/p5.sound.min.js",
+    "translations/**"
   ],
   "description": "[![npm version](https://badge.fury.io/js/p5.svg)](https://www.npmjs.com/package/p5)",
   "bugs": {

src/core/internationalization.js

@@ -1,12 +1,107 @@
 import i18next from 'i18next';
 import LanguageDetector from 'i18next-browser-languagedetector';
 
-let resources;
-// Do not include translations in the minified js
+let fallbackResources, languages;
 if (typeof IS_MINIFIED === 'undefined') {
-  resources = require('../../translations').default;
+  // internationalization is only for the unminified build
+
+  const translationsModule = require('../../translations');
+  fallbackResources = translationsModule.default;
+  languages = translationsModule.languages;
+
+  if (typeof P5_DEV_BUILD !== 'undefined') {
+    // When the library is built in development mode ( using npm run dev )
+    // we want to use the current translation files on the disk, which may have
+    // been updated but not yet pushed to the CDN.
+    let completeResources = require('../../translations/dev');
+    for (const language of Object.keys(completeResources)) {
+      // In es_translation, language is es and namespace is translation
+      // In es_MX_translation, language is es-MX and namespace is translation
+      const parts = language.split('_');
+      const lng = parts.slice(0, parts.length - 1).join('-');
+      const ns = parts[parts.length - 1];
+
+      fallbackResources[lng] = fallbackResources[lng] || {};
+      fallbackResources[lng][ns] = completeResources[language];
+    }
+  }
 }
 
+/**
+ * This is our i18next "backend" plugin. It tries to fetch languages
+ * from a CDN.
+ */
+class FetchResources {
+  constructor(services, options) {
+    this.init(services, options);
+  }
+
+  // run fetch with a timeout. Automatically rejects on timeout
+  // default timeout = 2000 ms
+  fetchWithTimeout(url, options, timeout = 2000) {
+    return Promise.race([
+      fetch(url, options),
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('timeout')), timeout)
+      )
+    ]);
+  }
+
+  init(services, options = {}) {
+    this.services = services;
+    this.options = options;
+  }
+
+  read(language, namespace, callback) {
+    const loadPath = this.options.loadPath;
+
+    if (language === this.options.fallback) {
+      // if the default language of the user is the same as our inbuilt fallback,
+      // there's no need to fetch resources from the cdn. This won't actually
+      // need to run when we use "partialBundledLanguages" in the init
+      // function.
+      callback(null, fallbackResources[language][namespace]);
+    } else if (languages.includes(language)) {
+      // The user's language is included in the list of languages
+      // that we so far added translations for.
+
+      const url = this.services.interpolator.interpolate(loadPath, {
+        lng: language,
+        ns: namespace
+      });
+      this.loadUrl(url, callback);
+    } else {
+      // We don't have translations for this language. i18next will use
+      // the default language instead.
+      callback('Not found', false);
+    }
+  }
+
+  loadUrl(url, callback) {
+    this.fetchWithTimeout(url)
+      .then(
+        response => {
+          const ok = response.ok;
+
+          if (!ok) {
+            // caught in the catch() below
+            throw new Error(`failed loading ${url}`);
+          }
+          return response.json();
+        },
+        () => {
+          // caught in the catch() below
+          throw new Error(`failed loading ${url}`);
+        }
+      )
+      .then(data => {
+        return callback(null, data);
+      })
+      .catch(callback);
+  }
+}
+FetchResources.type = 'backend';
+
 /**
  * This is our translation function. Give it a key and
  * it will retreive the appropriate string
@@ -18,38 +113,52 @@ if (typeof IS_MINIFIED === 'undefined') {
  * @returns {String} message (with values inserted) in the user's browser language
  * @private
  */
-export let translator = () => {
+export let translator = (key, values) => {
   console.debug('p5.js translator called before translations were loaded');
-  return '';
+
+  // Certain FES functionality may trigger before translations are downloaded.
+  // Using "partialBundledLanguages" option during initialization, we can
+  // still use our fallback language to display messages
+  i18next.t(key, values); /* i18next-extract-disable-line */
 };
 // (We'll set this to a real value in the init function below!)
 
 /**
  * Set up our translation function, with loaded languages
  */
-export const initialize = () =>
-  new Promise((resolve, reject) => {
-    i18next
-      .use(LanguageDetector)
-      .init({
-        fallbackLng: 'en',
-        nestingPrefix: '$tr(',
-        nestingSuffix: ')',
-        defaultNS: 'translation',
-        returnEmptyString: false,
-        interpolation: {
-          escapeValue: false
-        },
-        detection: {
-          checkWhitelist: false
-        },
-        resources
-      })
-      .then(
-        translateFn => {
-          translator = translateFn;
-          resolve();
-        },
-        e => reject(`Translations failed to load (${e})`)
-      );
-  });
+export const initialize = () => {
+  let i18init = i18next
+    .use(LanguageDetector)
+    .use(FetchResources)
+    .init({
+      fallbackLng: 'en',
+      nestingPrefix: '$tr(',
+      nestingSuffix: ')',
+      defaultNS: 'translation',
+      returnEmptyString: false,
+      interpolation: {
+        escapeValue: false
+      },
+      detection: {
+        checkWhitelist: false
+      },
+      backend: {
+        fallback: 'en',
+        loadPath:
+          'https://cdn.jsdelivr.net/npm/p5/translations/{{lng}}/{{ns}}.json'
+      },
+      partialBundledLanguages: true,
+      resources: fallbackResources
+    })
+    .then(
+      translateFn => {
+        translator = translateFn;
+      },
+      e => console.debug(`Translations failed to load (${e})`)
+    );
+
+  // i18next.init() returns a promise that resolves when the translations
+  // are loaded. We use this in core/init.js to hold p5 initialization until
+  // we have the translation files.
+  return i18init;
+};

tasks/build/browserify.js

@@ -19,6 +19,8 @@ module.exports = function(grunt) {
     function(param) {
       const isMin = param === 'min';
       const isTest = param === 'test';
+      const isDev = param === 'dev';
+
       const filename = isMin
         ? 'p5.pre-min.js'
         : isTest ? 'p5-test.js' : 'p5.js';
@@ -32,16 +34,21 @@ module.exports = function(grunt) {
       // Render the banner for the top of the file
       const banner = grunt.template.process(bannerTemplate);
 
+      let globalVars = {};
+      if (isDev) {
+        globalVars['P5_DEV_BUILD'] = () => true;
+      }
       // Invoke Browserify programatically to bundle the code
-      let browseified = browserify(srcFilePath, {
-        standalone: 'p5'
+      let browserified = browserify(srcFilePath, {
+        standalone: 'p5',
+        insertGlobalVars: globalVars
       });
 
       if (isMin) {
         // These paths should be the exact same as what are used in the import
         // statements in the source. They are not relative to this file. It's
         // just how browserify works apparently.
-        browseified = browseified
+        browserified = browserified
           .exclude('../../docs/reference/data.json')
           .exclude('../../../docs/parameterData.json')
           .exclude('../../translations')
@@ -50,13 +57,17 @@ module.exports = function(grunt) {
           .ignore('i18next-browser-languagedetector');
       }
 
+      if (!isDev) {
+        browserified = browserified.exclude('../../translations/dev');
+      }
+
       const babelifyOpts = { plugins: ['static-fs'] };
 
       if (isTest) {
         babelifyOpts.envName = 'test';
       }
 
-      const bundle = browseified.transform('babelify', babelifyOpts).bundle();
+      const bundle = browserified.transform('babelify', babelifyOpts).bundle();
 
       // Start the generated output with the banner comment,
       let code = banner + '\n';

translations/dev.js

@@ -0,0 +1,15 @@
+export { default as en_translation } from './en/translation';
+export { default as es_translation } from './es/translation';
+
+/** 
+ * When adding a new language, add a new "export" statement above this.
+ * For example, if we were to add fr ( French ), we would write:
+ * export { default as fr_translation } from './fr/translation';
+ *
+ * If the language key has a hypen (-), replace it with an underscore ( _ )
+ * e.g. for es-MX we would write:
+ * export { default as es_MX_translation } from './es-MX/translation';
+ * 
+ * "es_MX" is the language key whereas "translation" is the filename
+ * ( translation.json ) or the namespace
+*/

translations/index.js

@@ -1,18 +1,27 @@
 import en from './en/translation';
-import es from './es/translation';
+
+// Only one language is imported above. This is intentional as other languages
+// will be hosted online and then downloaded whenever needed
 
 /**
- * Maps our translations to their language key
- * (`en` is english, `es` es español)
- *
- * `translation` is the namespace we're using for
- * our initial set of translation strings.
+ * Here, we define a default/fallback language which we can use without internet.
+ * You won't have to change this when adding a new language.
+ * 
+ * `translation` is the namespace we are using for our initial set of strings
  */
 export default {
   en: {
     translation: en
-  },
-  es: {
-    translation: es
   }
 };
+
+/**
+ * This is a list of languages that we have added so far.
+ * If you have just added a new language (yay!), add its key to the list below
+ * (`en` is english, `es` es español). Also add its export to 
+ * dev.js, which is another file in this folder.
+ */
+export const languages = [
+  'en',
+  'es'
+];