diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index 8c745bef3071cbfadd576e89610860a4fb7c0712..8495f5577a6617e31affd811a93b27038295aadb 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -119,8 +119,8 @@ jobs: platform: "${{ matrix.arch }}" source_dir: "artifacts/keyman-srcpkg" sourcepackage: "keyman_${{ needs.sourcepackage.outputs.VERSION }}-1.dsc" - deb_fullname: $DEBFULLNAME - deb_email: $DEBEMAIL + deb_fullname: ${{env.DEBFULLNAME}} + deb_email: ${{env.DEBEMAIL}} prerelease_tag: ${{ needs.sourcepackage.outputs.PRERELEASE_TAG }} - name: Output resulting .deb files diff --git a/HISTORY.md b/HISTORY.md index c05c241eff96740ca9e41e287d1a851045aaf025..50d64334643faebc41b87d6c8af2d042dab78340 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,37 @@ # Keyman Version History +## 17.0.118 alpha 2023-06-05 + +* chore(ios): replace fv cert (#8900) +* fix(core): Fix compilation if hotdoc is installed (#8912) +* feat(linux): Rename column and add tooltip (#8918) + +## 17.0.117 alpha 2023-06-04 + +* chore(developer): verify long lines compile correctly (#8915) + +## 17.0.116 alpha 2023-06-02 + +* feat(linux): Add column for installation location (#8897) +* chore(developer): verify kvks files and report errors (#8892) +* refactor(developer): rearrange kmcmplib interface source (#8899) +* refactor(developer): move filename consistency check to kmc (#8907) +* chore(developer): loadFile callback error check and optimization (#8908) +* chore(common): remove url module ref from common/web/types (#8914) + +## 17.0.115 alpha 2023-06-01 + +* refactor(developer): complete fs move out of kmcmplib (#8882) +* fix(common): tweak pack/publish support for npm 9.5.1 and node 18.16.0 (#8894) +* feat(linux): Implement Options page and option to disable Sentry error reporting (#7989) + +## 17.0.114 alpha 2023-05-31 + +* chore(developer): replace cwrap wasm bindings (#8857) +* chore(developer): refactor kmcmplib interfaces (#8870) +* chore(developer): move keyboard repo fixtures (#8874) +* chore(linux): Move build steps to build.sh (#8864) + ## 17.0.113 alpha 2023-05-26 * docs(android): Update documentation for building Android on Linux (#8860) diff --git a/VERSION.md b/VERSION.md index a3a3dc809e3ff7ff42f204abcc79812de3a7b640..7d14d88a17924860f9933506ff2ae81ec0be6d1f 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -17.0.114 \ No newline at end of file +17.0.119 \ No newline at end of file diff --git a/android/.gitignore b/android/.gitignore index 09e84019e45cb0f92efbbb0e099af0ca4bf1ce69..ad30f1a79d8e55e1c31b759a4620188ca6a93956 100644 --- a/android/.gitignore +++ b/android/.gitignore @@ -34,6 +34,9 @@ gradle_models.ser KMEA/**/assets/ajax-loader.gif KMEA/**/assets/keymanandroid.js KMEA/**/assets/keyman.js.map +KMEA/**/assets/keymanweb-webview.js +KMEA/**/assets/keymanweb-webview.js.map +KMEA/**/assets/sentry.min.js KMEA/**/assets/keyman-sentry.js KMEA/**/assets/es6-shim.min.js KMEA/**/assets/kmwosk.css diff --git a/android/KMEA/app/src/main/assets/android-host.js b/android/KMEA/app/src/main/assets/android-host.js index 1bfcb0b1d660f1581e5914ed7f1876073cca076d..25e4223032e861c401d1775a1e2a4e31c41c6989 100644 --- a/android/KMEA/app/src/main/assets/android-host.js +++ b/android/KMEA/app/src/main/assets/android-host.js @@ -10,7 +10,7 @@ var oskHeight = Math.ceil(window.jsInterface.getKeyboardHeight() / window.device var oskWidth = 0; var fragmentToggle = 0; -var sentryManager = new com.keyman.KeymanSentryManager({ +var sentryManager = new KeymanSentryManager({ hostPlatform: "android" }); sentryManager.init(); @@ -25,24 +25,24 @@ function init() { //document.body.style.backgroundColor="transparent"; //window.console.log('Device type = '+device); //window.console.log('Keyboard height = '+oskHeight); - keyman.init({'app':device,'fonts':'packages/',root:'./'}); - keyman.util.setOption('attachType','manual'); - keyman.oninserttext = insertText; keyman.showKeyboardList = showMenu; - keyman.menuKeyUp = menuKeyUp; keyman.hideKeyboard = hideKeyboard; + keyman.menuKeyUp = menuKeyUp; keyman.getOskHeight = getOskHeight; keyman.getOskWidth = getOskWidth; keyman.beepKeyboard = beepKeyboard; - var ta = document.getElementById('ta'); - keyman.setActiveElement(ta); - - ta.readOnly = false; - checkTextArea(); - - // Tell KMW the default banner height to use - com.keyman.osk.Banner.DEFAULT_HEIGHT = - Math.ceil(window.jsInterface.getDefaultBannerHeight() / window.devicePixelRatio); + keyman.init({ + 'embeddingApp':device, + 'fonts':'packages/', + oninserttext: insertText, + root:'./' + }).then(function () { // Note: For non-upgraded API 21, arrow functions will break the keyboard! + const bannerHeight = Math.ceil(window.jsInterface.getDefaultBannerHeight() / window.devicePixelRatio); + + // The OSK is not available until initialization is complete. + keyman.osk.bannerView.activeBannerHeight = bannerHeight; + keyman.refreshOskLayout(); + }); keyman.addEventListener('keyboardloaded', setIsChiral); keyman.addEventListener('keyboardchange', setIsChiral); @@ -71,7 +71,7 @@ function setBannerHeight(h) { osk.banner.height = Math.ceil(h / window.devicePixelRatio); } // Refresh KMW OSK - keyman.correctOSKTextSize(); + keyman.refreshOskLayout(); } function setOskHeight(h) { @@ -81,7 +81,7 @@ function setOskHeight(h) { if(keyman && keyman.core && keyman.core.activeKeyboard) { keyman.core.activeKeyboard.refreshLayouts(); } - keyman.correctOSKTextSize(); + keyman.refreshOskLayout(); } function setOskWidth(w) { @@ -111,7 +111,7 @@ function onStateChange(change) { //window.console.log('onStateChange change: ' + change); // Refresh KMW OSK - keyman.correctOSKTextSize(); + keyman.refreshOskLayout(); fragmentToggle = (fragmentToggle + 1) % 100; if(change != 'configured') { // doesn't change the display; only initiates suggestions. @@ -129,16 +129,11 @@ function setIsChiral(keyboardProperties) { function setKeymanLanguage(k) { KeymanWeb.registerStub(k); - keyman.setActiveKeyboard(k.KP + '::'+k.KI, k.KLC); - keyman.osk.show(true); + keyman.setActiveKeyboard(k.KI, k.KLC); } function setSpacebarText(mode) { - keyman.options['spacebarText'] = mode; - keyman.osk.show(true); - - // Refresh KMW OSK - keyman.correctOSKTextSize(); + keyman.config.spacebarText = mode; } // #6665: we need to know when the user has pressed a hardware key so we don't @@ -164,29 +159,25 @@ function insertText(dn, s, dr) { } function deregisterModel(modelID) { - keyman.modelManager.deregister(modelID); + keyman.removeModel(modelID); } function enableSuggestions(model, mayPredict, mayCorrect) { // Set the options first so that KMW's ModelManager can properly handle model enablement states // the moment we actually register the new model. - keyman.osk.banner.setOptions({ - 'mayPredict': mayPredict, - 'mayCorrect': mayCorrect - }); + keyman.core.languageProcessor.mayPredict = mayPredict; + keyman.core.languageProcessor.mayCorrect = mayCorrect; registerModel(model); } function setBannerOptions(mayPredict) { - keyman.osk.banner.setOptions({ - 'mayPredict': mayPredict - }); + keyman.core.languageProcessor.mayPredict = mayPredict; } function registerModel(model) { //window.console.log('registerModel: ' + model); - keyman.registerModel(model); + keyman.addModel(model); } function resetContext() { @@ -201,16 +192,15 @@ function setNumericLayer() { } function updateKMText(text) { - var ta = document.getElementById('ta'); - console_debug('updateKMText(text='+text+') ta.value='+ta.value); - if(text == undefined) { text = ''; } - if(ta.value != text) { - ta.value = text; - window.resetContext(); + console_debug('updateKMText(text='+text+') context.value='+keyman.context.getText()); + + if(text != keyman.context.getText()) { + keyman.context.setText(text); + keyman.resetContext(); } } @@ -221,22 +211,19 @@ function console_debug(s) { } function updateKMSelectionRange(start, end) { - var ta = document.getElementById('ta'); - console_debug('updateKMSelectionRange('+start+','+end+'): ta.selectionStart='+ta.selectionStart+' '+ - '['+ta._KeymanWebSelectionStart+'] ta.selectionEnd='+ta.selectionEnd+' '+ta._KeymanWebSelectionEnd); + var context = keyman.context; + + // console_debug('updateKMSelectionRange('+start+','+end+'): context.selStart='+ta.selectionStart+' '+ + // '['+ta._KeymanWebSelectionStart+'] context.selEnd='+ta.selectionEnd+' '+ta._KeymanWebSelectionEnd); - var selDirection = 'forward'; if(start > end) { var e0 = end; end = start; start = e0; - selDirection = 'backward'; } - if(ta.selectionStart != start || ta.selectionEnd != end || ta.selectionDirection != selDirection) { - ta.selectionStart = ta._KeymanWebSelectionStart = start; - ta.selectionEnd = ta._KeymanWebSelectionEnd = end; - ta.selectionDirection = selDirection; + if(context.selStart != start || context.selEnd != end) { + keyman.context.setSelection(start, end); keyman.resetContext(); } } @@ -330,7 +317,7 @@ function hideKeyboard() { function showKeyboard() { // Refresh KMW OSK - keyman.correctOSKTextSize(); + keyman.refreshOskLayout(); } function executePopupKey(keyID, keyText) { @@ -339,19 +326,24 @@ function executePopupKey(keyID, keyText) { keyman.executePopupKey(keyID); } +// Cannot make it explicitly async / await on API 21. function executeHardwareKeystroke(code, shift, lstates, eventModifiers) { console_debug('executeHardwareKeystroke(code='+code+',shift='+shift+',lstates='+lstates+',eventModifiers='+eventModifiers+')'); - try { - executingHardwareKeystroke = true; - if (keyman.executeHardwareKeystroke(code, shift, lstates)) { // false if matched, true if not + + executingHardwareKeystroke = true; + + // Would be cleaner if we could async / await here, which would give us a simple try-catch implementation. + var promise = keyman.hardKeyboard.raiseKeyEvent(code, shift, lstates); + promise.then(function (result) { + if(result) { // false if matched, true if not // KMW didn't process the key, so have the Android app dispatch the key with the original event modifiers window.jsInterface.dispatchKey(code, eventModifiers); + executingHardwareKeystroke = false; } - executingHardwareKeystroke = false; - } catch(e) { + }).catch(function (e) { window.console.log('executeHardwareKeystroke exception: '+e); executingHardwareKeystroke = false; - } + }); } function popupVisible(value) { @@ -373,24 +365,3 @@ function toHex(theString) { } return hexString; } - -/** - * Check the WebView version and determine if the textarea that KeymanWeb uses needs to be "visible". - * Normally, this textarea is not displayed to avoid redundant layout calculations. - * In older WebViews on Android 5.0 though, selectionStart and selectionEnd positions fail to - * update unless the textarea is visible. - * Reference: Issue #5376 - */ -function checkTextArea() { - var uaRe = /Chrome\/([0-9]*)\./g; - var chromeMajorVersion = uaRe.exec(navigator.userAgent); - if (chromeMajorVersion && parseInt(chromeMajorVersion[1]) <= 37) { - var ta = document.getElementById('ta'); - if (ta != null) { - ta.style.display = ''; - ta.style.position = 'absolute'; - ta.style.left = '-500px'; - ta.style.top = '0px'; - } - } -} diff --git a/android/KMEA/app/src/main/assets/keyboard.html b/android/KMEA/app/src/main/assets/keyboard.html index e677ba4c46ec95cec1d0dbbafb670b6951b25482..b11a21124eca991f2417bb409e3d11cf11182c32 100644 --- a/android/KMEA/app/src/main/assets/keyboard.html +++ b/android/KMEA/app/src/main/assets/keyboard.html @@ -15,7 +15,8 @@ polyfill. --> - + + - diff --git a/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java b/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java index ebb8727bef53e5c39457f7f5bfeb1fcad89d7e82..7af3c2cf9672cd11a978dd625b926fc7f00082ea 100644 --- a/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java +++ b/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java @@ -272,9 +272,10 @@ public final class KMManager { // Keyman files protected static final String KMFilename_KeyboardHtml = "keyboard.html"; - protected static final String KMFilename_JSEngine = "keymanandroid.js"; - protected static final String KMFilename_JSEngine_Sourcemap = "keyman.js.map"; - protected static final String KMFilename_JSSentry = "keyman-sentry.js"; + protected static final String KMFilename_JSEngine = "keymanweb-webview.js"; + protected static final String KMFilename_JSEngine_Sourcemap = "keymanweb-webview.js.map"; + protected static final String KMFilename_JSSentry = "sentry.min.js"; + protected static final String KMFilename_JSSentryInit = "keyman-sentry.js"; protected static final String KMFilename_AndroidHost = "android-host.js"; protected static final String KMFilename_KmwCss = "kmwosk.css"; protected static final String KMFilename_KmwGlobeHintCss = "globe-hint.css"; @@ -796,6 +797,7 @@ public final class KMManager { copyAsset(context, KMFilename_KeyboardHtml, "", true); copyAsset(context, KMFilename_JSEngine, "", true); copyAsset(context, KMFilename_JSSentry, "", true); + copyAsset(context, KMFilename_JSSentryInit, "", true); copyAsset(context, KMFilename_AndroidHost, "", true); if(KMManager.isDebugMode()) { copyAsset(context, KMFilename_JSEngine_Sourcemap, "", true); diff --git a/android/KMEA/build.sh b/android/KMEA/build.sh index c479d9a13375a226ef115a6820c5c3efecb14855..ff78403c03ef0992d44d3e96fe22e26a6fe24a3e 100755 --- a/android/KMEA/build.sh +++ b/android/KMEA/build.sh @@ -29,7 +29,8 @@ TEST_FLAGS="-x aR lintRelease testRelease" # Gradle test w/o build JUNIT_RESULTS="##teamcity[importData type='junit' path='keyman\android\KMEA\app\build\test-results\testReleaseUnitTest\']" builder_describe "Builds Keyman Engine for Android." \ - "@/web" \ + "@/web/src/app/webview" \ + "@/common/web/sentry-manager" \ "clean" \ "configure" \ "build" \ @@ -49,7 +50,6 @@ if builder_is_debug_build; then fi builder_describe_outputs \ - configure:engine /android/KMEA/app/src/main/assets/keymanandroid.js \ build:engine /android/KMEA/app/build/outputs/aar/keyman-engine-${CONFIG}.aar #### Build @@ -82,14 +82,15 @@ if builder_start_action build:engine; then # Copy KeymanWeb artifacts echo "Copying Keyman Web artifacts" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/osk/ajax-loader.gif" "$ENGINE_ASSETS/ajax-loader.gif" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/keyman.js" "$ENGINE_ASSETS/keymanandroid.js" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/keyman.js.map" "$ENGINE_ASSETS/keyman.js.map" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/osk/kmwosk.css" "$ENGINE_ASSETS/kmwosk.css" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/osk/globe-hint.css" "$ENGINE_ASSETS/globe-hint.css" - cp "$KEYMAN_WEB_ROOT/build/app/embed/$CONFIG/osk/keymanweb-osk.ttf" "$ENGINE_ASSETS/keymanweb-osk.ttf" - - cp "$KEYMAN_ROOT/common/web/sentry-manager/build/index.js" "$ENGINE_ASSETS/keyman-sentry.js" + cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.js" "$ENGINE_ASSETS/keymanweb-webview.js" + cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.js.map" "$ENGINE_ASSETS/keymanweb-webview.js.map" + cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/ajax-loader.gif" "$ENGINE_ASSETS/ajax-loader.gif" + cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/kmwosk.css" "$ENGINE_ASSETS/kmwosk.css" + cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/globe-hint.css" "$ENGINE_ASSETS/globe-hint.css" + cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/keymanweb-osk.ttf" "$ENGINE_ASSETS/keymanweb-osk.ttf" + + cp "$KEYMAN_ROOT/node_modules/@sentry/browser/build/bundle.min.js" "$ENGINE_ASSETS/sentry.min.js" + cp "$KEYMAN_ROOT/common/web/sentry-manager/build/lib/index.js" "$ENGINE_ASSETS/keyman-sentry.js" echo "Copying es6-shim polyfill" cp "$KEYMAN_ROOT/node_modules/es6-shim/es6-shim.min.js" "$ENGINE_ASSETS/es6-shim.min.js" diff --git a/android/Tests/KeyboardHarness/app/src/main/assets/keyboardharness.kmp b/android/Tests/KeyboardHarness/app/src/main/assets/keyboardharness.kmp index 9db5e29cc24510d3493b9d38ad6e812e45dd006a..3a8f1cb624cd78f89c8104361f9287581c4dde49 100644 Binary files a/android/Tests/KeyboardHarness/app/src/main/assets/keyboardharness.kmp and b/android/Tests/KeyboardHarness/app/src/main/assets/keyboardharness.kmp differ diff --git a/common/models/templates/build-bundler.js b/common/models/templates/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..49693c0d2a66b8f5a94f96e50d78b2da9b6fae2c --- /dev/null +++ b/common/models/templates/build-bundler.js @@ -0,0 +1,33 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +await esbuild.build({ + bundle: true, + sourcemap: true, + format: "esm", + nodePaths: ['..'], + entryPoints: ['build/obj/index.js'], + outfile: "build/lib/index.mjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); + + +// Bundled CommonJS (classic Node) module version +esbuild.buildSync({ + bundle: true, + sourcemap: true, + format: "cjs", + nodePaths: ['..'], + entryPoints: ['build/obj/index.js'], + outfile: "build/lib/index.cjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/common/models/templates/build.sh b/common/models/templates/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..0fe305ca5ad6dbc337deb59dad00554b7e3406c5 --- /dev/null +++ b/common/models/templates/build.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Compile our sourcemap-path remapping module for use by Web builds, releases, etc. +# +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# This script runs from its own folder +cd "$(dirname "$THIS_SCRIPT")" + +################################ Main script ################################ + +builder_describe "Builds the predictive-text model template implementation module" \ + "@/common/web/keyman-version" \ + "@/common/web/tslib" \ + "@/common/models/wordbreakers" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + "--ci" + +builder_describe_outputs \ + configure /node_modules \ + build build/lib/index.mjs + +builder_parse "$@" + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +### CLEAN ACTIONS + +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### BUILD ACTIONS + +if builder_start_action build; then + npm run tsc -- -b + node build-bundler.js + + # Declaration bundling. + npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/index.d.ts + + builder_finish_action success build +fi + +if builder_start_action test; then + FLAGS= + if builder_has_option --ci; then + FLAGS="-reporter mocha-teamcity-reporter" + fi + + npm run mocha -- $FLAGS --require test/helpers.js --recursive test + + builder_finish_action success test +fi \ No newline at end of file diff --git a/common/models/templates/package.json b/common/models/templates/package.json index 67699e342a93ecf2ab13046640bc8472f87d5351..f815aa1f94a7dc678e9e7a5b088cf2a324117c29 100644 --- a/common/models/templates/package.json +++ b/common/models/templates/package.json @@ -16,8 +16,19 @@ ], "homepage": "https://github.com/keymanapp/keyman#readme", "license": "MIT", - "main": "index.js", - "types": "index.d.ts", + "main": "./build/obj/index.js", + "types": "./build/obj/index.d.ts", + "exports": { + ".": "./build/obj/index.js", + "./lib": { + "types": "./build/lib/index.d.ts", + "import": "./build/lib/index.mjs", + "require": "./build/lib/index.cjs" + }, + "./obj/*.js": "./build/obj/*.js" + }, + + "type": "module", "directories": { "test": "test" }, @@ -32,8 +43,9 @@ "url": "git+https://github.com/keymanapp/keyman.git" }, "scripts": { - "build": "tsc", - "pretest": "tsc -b tsconfig.bundled.json", + "build": "gosh ./build.sh", + "tsc": "tsc", + "mocha": "mocha", "test": "mocha -r test/helpers.js" }, "bugs": { @@ -47,9 +59,13 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "typescript": "^4.9.5" }, "dependencies": { - "@keymanapp/models-wordbreakers": "*" + "@keymanapp/keyman-version": "*", + "@keymanapp/models-types": "*", + "@keymanapp/models-wordbreakers": "*", + "@keymanapp/resources-gosh": "*" } } diff --git a/common/models/templates/src/common.ts b/common/models/templates/src/common.ts index 9117ec7d6f4c40df3b29133926e005ee2b021119..9f3eae1dc1a8c23db83682073d7dac795dd03c76 100644 --- a/common/models/templates/src/common.ts +++ b/common/models/templates/src/common.ts @@ -1,148 +1,151 @@ -namespace models { - export const SENTINEL_CODE_UNIT = '\uFDD0'; - - export function applyTransform(transform: Transform, context: Context): Context { - // First, get the current context - let fullLeftContext = context.left || ''; - let lLen = fullLeftContext.kmwLength(); - let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft; - - let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || ''); - - let fullRightContext = context.right || ''; - let rLen = fullRightContext.kmwLength(); - let rDel = rLen < transform.deleteRight ? rLen : transform.deleteRight; - - let rightContext = fullRightContext.kmwSubstr(rDel); - - return { - left: leftContext, - right: rightContext, - startOfBuffer: context.startOfBuffer, - endOfBuffer: context.endOfBuffer, - casingForm: context.casingForm - }; - } +// Allows the kmwstring bindings to resolve. +import { extendString } from "@keymanapp/web-utils"; - /** - * Merges two Transforms as if they were applied to a `Context` successively. - * @param first - * @param second - */ - export function buildMergedTransform(first: Transform, second: Transform): Transform { - // These exist to avoid parameter mutation. - let mergedFirstInsert: string = first.insert; - let mergedSecondDelete: number = second.deleteLeft; - - // The 'fun' case: the second Transform wants to delete something from the first. - if(second.deleteLeft) { - let firstLength = first.insert.kmwLength(); - if(firstLength <= second.deleteLeft) { - mergedFirstInsert = ''; - mergedSecondDelete = second.deleteLeft - firstLength; - } else { - mergedFirstInsert = first.insert.kmwSubstr(0, firstLength - second.deleteLeft); - mergedSecondDelete = 0; - } - } +extendString(); - return { - insert: mergedFirstInsert + second.insert, - deleteLeft: first.deleteLeft + mergedSecondDelete, - // As `first` would affect the context before `second` could take effect, - // this is the correct way to merge `deleteRight`. - deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0) - } - } +export const SENTINEL_CODE_UNIT = '\uFDD0'; - /** - * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate. - * - * @param char A single JavaScript (UCS-2) char corresponding to a single code unit. - */ - export function isHighSurrogate(char: string): boolean; - /** - * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate. - * - * @param codeUnit A code unit corresponding to a single UCS-2 char. - */ - export function isHighSurrogate(codeUnit: number): boolean; - export function isHighSurrogate(codeUnit: string|number): boolean { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } +export function applyTransform(transform: Transform, context: Context): Context { + // First, get the current context + let fullLeftContext = context.left || ''; + let lLen = fullLeftContext.kmwLength(); + let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft; - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; - } + let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || ''); - /** - * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate. - * - * @param char A single JavaScript (UCS-2) char corresponding to a single code unit. - */ - export function isLowSurrogate(char: string): boolean; - /** - * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate. - * - * @param codeUnit A code unit corresponding to a single UCS-2 char. - */ - export function isLowSurrogate(codeUnit: number): boolean; - export function isLowSurrogate(codeUnit: string|number): boolean { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); + let fullRightContext = context.right || ''; + let rLen = fullRightContext.kmwLength(); + let rDel = (rLen < (transform.deleteRight ?? 0)) ? rLen : (transform.deleteRight ?? 0); + + let rightContext = fullRightContext.kmwSubstr(rDel); + + return { + left: leftContext, + right: rightContext, + startOfBuffer: context.startOfBuffer, + endOfBuffer: context.endOfBuffer, + casingForm: context.casingForm + }; +} + +/** + * Merges two Transforms as if they were applied to a `Context` successively. + * @param first + * @param second + */ +export function buildMergedTransform(first: Transform, second: Transform): Transform { + // These exist to avoid parameter mutation. + let mergedFirstInsert: string = first.insert; + let mergedSecondDelete: number = second.deleteLeft; + + // The 'fun' case: the second Transform wants to delete something from the first. + if(second.deleteLeft) { + let firstLength = first.insert.kmwLength(); + if(firstLength <= second.deleteLeft) { + mergedFirstInsert = ''; + mergedSecondDelete = second.deleteLeft - firstLength; + } else { + mergedFirstInsert = first.insert.kmwSubstr(0, firstLength - second.deleteLeft); + mergedSecondDelete = 0; } + } - return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + return { + insert: mergedFirstInsert + second.insert, + deleteLeft: first.deleteLeft + mergedSecondDelete, + // As `first` would affect the context before `second` could take effect, + // this is the correct way to merge `deleteRight`. + deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0) } +} - export function isSentinel(char: string): boolean { - return char == models.SENTINEL_CODE_UNIT; +/** + * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate. + * + * @param char A single JavaScript (UCS-2) char corresponding to a single code unit. + */ +export function isHighSurrogate(char: string): boolean; +/** + * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate. + * + * @param codeUnit A code unit corresponding to a single UCS-2 char. + */ +export function isHighSurrogate(codeUnit: number): boolean; +export function isHighSurrogate(codeUnit: string|number): boolean { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); } + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; +} + /** - * Builds a Suggestion based on a Transform corresponding to a predictive-text op. - * - * Assumes that the Transform's `insert` property represents a completed word, - * as models generally delete the whole prefix, replacing it with the full lexical entry. - * @param transform - */ - export function transformToSuggestion(transform: Transform): Suggestion; - export function transformToSuggestion(transform: Transform, p: number): WithOutcome; - export function transformToSuggestion(transform: Transform, p?: number): Outcome { - let suggestion: Outcome = { - transform: transform, - transformId: transform.id, - displayAs: transform.insert - }; - - if(p === 0 || p) { - suggestion.p = p; - } - return suggestion; + * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate. + * + * @param char A single JavaScript (UCS-2) char corresponding to a single code unit. + */ +export function isLowSurrogate(char: string): boolean; +/** + * Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate. + * + * @param codeUnit A code unit corresponding to a single UCS-2 char. + */ +export function isLowSurrogate(codeUnit: number): boolean; +export function isLowSurrogate(codeUnit: string|number): boolean { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; +} + +export function isSentinel(char: string): boolean { + return char == SENTINEL_CODE_UNIT; +} + +/** + * Builds a Suggestion based on a Transform corresponding to a predictive-text op. + * + * Assumes that the Transform's `insert` property represents a completed word, + * as models generally delete the whole prefix, replacing it with the full lexical entry. + * @param transform + */ +export function transformToSuggestion(transform: Transform): Suggestion; +export function transformToSuggestion(transform: Transform, p: number): WithOutcome; +export function transformToSuggestion(transform: Transform, p?: number): Outcome { + let suggestion: Outcome = { + transform: transform, + transformId: transform.id, + displayAs: transform.insert + }; + + if(p === 0 || p) { + suggestion.p = p; } + return suggestion; +} - export function defaultApplyCasing(casing: CasingForm, text: string): string { - switch(casing) { - case 'lower': - return text.toLowerCase(); - case 'upper': - return text.toUpperCase(); - case 'initial': - // The length of the first code unit, as measured in code points. - let headUnitLength = 1; - - // Is the first character a high surrogate, indicating possible use of UTF-16 - // surrogate pairs? Also, is the string long enough for there to BE a pair? - if(text.length > 1 && isHighSurrogate(text.charAt(0))) { - // It's possible, so now we check for low surrogates. - if(isLowSurrogate(text.charCodeAt(1))) { - // We have a surrogate pair; this pair is the 'first' character. - headUnitLength = 2; - } +export function defaultApplyCasing(casing: CasingForm, text: string): string { + switch(casing) { + case 'lower': + return text.toLowerCase(); + case 'upper': + return text.toUpperCase(); + case 'initial': + // The length of the first code unit, as measured in code points. + let headUnitLength = 1; + + // Is the first character a high surrogate, indicating possible use of UTF-16 + // surrogate pairs? Also, is the string long enough for there to BE a pair? + if(text.length > 1 && isHighSurrogate(text.charAt(0))) { + // It's possible, so now we check for low surrogates. + if(isLowSurrogate(text.charCodeAt(1))) { + // We have a surrogate pair; this pair is the 'first' character. + headUnitLength = 2; } + } - // Capitalizes the first code unit of the string, leaving the rest intact. - return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength)); - } + // Capitalizes the first code unit of the string, leaving the rest intact. + return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength)); } } diff --git a/common/models/templates/src/index.ts b/common/models/templates/src/index.ts index a20b43f60284806d2f8c14dbc7e685b4949c48dc..f828066846b72d8ef14a4482598935759cb744d5 100644 --- a/common/models/templates/src/index.ts +++ b/common/models/templates/src/index.ts @@ -1,7 +1,22 @@ -/// -/// +import { + SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, + transformToSuggestion, defaultApplyCasing +} from "./common.js"; +import PriorityQueue, { Comparator } from "./priority-queue.js"; +import QuoteBehavior from "./quote-behavior.js"; +import { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js"; +import TrieModel, { TrieModelOptions } from "./trie-model.js"; -// Add all namespaces defined here to the global scope: -if (typeof module != 'undefined' && typeof module.exports != 'undefined') { - module.exports['models'] = models; -} +import { extendString } from "@keymanapp/web-utils"; + +// This package requires our string-extension functions. +extendString(); + +export { + SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, + transformToSuggestion, defaultApplyCasing, // "common.ts" + PriorityQueue, Comparator, // "priority-queue.ts" + QuoteBehavior, // "quote-behavior.ts", + Tokenization, tokenize, getLastPreCaretToken, wordbreak, // "tokenization.ts" + TrieModel, TrieModelOptions // "trie-model.ts" +}; diff --git a/common/models/templates/src/priority-queue.ts b/common/models/templates/src/priority-queue.ts index c372d881c58074ad3caf1e502066d163244a697c..8ecae0533f37b808886f946d2f5f74ffe1fe728a 100644 --- a/common/models/templates/src/priority-queue.ts +++ b/common/models/templates/src/priority-queue.ts @@ -4,216 +4,215 @@ * Defines a mildly abstracted priority queue implementation. */ -namespace models { +/** + * Used to compare two instances of a type. + * @returns + * - value < 0 if `a` should come before `b` + * - value > 0 if `b` should come before `a` + * - 0 if they should be treated equally. + */ +export type Comparator = (a: Type, b: Type) => number; + +export default class PriorityQueue { + + private comparator: Comparator; + private heap: Type[]; + /** - * Used to compare two instances of a type. - * @returns - * - value < 0 if `a` should come before `b` - * - value > 0 if `b` should come before `a` - * - 0 if they should be treated equally. + * Constructs an empty priority queue. + * @param comparator A `Comparator` returning negative values when and only when + * the first parameter should precede the second parameter. */ - export type Comparator = (a: Type, b: Type) => number; - - export class PriorityQueue { - - private comparator: Comparator; - private heap: Type[]; - - /** - * Constructs an empty priority queue. - * @param comparator A `Comparator` returning negative values when and only when - * the first parameter should precede the second parameter. - */ - constructor(comparator: Comparator, initialEntries: Type[] = []) { - // TODO: We may wish to allow options specifying a limit or threshold for adding - // items to the priority queue. Possibly both. - // - // When that time comes, consider a min-max heap. - // https://en.wikipedia.org/wiki/Min-max_heap - this.comparator = comparator; - - this.heap = Array.from(initialEntries); - this.heapify(); - } + constructor(comparator: Comparator, initialEntries: Type[] = []) { + // TODO: We may wish to allow options specifying a limit or threshold for adding + // items to the priority queue. Possibly both. + // + // When that time comes, consider a min-max heap. + // https://en.wikipedia.org/wiki/Min-max_heap + this.comparator = comparator; + + this.heap = Array.from(initialEntries); + this.heapify(); + } - private static leftChildIndex(index: number): number { - return index * 2 + 1; - } + private static leftChildIndex(index: number): number { + return index * 2 + 1; + } - private static rightChildIndex(index: number): number { - return index * 2 + 2; - } + private static rightChildIndex(index: number): number { + return index * 2 + 2; + } - private static parentIndex(index: number): number { - return Math.floor((index-1)/2); - } + private static parentIndex(index: number): number { + return Math.floor((index-1)/2); + } - /** - * Maintains internal state, rearranging the internal state until all heap constraints - * are properly satisfied. - * - O(N) when 'heapifying' the whole heap - * - O(N) worst-case for partial heap operations (as part of an enqueueAll) - *

- */ - private heapify(): void; - private heapify(start: number, end: number): void; - private heapify(start?: number, end?: number): void { - if(start == undefined || end == undefined) { - this.heapify(0, this.count - 1); - } + /** + * Maintains internal state, rearranging the internal state until all heap constraints + * are properly satisfied. + * - O(N) when 'heapifying' the whole heap + * - O(N) worst-case for partial heap operations (as part of an enqueueAll) + *

+ */ + private heapify(): void; + private heapify(start: number, end: number): void; + private heapify(start?: number, end?: number): void { + if(start == undefined || end == undefined) { + this.heapify(0, this.count - 1); + return; + } - // Use of 'indices' here is a bit of a customization. - // At the cost of (temporary) extra storage space, we can more efficiently enqueue - // multiple elements simultaneously. - let queuedIndices: number[] = []; - let lastParent = -1; - - for(let i = end; i >= start; i--) { - let parent = PriorityQueue.parentIndex(i); - if(this.siftDown(i) && parent < start && lastParent != parent) { - // We only need to queue examination for a heap node if its children have changed - // and it isn't already being examined. - queuedIndices.push(parent); - lastParent = parent; - } + // Use of 'indices' here is a bit of a customization. + // At the cost of (temporary) extra storage space, we can more efficiently enqueue + // multiple elements simultaneously. + let queuedIndices: number[] = []; + let lastParent = -1; + + for(let i = end; i >= start; i--) { + let parent = PriorityQueue.parentIndex(i); + if(this.siftDown(i) && parent < start && lastParent != parent) { + // We only need to queue examination for a heap node if its children have changed + // and it isn't already being examined. + queuedIndices.push(parent); + lastParent = parent; } + } - lastParent = -1; - while(queuedIndices.length > 0) { - let index = queuedIndices.shift(); - let parent = PriorityQueue.parentIndex(index); - if(this.siftDown(index) && parent >= 0 && lastParent != parent) { - // We only need to queue examination for a heap node if its children have changed. - queuedIndices.push(parent); - lastParent = parent; - } + lastParent = -1; + while(queuedIndices.length > 0) { + let index = queuedIndices.shift() as number; + let parent = PriorityQueue.parentIndex(index); + if(this.siftDown(index) && parent >= 0 && lastParent != parent) { + // We only need to queue examination for a heap node if its children have changed. + queuedIndices.push(parent); + lastParent = parent; } } + } - /** - * Returns the number of elements currently held by the priority queue. - */ - get count(): number { - return this.heap.length; - } + /** + * Returns the number of elements currently held by the priority queue. + */ + get count(): number { + return this.heap.length; + } - /** - * Returns the highest-priority item within the priority queue. - *

- * Is O(1). - */ - peek() { - return this.heap[0]; // undefined if it doesn't exist... which is completely correct. - } + /** + * Returns the highest-priority item within the priority queue. + *

+ * Is O(1). + */ + peek() { + return this.heap[0]; // undefined if it doesn't exist... which is completely correct. + } - /** - * Inserts a new element into the priority queue, placing it in order. - *

- * Is O(log N), where N = # of items in the priority queue. - * @param element - */ - enqueue(element: Type) { - let index = this.heap.length; - this.heap.push(element); - - let parent = PriorityQueue.parentIndex; - let parentIndex = parent(index); - while(index !== 0 && this.comparator(this.heap[index], this.heap[parentIndex]) < 0) { - let a = this.heap[index]; - this.heap[index] = this.heap[parentIndex]; - this.heap[parentIndex] = a; - - index = parentIndex; - parentIndex = parent(index); - } + /** + * Inserts a new element into the priority queue, placing it in order. + *

+ * Is O(log N), where N = # of items in the priority queue. + * @param element + */ + enqueue(element: Type) { + let index = this.heap.length; + this.heap.push(element); + + let parent = PriorityQueue.parentIndex; + let parentIndex = parent(index); + while(index !== 0 && this.comparator(this.heap[index], this.heap[parentIndex]) < 0) { + let a = this.heap[index]; + this.heap[index] = this.heap[parentIndex]; + this.heap[parentIndex] = a; + + index = parentIndex; + parentIndex = parent(index); } + } - /** - * Efficiently batch-enqueues multiple elements. - * Worst-case is the _better_ of the following: - * - O(`elements.count` + `heap.count`) - large element counts will trigger in-place - * heap reconstruction. - * - O(`elements.count` * log(`heap.count`)) - logarithmic when elements.count << heap.count - * @param elements A group of elements to enqueue simultaneously. - */ - enqueueAll(elements: Type[]) { - if(elements.length == 0) { - return; - } - - let firstIndex = this.count - this.heap = this.heap.concat(elements); - let firstParent = PriorityQueue.parentIndex(firstIndex); - - // The 'parent' of index 0 will return -1, which is illegal. - this.heapify(firstParent >= 0 ? firstParent : 0, PriorityQueue.parentIndex(this.count-1)); + /** + * Efficiently batch-enqueues multiple elements. + * Worst-case is the _better_ of the following: + * - O(`elements.count` + `heap.count`) - large element counts will trigger in-place + * heap reconstruction. + * - O(`elements.count` * log(`heap.count`)) - logarithmic when elements.count << heap.count + * @param elements A group of elements to enqueue simultaneously. + */ + enqueueAll(elements: Type[]) { + if(elements.length == 0) { + return; } - /** - * Removes the highest-priority element from the queue, returning it. - *

- * Is O(log N), where N = number of items in the priority queue. - */ - dequeue(): Type { - if(this.count == 0) { - return undefined; - } + let firstIndex = this.count + this.heap = this.heap.concat(elements); + let firstParent = PriorityQueue.parentIndex(firstIndex); - const root = this.heap[0]; - let tail = this.heap.pop(); - if(this.heap.length > 0) { - this.heap[0] = tail; - this.siftDown(0); - } + // The 'parent' of index 0 will return -1, which is illegal. + this.heapify(firstParent >= 0 ? firstParent : 0, PriorityQueue.parentIndex(this.count-1)); + } - return root; + /** + * Removes the highest-priority element from the queue, returning it. + *

+ * Is O(log N), where N = number of items in the priority queue. + */ + dequeue(): Type | undefined { + if(this.count == 0) { + return undefined; } - /** - * Compares the entry at the specified index against its children, - * propagating it downward within the heap until heap requirements are specified. - *

- * Is O(log N), where N = number of items in the priority queue. - * - * @param index The index of the top-most node that must be examined - * for repositioning. - * @returns `true` if a swap occurred, `false` otherwise. - */ - private siftDown(index: number): boolean { - let leftIndex = PriorityQueue.leftChildIndex(index); - let rightIndex = PriorityQueue.rightChildIndex(index); - let topMostIndex = index; - - if(leftIndex < this.heap.length && this.comparator(this.heap[leftIndex], this.heap[topMostIndex]) < 0) { - topMostIndex = leftIndex; - } + const root = this.heap[0]; + let tail = this.heap.pop() as Type; + if(this.heap.length > 0) { + this.heap[0] = tail; + this.siftDown(0); + } - if(rightIndex < this.heap.length && this.comparator(this.heap[rightIndex], this.heap[topMostIndex]) < 0) { - topMostIndex = rightIndex; - } + return root; + } - if(topMostIndex != index) { - let a = this.heap[index]; - this.heap[index] = this.heap[topMostIndex]; - this.heap[topMostIndex] = a; + /** + * Compares the entry at the specified index against its children, + * propagating it downward within the heap until heap requirements are specified. + *

+ * Is O(log N), where N = number of items in the priority queue. + * + * @param index The index of the top-most node that must be examined + * for repositioning. + * @returns `true` if a swap occurred, `false` otherwise. + */ + private siftDown(index: number): boolean { + let leftIndex = PriorityQueue.leftChildIndex(index); + let rightIndex = PriorityQueue.rightChildIndex(index); + let topMostIndex = index; - this.siftDown(topMostIndex); - return true; - } else { - return false; - } + if(leftIndex < this.heap.length && this.comparator(this.heap[leftIndex], this.heap[topMostIndex]) < 0) { + topMostIndex = leftIndex; } - /** - * Returns an array containing all entries of the priority queue. - * Altering the returned array will not affect the queue, but mutating - * the array's elements can cause unintended side effects. - * - * This function makes no guarantees on the ordering of the returned elements; - * they will almost certainly be unsorted. - */ - toArray(): Type[] { - return Array.from(this.heap); + if(rightIndex < this.heap.length && this.comparator(this.heap[rightIndex], this.heap[topMostIndex]) < 0) { + topMostIndex = rightIndex; } + + if(topMostIndex != index) { + let a = this.heap[index]; + this.heap[index] = this.heap[topMostIndex]; + this.heap[topMostIndex] = a; + + this.siftDown(topMostIndex); + return true; + } else { + return false; + } + } + + /** + * Returns an array containing all entries of the priority queue. + * Altering the returned array will not affect the queue, but mutating + * the array's elements can cause unintended side effects. + * + * This function makes no guarantees on the ordering of the returned elements; + * they will almost certainly be unsorted. + */ + toArray(): Type[] { + return Array.from(this.heap); } -} +} \ No newline at end of file diff --git a/common/models/templates/src/quote-behavior.ts b/common/models/templates/src/quote-behavior.ts index 7a95d04c49617afa382caf89f500cc736c38833e..2ca7bbd04c413e4d8d35d44d97ea414fb7421b33 100644 --- a/common/models/templates/src/quote-behavior.ts +++ b/common/models/templates/src/quote-behavior.ts @@ -1,38 +1,38 @@ -namespace models { - export enum QuoteBehavior { - noQuotes = "no-quotes", - useQuotes = "use-quotes", - default = "default-quotes" - } - - // For an "enum/namespace merge". See https://www.typescriptlang.org/docs/handbook/declaration-merging.html#merging-namespaces-with-classes - export namespace QuoteBehavior { - /** - * Applies the specified quote behavior to the provided `Transform`, mutating it as appropriate. - * @param transform The Transform to be mutated - * @param punctuation The active `LexicalModelPunctuation` settings - * @param defaultTo The default quote behavior to use (in case the current value is `.default`) - */ - export function apply(behavior: QuoteBehavior, text: string, punctuation: LexicalModelPunctuation, defaultTo: QuoteBehavior): string { - if(defaultTo == QuoteBehavior.default || !defaultTo) { - throw "Specified quote behavior may be ambiguous - default behavior not specified (may not be .default)"; - } - - if(behavior == QuoteBehavior.default) { - behavior = defaultTo; - } - - switch(behavior) { - case QuoteBehavior.noQuotes: - return text; - case QuoteBehavior.useQuotes: - let {open, close} = punctuation.quotesForKeepSuggestion; - - // This part's simple enough, at least. - return open + text + close; - default: - throw "Unsupported quote behavior state detected; implementation missing!"; - } +export enum QuoteBehavior { + noQuotes = "no-quotes", + useQuotes = "use-quotes", + default = "default-quotes" +} + +// For an "enum/namespace merge". See https://www.typescriptlang.org/docs/handbook/declaration-merging.html#merging-namespaces-with-classes +export namespace QuoteBehavior { + /** + * Applies the specified quote behavior to the provided `Transform`, mutating it as appropriate. + * @param transform The Transform to be mutated + * @param punctuation The active `LexicalModelPunctuation` settings + * @param defaultTo The default quote behavior to use (in case the current value is `.default`) + */ + export function apply(behavior: QuoteBehavior, text: string, punctuation: LexicalModelPunctuation, defaultTo: QuoteBehavior): string { + if(defaultTo == QuoteBehavior.default || !defaultTo) { + throw "Specified quote behavior may be ambiguous - default behavior not specified (may not be .default)"; + } + + if(behavior == QuoteBehavior.default) { + behavior = defaultTo; + } + + switch(behavior) { + case QuoteBehavior.noQuotes: + return text; + case QuoteBehavior.useQuotes: + let {open, close} = punctuation.quotesForKeepSuggestion; + + // This part's simple enough, at least. + return open + text + close; + default: + throw "Unsupported quote behavior state detected; implementation missing!"; } } -} \ No newline at end of file +} + +export default QuoteBehavior; \ No newline at end of file diff --git a/common/models/templates/src/tokenization.ts b/common/models/templates/src/tokenization.ts index 9993050eab3dc86bbc8f1ed92f8a74ec97ae4b2e..2aa5c392e6d923df44954b0cf6bef3cf7453cbdb 100644 --- a/common/models/templates/src/tokenization.ts +++ b/common/models/templates/src/tokenization.ts @@ -1,129 +1,127 @@ // While we _could_ define this within @keymanapp/models-wordbreakers instead, it's probably -// better to leave that package as _just_ the wordbreakers. - -namespace models { - export interface Tokenization { - /** - * An array of tokens to the left of the caret. If the caret is in the middle of a token, - * only the part to the left of the caret is included. - */ - left: USVString[], - - /** - * An array of tokens to the right of the caret. If the caret is in the middle of a token, - * only the part to the right of the caret is included. - */ - right: USVString[], - - /** - * A flag indicating whether or not the caret's position in the context caused a token - * to be split. If `true`, the last entry of `left` is from the same token as the first - * entry of `right`. - */ - caretSplitsToken: boolean - } +// better to leave that package as _just_ the wordbreakers. - export function tokenize(wordBreaker: WordBreakingFunction, context?: Partial): Tokenization { - context = context || { - left: undefined, - startOfBuffer: undefined, - endOfBuffer: undefined - }; +export interface Tokenization { + /** + * An array of tokens to the left of the caret. If the caret is in the middle of a token, + * only the part to the left of the caret is included. + */ + left: USVString[], - let leftSpans = wordBreaker(context.left || '') || []; - let rightSpans = wordBreaker(context.right || '') || []; + /** + * An array of tokens to the right of the caret. If the caret is in the middle of a token, + * only the part to the right of the caret is included. + */ + right: USVString[], - let leftTail: Span; - if(leftSpans.length > 0) { - leftTail = leftSpans[leftSpans.length - 1]; - } + /** + * A flag indicating whether or not the caret's position in the context caused a token + * to be split. If `true`, the last entry of `left` is from the same token as the first + * entry of `right`. + */ + caretSplitsToken: boolean +} + +export function tokenize(wordBreaker: WordBreakingFunction, context?: Partial): Tokenization { + context = context || { + left: undefined, + startOfBuffer: undefined, + endOfBuffer: undefined + }; + + let leftSpans = wordBreaker(context.left || '') || []; + let rightSpans = wordBreaker(context.right || '') || []; + + let leftTail: Span; + if(leftSpans.length > 0) { + leftTail = leftSpans[leftSpans.length - 1]; + } - // Handle any desired special handling for directly-pre-caret scenarios - where for this - // _specific_ context, we should not make a token division where one normally would exist otherwise. - // - // One notable example: word-final apostrophe is tokenized separate from preceding text, but - // word-internal apostrophe is treated as part of the same word (i.e, English contractions). - // But, if the user is editing text and the caret is directly after a caret, there's a notable - // chance they may in the middle of typing a contraction. Refer to - // https://github.com/keymanapp/keyman/issues/6572. - if(leftSpans.length > 1) { - const leftTailBase = leftSpans[leftSpans.length - 2]; - - // If the final two pre-caret spans are adjacent - without intervening whitespace... - if(leftTailBase.end == leftTail.start) { - // Ideal: if(leftTailBase is standard-char-class && leftTail is single-quote-class) - // But we don't have character class access here; it's all wordbreaker-function internal. - // Upon inspection of the wordbreaker data definitions... the single-quote-class is ONLY "'". - // So... we'll just be lazy for now and append the `'`. - if(leftTail.text == "'") { - let mergedSpan: Span = { - text: leftTailBase.text + leftTail.text, - start: leftTailBase.start, - end: leftTail.end, - length: leftTailBase.length + leftTail.length - }; - - leftSpans.pop(); // leftTail - leftSpans.pop(); // leftTailBase - leftSpans.push(mergedSpan); - leftTail = mergedSpan; // don't forget to update the `leftTail` Span! - } + // Handle any desired special handling for directly-pre-caret scenarios - where for this + // _specific_ context, we should not make a token division where one normally would exist otherwise. + // + // One notable example: word-final apostrophe is tokenized separate from preceding text, but + // word-internal apostrophe is treated as part of the same word (i.e, English contractions). + // But, if the user is editing text and the caret is directly after a caret, there's a notable + // chance they may in the middle of typing a contraction. Refer to + // https://github.com/keymanapp/keyman/issues/6572. + if(leftSpans.length > 1) { + const leftTailBase = leftSpans[leftSpans.length - 2]; + + // If the final two pre-caret spans are adjacent - without intervening whitespace... + if(leftTailBase.end == leftTail!.start) { + // Ideal: if(leftTailBase is standard-char-class && leftTail is single-quote-class) + // But we don't have character class access here; it's all wordbreaker-function internal. + // Upon inspection of the wordbreaker data definitions... the single-quote-class is ONLY "'". + // So... we'll just be lazy for now and append the `'`. + if(leftTail!.text == "'") { + let mergedSpan: Span = { + text: leftTailBase.text + leftTail!.text, + start: leftTailBase.start, + end: leftTail!.end, + length: leftTailBase.length + leftTail!.length + }; + + leftSpans.pop(); // leftTail + leftSpans.pop(); // leftTailBase + leftSpans.push(mergedSpan); + leftTail = mergedSpan; // don't forget to update the `leftTail` Span! } } + } - // With any 'detokenization' cases already handled, we may now begin to build the return object. - let tokenization: Tokenization = { - left: leftSpans.map(span => span.text), - right: rightSpans.map(span => span.text), - - // A default initialization of the value. - caretSplitsToken: false - }; - - // Now the hard part - determining whether or not the caret caused a token split. - if(leftSpans.length > 0 && rightSpans.length > 0) { - let rightHead = rightSpans[0]; - - // If tokenization includes all characters on each side of the caret, - // we have a good candidate for a caret-splitting scenario. - let leftSuffixWordbreak = leftTail.end != context.left.length; - let rightPrefixWordbreak = rightHead.start != 0; - - if(leftSuffixWordbreak || rightPrefixWordbreak) { - // Bypass the final test, as we already know the caret didn't split a token. - // (The tokenization process already removed characters between the two.) - return tokenization; - } + // With any 'detokenization' cases already handled, we may now begin to build the return object. + let tokenization: Tokenization = { + left: leftSpans.map(span => span.text), + right: rightSpans.map(span => span.text), - // Worth note - some languages don't use wordbreaking characters. So, a final check: - // - // Does the wordbreaker split a merge of the 'two center' tokens? - // If not, then the caret is responsible for the split. - if(wordBreaker(leftTail.text + rightHead.text).length == 1) { - tokenization.caretSplitsToken = true; - } - } + // A default initialization of the value. + caretSplitsToken: false + }; - return tokenization; - } + // Now the hard part - determining whether or not the caret caused a token split. + if(leftSpans.length > 0 && rightSpans.length > 0) { + let rightHead = rightSpans[0]; - /** - * Get the last word of the phrase before the caret or nothing. - * @param fullLeftContext the entire left context of the string. - */ - export function getLastPreCaretToken(wordBreaker: WordBreakingFunction, context: Context): string { - let tokenization = tokenize(wordBreaker, context); - if (tokenization.left.length > 0) { - return tokenization.left.pop(); + // If tokenization includes all characters on each side of the caret, + // we have a good candidate for a caret-splitting scenario. + let leftSuffixWordbreak = leftTail!.end != context.left!.length; + let rightPrefixWordbreak = rightHead.start != 0; + + if(leftSuffixWordbreak || rightPrefixWordbreak) { + // Bypass the final test, as we already know the caret didn't split a token. + // (The tokenization process already removed characters between the two.) + return tokenization; } - return ''; + // Worth note - some languages don't use wordbreaking characters. So, a final check: + // + // Does the wordbreaker split a merge of the 'two center' tokens? + // If not, then the caret is responsible for the split. + if(wordBreaker(leftTail!.text + rightHead.text).length == 1) { + tokenization.caretSplitsToken = true; + } } - // While it is currently identical to getLastWord, this may change in the future. - // It's best not to write ourselves into a corner on this one, as disambiguating later - // would likely be pretty painful. - export function wordbreak(wordBreaker: WordBreakingFunction, context: Context): USVString { - return getLastPreCaretToken(wordBreaker, context); + return tokenization; +} + +/** + * Get the last word of the phrase before the caret or nothing. + * @param fullLeftContext the entire left context of the string. + */ +export function getLastPreCaretToken(wordBreaker: WordBreakingFunction, context: Context): string { + let tokenization = tokenize(wordBreaker, context); + if (tokenization.left.length > 0) { + return tokenization.left.pop() as string; } + + return ''; +} + +// While it is currently identical to getLastWord, this may change in the future. +// It's best not to write ourselves into a corner on this one, as disambiguating later +// would likely be pretty painful. +export function wordbreak(wordBreaker: WordBreakingFunction, context: Context): USVString { + return getLastPreCaretToken(wordBreaker, context); } \ No newline at end of file diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 4eb6b80afedfb9be959197202f9a458b74ee5d3b..990ab61dc6e5d1f950378c0930fc81924222b4f5 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -24,10 +24,14 @@ // Worth noting: we're starting to get quite a 'library' of common model/LMLayer functionality. // Should probably make a 'lm-utils' submodule. -/// -/// -/// -/// + +// Allows the kmwstring bindings to resolve. +import { extendString } from "@keymanapp/web-utils"; +import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; + +import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js"; +import { getLastPreCaretToken } from "./tokenization.js"; +import PriorityQueue from "./priority-queue.js"; /** * @file trie-model.ts @@ -35,524 +39,510 @@ * Defines a simple word list (unigram) model. */ - namespace models { - /** Upper bound on the amount of suggestions to generate. */ - const MAX_SUGGESTIONS = 12; +/** Upper bound on the amount of suggestions to generate. */ +const MAX_SUGGESTIONS = 12; +/** + * Additional arguments to pass into the model, in addition to the model + * parameters themselves. + */ +export interface TrieModelOptions { /** - * Additional arguments to pass into the model, in addition to the model - * parameters themselves. + * How to break words in a phrase. */ - interface TrieModelOptions { - /** - * How to break words in a phrase. - */ - wordBreaker?: WordBreakingFunction; - /** - * This should simplify a search term into a key. - */ - searchTermToKey?: (searchTerm: string) => string; - - /** - * Indicates that the model's written form has 'casing' behavior. - */ - languageUsesCasing?: boolean; - - /** - * Specifies a function used to apply the language's casing rules to a word. - */ - applyCasing?: CasingFunction; - - /** - * Any punctuation to expose to the user. - */ - punctuation?: LexicalModelPunctuation; - } - + wordBreaker?: WordBreakingFunction; /** - * Used to determine the probability of an entry from the trie. + * This should simplify a search term into a key. */ - type TextWithProbability = { - text: string; - // TODO: use negative-log scaling instead? - p: number; // real-number weight, from 0 to 1 - } + searchTermToKey?: (searchTerm: string) => string; /** - * @class TrieModel - * - * Defines a trie-backed word list model, or the unigram model. - * Unigram models throw away all preceding words, and search - * for the next word exclusively. As such, they can perform simple - * prefix searches within words, however they are not very good - * at predicting the next word. + * Indicates that the model's written form has 'casing' behavior. */ - export class TrieModel implements LexicalModel { - configuration: Configuration; - private _trie: Trie; - readonly breakWords: WordBreakingFunction; - readonly punctuation?: LexicalModelPunctuation; - readonly languageUsesCasing?: boolean; - - readonly applyCasing?: CasingFunction; - - constructor(trieData: object, options: TrieModelOptions = {}) { - this.languageUsesCasing = options.languageUsesCasing; - this.applyCasing = options.applyCasing; - - this._trie = new Trie( - trieData['root'], - trieData['totalWeight'], - options.searchTermToKey as Wordform2Key || defaultSearchTermToKey - ); - this.breakWords = options.wordBreaker || getDefaultWordBreaker(); - this.punctuation = options.punctuation; - } + languageUsesCasing?: boolean; - configure(capabilities: Capabilities): Configuration { - return this.configuration = { - leftContextCodePoints: capabilities.maxLeftContextCodePoints, - rightContextCodePoints: capabilities.maxRightContextCodePoints - }; - } + /** + * Specifies a function used to apply the language's casing rules to a word. + */ + applyCasing?: CasingFunction; - toKey(text: USVString): USVString { - return this._trie.toKey(text); - } + /** + * Any punctuation to expose to the user. + */ + punctuation?: LexicalModelPunctuation; +} - predict(transform: Transform, context: Context): Distribution { - // Special-case the empty buffer/transform: return the top suggestions. - if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) { - return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({ - transform: { - insert: text, - deleteLeft: 0 - }, - displayAs: text, - p: p - }))); - } +/** + * Used to determine the probability of an entry from the trie. + */ +type TextWithProbability = { + text: string; + // TODO: use negative-log scaling instead? + p: number; // real-number weight, from 0 to 1 +} - // Compute the results of the keystroke: - let newContext = models.applyTransform(transform, context); +/** + * @class TrieModel + * + * Defines a trie-backed word list model, or the unigram model. + * Unigram models throw away all preceding words, and search + * for the next word exclusively. As such, they can perform simple + * prefix searches within words, however they are not very good + * at predicting the next word. + */ +export default class TrieModel implements LexicalModel { + configuration?: Configuration; + private _trie: Trie; + readonly breakWords: WordBreakingFunction; + readonly punctuation?: LexicalModelPunctuation; + readonly languageUsesCasing?: boolean; + + readonly applyCasing?: CasingFunction; + + constructor(trieData: {root: Node, totalWeight: number}, options: TrieModelOptions = {}) { + this.languageUsesCasing = options.languageUsesCasing; + this.applyCasing = options.applyCasing; + + this._trie = new Trie( + trieData['root'], + trieData['totalWeight'], + options.searchTermToKey as Wordform2Key || defaultSearchTermToKey + ); + this.breakWords = options.wordBreaker || defaultWordBreaker; + this.punctuation = options.punctuation; + } - // Computes the different in word length after applying the transform above. - let leftDelOffset = transform.deleteLeft - transform.insert.kmwLength(); + configure(capabilities: Capabilities): Configuration { + return this.configuration = { + leftContextCodePoints: capabilities.maxLeftContextCodePoints, + rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0 + }; + } - // All text to the left of the cursor INCLUDING anything that has - // just been typed. - let prefix = models.getLastPreCaretToken(this.breakWords, newContext); + toKey(text: USVString): USVString { + return this._trie.toKey(text); + } - // Return suggestions from the trie. - return makeDistribution(this._trie.lookup(prefix).map(({text, p}) => - models.transformToSuggestion({ + predict(transform: Transform, context: Context): Distribution { + // Special-case the empty buffer/transform: return the top suggestions. + if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) { + return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({ + transform: { insert: text, - // Delete whatever the prefix that the user wrote. - deleteLeft: leftDelOffset + prefix.kmwLength() - // Note: a separate capitalization/orthography engine can take this - // result and transform it as needed. + deleteLeft: 0 }, - p - ))); + displayAs: text, + p: p + }))); + } - /* Helper */ + // Compute the results of the keystroke: + let newContext = applyTransform(transform, context); - function makeDistribution(suggestions: WithOutcome[]): Distribution { - let distribution: Distribution = []; + // Computes the different in word length after applying the transform above. + let leftDelOffset = transform.deleteLeft - transform.insert.kmwLength(); - for(let s of suggestions) { - distribution.push({sample: s, p: s.p}); - } + // All text to the left of the cursor INCLUDING anything that has + // just been typed. + let prefix = getLastPreCaretToken(this.breakWords, newContext); - return distribution; + // Return suggestions from the trie. + return makeDistribution(this._trie.lookup(prefix).map(({text, p}) => + transformToSuggestion({ + insert: text, + // Delete whatever the prefix that the user wrote. + deleteLeft: leftDelOffset + prefix.kmwLength() + // Note: a separate capitalization/orthography engine can take this + // result and transform it as needed. + }, + p + ))); + + /* Helper */ + + function makeDistribution(suggestions: WithOutcome[]): Distribution { + let distribution: Distribution = []; + + for(let s of suggestions) { + distribution.push({sample: s, p: s.p}); } - } - get wordbreaker(): WordBreakingFunction { - return this.breakWords; + return distribution; } + } - public traverseFromRoot(): LexiconTraversal { - return new TrieModel.Traversal(this._trie['root'], ''); - } + get wordbreaker(): WordBreakingFunction { + return this.breakWords; + } - private static Traversal = class implements LexiconTraversal { - /** - * The lexical prefix corresponding to the current traversal state. - */ - prefix: String; - - /** - * The current traversal node. Serves as the 'root' of its own sub-Trie, - * and we cannot navigate back to its parent. - */ - root: Node; - - constructor(root: Node, prefix: string) { - this.root = root; - this.prefix = prefix; - } + public traverseFromRoot(): LexiconTraversal { + return new TrieModel.Traversal(this._trie['root'], ''); + } - *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { - let root = this.root; - - if(root.type == 'internal') { - for(let entry of root.values) { - let entryNode = root.children[entry]; - - // UTF-16 astral plane check. - if(models.isHighSurrogate(entry)) { - // First code unit of a UTF-16 code point. - // For now, we'll just assume the second always completes such a char. - // - // Note: Things get nasty here if this is only sometimes true; in the future, - // we should compile-time enforce that this assumption is always true if possible. - if(entryNode.type == 'internal') { - let internalNode = entryNode; - for(let lowSurrogate of internalNode.values) { - let prefix = this.prefix + entry + lowSurrogate; - yield { - char: entry + lowSurrogate, - traversal: function() { return new TrieModel.Traversal(internalNode.children[lowSurrogate], prefix) } - } - } - } else { - // Determine how much of the 'leaf' entry has no Trie nodes, emulate them. - let fullText = entryNode.entries[0].key; - entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. - let prefix = this.prefix + entry; + private static Traversal = class implements LexiconTraversal { + /** + * The lexical prefix corresponding to the current traversal state. + */ + prefix: String; + + /** + * The current traversal node. Serves as the 'root' of its own sub-Trie, + * and we cannot navigate back to its parent. + */ + root: Node; + + constructor(root: Node, prefix: string) { + this.root = root; + this.prefix = prefix; + } + *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { + let root = this.root; + + if(root.type == 'internal') { + for(let entry of root.values) { + let entryNode = root.children[entry]; + + // UTF-16 astral plane check. + if(isHighSurrogate(entry)) { + // First code unit of a UTF-16 code point. + // For now, we'll just assume the second always completes such a char. + // + // Note: Things get nasty here if this is only sometimes true; in the future, + // we should compile-time enforce that this assumption is always true if possible. + if(entryNode.type == 'internal') { + let internalNode = entryNode; + for(let lowSurrogate of internalNode.values) { + let prefix = this.prefix + entry + lowSurrogate; yield { - char: entry, - traversal: function () {return new TrieModel.Traversal(entryNode, prefix)} + char: entry + lowSurrogate, + traversal: function() { return new TrieModel.Traversal(internalNode.children[lowSurrogate], prefix) } } } - } else if(models.isSentinel(entry)) { - continue; - } else if(!entry) { - // Prevent any accidental 'null' or 'undefined' entries from having an effect. - continue; } else { + // Determine how much of the 'leaf' entry has no Trie nodes, emulate them. + let fullText = entryNode.entries[0].key; + entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. let prefix = this.prefix + entry; + yield { char: entry, - traversal: function() { return new TrieModel.Traversal(entryNode, prefix)} + traversal: function () {return new TrieModel.Traversal(entryNode, prefix)} } } + } else if(isSentinel(entry)) { + continue; + } else if(!entry) { + // Prevent any accidental 'null' or 'undefined' entries from having an effect. + continue; + } else { + let prefix = this.prefix + entry; + yield { + char: entry, + traversal: function() { return new TrieModel.Traversal(entryNode, prefix)} + } } + } - return; - } else { // type == 'leaf' - let prefix = this.prefix; + return; + } else { // type == 'leaf' + let prefix = this.prefix; - let children = root.entries.filter(function(entry) { - return entry.key != prefix && prefix.length < entry.key.length; - }) + let children = root.entries.filter(function(entry) { + return entry.key != prefix && prefix.length < entry.key.length; + }) - for(let {key} of children) { - let nodeKey = key[prefix.length]; + for(let {key} of children) { + let nodeKey = key[prefix.length]; - if(models.isHighSurrogate(nodeKey)) { - // Merge the other half of an SMP char in! - nodeKey = nodeKey + key[prefix.length+1]; - } - yield { - char: nodeKey, - traversal: function() { return new TrieModel.Traversal(root, prefix + nodeKey)} - } - }; - return; - } + if(isHighSurrogate(nodeKey)) { + // Merge the other half of an SMP char in! + nodeKey = nodeKey + key[prefix.length+1]; + } + yield { + char: nodeKey, + traversal: function() { return new TrieModel.Traversal(root, prefix + nodeKey)} + } + }; + return; } + } - get entries(): string[] { - if(this.root.type == 'leaf') { - let prefix = this.prefix; - let matches = this.root.entries.filter(function(entry) { - return entry.key == prefix; - }); - - return matches.map(function(value) { return value.content }); + get entries(): string[] { + if(this.root.type == 'leaf') { + let prefix = this.prefix; + let matches = this.root.entries.filter(function(entry) { + return entry.key == prefix; + }); + + return matches.map(function(value) { return value.content }); + } else { + let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; + if(matchingLeaf && matchingLeaf.type == 'leaf') { + return matchingLeaf.entries.map(function(value) { return value.content }); } else { - let matchingLeaf = this.root.children[models.SENTINEL_CODE_UNIT]; - if(matchingLeaf && matchingLeaf.type == 'leaf') { - return matchingLeaf.entries.map(function(value) { return value.content }); - } else { - return []; - } + return []; } } } - }; + } +}; - ///////////////////////////////////////////////////////////////////////////////// - // What remains in this file is the trie implementation proper. Note: to // - // reduce bundle size, any functions/methods related to creating the trie have // - // been removed. // - ///////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////// +// What remains in this file is the trie implementation proper. Note: to // +// reduce bundle size, any functions/methods related to creating the trie have // +// been removed. // +///////////////////////////////////////////////////////////////////////////////// - /** - * An **opaque** type for a string that is exclusively used as a search key in - * the trie. There should be a function that converts arbitrary strings - * (queries) and converts them into a standard search key for a given language - * model. - * - * Fun fact: This opaque type has ALREADY saved my bacon and found a bug! - */ - type SearchKey = string & { _: 'SearchKey'}; +/** + * An **opaque** type for a string that is exclusively used as a search key in + * the trie. There should be a function that converts arbitrary strings + * (queries) and converts them into a standard search key for a given language + * model. + * + * Fun fact: This opaque type has ALREADY saved my bacon and found a bug! + */ +type SearchKey = string & { _: 'SearchKey'}; - /** - * The priority queue will always pop the most weighted item. There can only - * be two kinds of items right now: nodes, and entries; both having a weight - * attribute. - */ - type Weighted = Node | Entry; +/** + * The priority queue will always pop the most weighted item. There can only + * be two kinds of items right now: nodes, and entries; both having a weight + * attribute. + */ +type Weighted = Node | Entry; - /** - * A function that converts a string (word form or query) into a search key - * (secretly, this is also a string). - */ - interface Wordform2Key { - (wordform: string): SearchKey; - } +/** + * A function that converts a string (word form or query) into a search key + * (secretly, this is also a string). + */ +interface Wordform2Key { + (wordform: string): SearchKey; +} - // The following trie implementation has been (heavily) derived from trie-ing - // by Conrad Irwin. - // trie-ing is copyright (C) 2015–2017 Conrad Irwin. - // Distributed under the terms of the MIT license: - // https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE +// The following trie implementation has been (heavily) derived from trie-ing +// by Conrad Irwin. +// trie-ing is copyright (C) 2015–2017 Conrad Irwin. +// Distributed under the terms of the MIT license: +// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE - type Node = InternalNode | Leaf; +type Node = InternalNode | Leaf; +/** + * An internal node in the trie. Internal nodes NEVER contain entries; if an + * internal node should contain an entry, then it has a dummy leaf node (see + * below), that can be accessed by node.children["\uFDD0"]. + */ +interface InternalNode { + type: 'internal'; + weight: number; + /** Maintains the keys of children in descending order of weight. */ + values: string[]; // TODO: As an optimization, "values" can be a single string! /** - * An internal node in the trie. Internal nodes NEVER contain entries; if an - * internal node should contain an entry, then it has a dummy leaf node (see - * below), that can be accessed by node.children["\uFDD0"]. + * Maps a single UTF-16 code unit to a child node in the trie. This child + * node may be a leaf or an internal node. The keys of this object are kept + * in sorted order in the .values array. */ - interface InternalNode { - type: 'internal'; - weight: number; - /** Maintains the keys of children in descending order of weight. */ - values: string[]; // TODO: As an optimization, "values" can be a single string! - /** - * Maps a single UTF-16 code unit to a child node in the trie. This child - * node may be a leaf or an internal node. The keys of this object are kept - * in sorted order in the .values array. - */ - children: { [codeunit: string]: Node }; - } - /** Only leaf nodes actually contain entries (i.e., the words proper). */ - interface Leaf { - type: 'leaf'; - weight: number; - entries: Entry[]; - } + children: { [codeunit: string]: Node }; +} +/** Only leaf nodes actually contain entries (i.e., the words proper). */ +interface Leaf { + type: 'leaf'; + weight: number; + entries: Entry[]; +} + +/** + * An entry in the prefix trie (stored in leaf nodes exclusively!) + */ +interface Entry { + /** The actual word form, stored in the trie. */ + content: string; + /** A search key that usually simplifies the word form, for ease of search. */ + key: SearchKey; + weight: number; +} +/** + * Wrapper class for the trie and its nodes. + */ +class Trie { + private root: Node; + /** The total weight of the entire trie. */ + private totalWeight: number; /** - * An entry in the prefix trie (stored in leaf nodes exclusively!) + * Converts arbitrary strings to a search key. The trie is built up of + * search keys; not each entry's word form! */ - interface Entry { - /** The actual word form, stored in the trie. */ - content: string; - /** A search key that usually simplifies the word form, for ease of search. */ - key: SearchKey; - weight: number; + toKey: Wordform2Key; + + constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) { + this.root = root; + this.toKey = wordform2key; + this.totalWeight = totalWeight; } /** - * Wrapper class for the trie and its nodes. + * Lookups an arbitrary prefix (a query) in the trie. Returns the top 3 + * results in sorted order. + * + * @param prefix */ - class Trie { - private root: Node; - /** The total weight of the entire trie. */ - private totalWeight: number; - /** - * Converts arbitrary strings to a search key. The trie is built up of - * search keys; not each entry's word form! - */ - toKey: Wordform2Key; - - constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) { - this.root = root; - this.toKey = wordform2key; - this.totalWeight = totalWeight; - } - - /** - * Lookups an arbitrary prefix (a query) in the trie. Returns the top 3 - * results in sorted order. - * - * @param prefix - */ - lookup(prefix: string): TextWithProbability[] { - let searchKey = this.toKey(prefix); - let lowestCommonNode = findPrefix(this.root, searchKey); - if (lowestCommonNode === null) { - return []; - } - - return getSortedResults(lowestCommonNode, searchKey, this.totalWeight); + lookup(prefix: string): TextWithProbability[] { + let searchKey = this.toKey(prefix); + let lowestCommonNode = findPrefix(this.root, searchKey); + if (lowestCommonNode === null) { + return []; } - /** - * Returns the top N suggestions from the trie. - * @param n How many suggestions, maximum, to return. - */ - firstN(n: number): TextWithProbability[] { - return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n); - } + return getSortedResults(lowestCommonNode, searchKey, this.totalWeight); } /** - * Finds the deepest descendent in the trie with the given prefix key. - * - * This means that a search in the trie for a given prefix has a best-case - * complexity of O(m) where m is the length of the prefix. - * - * @param key The prefix to search for. - * @param index The index in the prefix. Initially 0. + * Returns the top N suggestions from the trie. + * @param n How many suggestions, maximum, to return. */ - function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null { - // An important note - the Trie itself is built on a per-JS-character basis, - // not on a UTF-8 character-code basis. - if (node.type === 'leaf' || index === key.length) { - return node; - } + firstN(n: number): TextWithProbability[] { + return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n); + } +} - // So, for SMP models, we need to match each char of the supplementary pair - // in sequence. Each has its own node in the Trie. - let char = key[index]; - if (node.children[char]) { - return findPrefix(node.children[char], key, index + 1); - } +/** + * Finds the deepest descendent in the trie with the given prefix key. + * + * This means that a search in the trie for a given prefix has a best-case + * complexity of O(m) where m is the length of the prefix. + * + * @param key The prefix to search for. + * @param index The index in the prefix. Initially 0. + */ +function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null { + // An important note - the Trie itself is built on a per-JS-character basis, + // not on a UTF-8 character-code basis. + if (node.type === 'leaf' || index === key.length) { + return node; + } - return null; + // So, for SMP models, we need to match each char of the supplementary pair + // in sequence. Each has its own node in the Trie. + let char = key[index]; + if (node.children[char]) { + return findPrefix(node.children[char], key, index + 1); } - /** - * Returns all entries matching the given prefix, in descending order of - * weight. - * - * @param prefix the prefix to match. - * @param results the current results - * @param queue - */ - function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] { - let queue = new PriorityQueue(function(a: Weighted, b: Weighted) { - // In case of Trie compilation issues that emit `null` or `undefined` - return (b ? b.weight : 0) - (a ? a.weight : 0); - }); - let results: TextWithProbability[] = []; - - if (node.type === 'leaf') { - // Assuming the values are sorted, we can just add all of the values in the - // leaf, until we reach the limit. - for (let item of node.entries) { - // String.startsWith is not supported on certain Android (5.0) devices we wish to support. - // Requires a minimum of Chrome 36, as opposed to 5.0's default of 35. - if (item.key.indexOf(prefix) == 0) { - let { content, weight } = item; - results.push({ - text: content, - p: weight / N - }); - - if (results.length >= limit) { - return results; - } + return null; +} + +/** + * Returns all entries matching the given prefix, in descending order of + * weight. + * + * @param prefix the prefix to match. + * @param results the current results + * @param queue + */ +function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] { + let queue = new PriorityQueue(function(a: Weighted, b: Weighted) { + // In case of Trie compilation issues that emit `null` or `undefined` + return (b ? b.weight : 0) - (a ? a.weight : 0); + }); + let results: TextWithProbability[] = []; + + if (node.type === 'leaf') { + // Assuming the values are sorted, we can just add all of the values in the + // leaf, until we reach the limit. + for (let item of node.entries) { + // String.startsWith is not supported on certain Android (5.0) devices we wish to support. + // Requires a minimum of Chrome 36, as opposed to 5.0's default of 35. + if (item.key.indexOf(prefix) == 0) { + let { content, weight } = item; + results.push({ + text: content, + p: weight / N + }); + + if (results.length >= limit) { + return results; } } - } else { - queue.enqueue(node); - let next: Weighted; - - while (next = queue.dequeue()) { - if (isNode(next)) { - // When a node is next up in the queue, that means that next least - // likely suggestion is among its decsendants. - // So we search all of its descendants! - if (next.type === 'leaf') { - queue.enqueueAll(next.entries); - } else { - // XXX: alias `next` so that TypeScript can be SURE that internal is - // in fact an internal node. Because of the callback binding to the - // original definition of node (i.e., a Node | Entry), this will not - // type-check otherwise. - let internal = next; - queue.enqueueAll(next.values.map(char => { - return internal.children[char]; - })); - } + } + } else { + queue.enqueue(node); + let next: Weighted | undefined; + + while (next = queue.dequeue()) { + if (isNode(next)) { + // When a node is next up in the queue, that means that next least + // likely suggestion is among its decsendants. + // So we search all of its descendants! + if (next.type === 'leaf') { + queue.enqueueAll(next.entries); } else { - // When an entry is up next in the queue, we just add its contents to - // the results! - results.push({ - text: next.content, - p: next.weight / N - }); - if (results.length >= limit) { - return results; - } + // XXX: alias `next` so that TypeScript can be SURE that internal is + // in fact an internal node. Because of the callback binding to the + // original definition of node (i.e., a Node | Entry), this will not + // type-check otherwise. + let internal = next; + queue.enqueueAll(next.values.map(char => { + return internal.children[char]; + })); + } + } else { + // When an entry is up next in the queue, we just add its contents to + // the results! + results.push({ + text: next.content, + p: next.weight / N + }); + if (results.length >= limit) { + return results; } } } - return results; - } + return results; - /** TypeScript type guard that returns whether the thing is a Node. */ - function isNode(x: Entry | Node): x is Node { - return 'type' in x; - } +} +/** TypeScript type guard that returns whether the thing is a Node. */ +function isNode(x: Entry | Node): x is Node { + return 'type' in x; +} + +/** + * Converts wordforms into an indexable form. It does this by + * normalizing into NFD, removing diacritics, and then converting + * the result to lowercase. + * + * This is a very naïve implementation, that I only think will work on + * some languages that use the Latin script. As of 2020-04-08, only + * 4 out of 11 (36%) of published language models use the Latin script, + * so this might not actually be a great default. + * + * This uses String.prototype.normalize() to convert normalize into NFD. + * NFD is an easy way to separate a Latin character from its diacritics; + * Even then, some Latin-based orthographies use code points that, + * under NFD normalization, do NOT decompose into an ASCII letter and a + * combining diacritical mark (e.g., SENĆOŦEN). + * + * Use this only in early iterations of the model. For a production lexical + * model, you SHOULD write/generate your own key function, tailored to your + * language. + */ +function defaultSearchTermToKey(wordform: string): SearchKey { /** - * Converts wordforms into an indexable form. It does this by - * normalizing into NFD, removing diacritics, and then converting - * the result to lowercase. - * - * This is a very naïve implementation, that I only think will work on - * some languages that use the Latin script. As of 2020-04-08, only - * 4 out of 11 (36%) of published language models use the Latin script, - * so this might not actually be a great default. + * N.B.: this is (slightly) DIFFERENT than the version in + * keymanapp/lexical-model-compiler/build-trie + * as this is for compatibility for models built + * BEFORE the searchTermToKey function was bundled with + * all models. * - * This uses String.prototype.normalize() to convert normalize into NFD. - * NFD is an easy way to separate a Latin character from its diacritics; - * Even then, some Latin-based orthographies use code points that, - * under NFD normalization, do NOT decompose into an ASCII letter and a - * combining diacritical mark (e.g., SENĆOŦEN). - * - * Use this only in early iterations of the model. For a production lexical - * model, you SHOULD write/generate your own key function, tailored to your - * language. + * This compatibility version lowercases AFTER removing diacritics; + * the new version (bundled in future models) lowercases, + * NFD normalizes, THEN removes diacritics. */ - function defaultSearchTermToKey(wordform: string): SearchKey { - /** - * N.B.: this is (slightly) DIFFERENT than the version in - * keymanapp/lexical-model-compiler/build-trie - * as this is for compatibility for models built - * BEFORE the searchTermToKey function was bundled with - * all models. - * - * This compatibility version lowercases AFTER removing diacritics; - * the new version (bundled in future models) lowercases, - * NFD normalizes, THEN removes diacritics. - */ - return wordform - .normalize('NFD') - // Remove all combining diacritics (if input is in NFD) - // common to Latin-orthographies. - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() as SearchKey; - } - - function getDefaultWordBreaker() { - let namespace: {}; - // @ts-ignore - if (typeof wordBreakers !== 'undefined') { - // @ts-ignore - namespace = wordBreakers; - } else { - namespace = require('@keymanapp/models-wordBreakers').wordBreakers; - } - return namespace['default']; - } + return wordform + .normalize('NFD') + // Remove all combining diacritics (if input is in NFD) + // common to Latin-orthographies. + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() as SearchKey; } diff --git a/common/models/templates/test/helpers.js b/common/models/templates/test/helpers.js index 268e24b7ddc15b2fe3b3575735134f0cd5b3afac..40aca6ffbb2d6986d3cc5b66ca70df0cf88d8d82 100644 --- a/common/models/templates/test/helpers.js +++ b/common/models/templates/test/helpers.js @@ -1,5 +1,5 @@ -const path = require('path'); -const assert = require('assert'); +import path from 'path'; +import { assert } from 'chai'; var _ = global; @@ -9,7 +9,12 @@ var _ = global; // Ensure that we can successfully load the module & apply kmwLength, as it's // needed for some of the unit tests. -require('../build/index.bundled.js'); + +// Verify that the KMW string extensions are loaded via side-effect. +import * as models from '@keymanapp/models-templates/lib'; + +import { createRequire } from "module"; +import { fileURLToPath } from 'url'; assert.ok('💩'.kmwLength); @@ -17,6 +22,13 @@ assert.ok('💩'.kmwLength); * Load JSON fixtures from a well-known place. */ _.jsonFixture = function (name) { + // The most straight-forward way... is to use CommonJS-style require to load JSON. + // Fortunately, Node provides the tools needed to recreate it. + const require = createRequire(import.meta.url); + + // ES-module mode also leaves out `__dirname`, so we rebuild that too. + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); return require(path.join(__dirname, 'fixtures', `${name}.json`)); } diff --git a/common/models/templates/test/test-common.js b/common/models/templates/test/test-common.js index 7dfc565369ae9a66e721dbbb560e16c78a742953..dfd5c875cb4f701e5a833ebc552be1a42ee5f8b4 100644 --- a/common/models/templates/test/test-common.js +++ b/common/models/templates/test/test-common.js @@ -2,8 +2,8 @@ * Unit tests for common utility functions/methods. */ -var assert = require('chai').assert; -var models = require('../build/index.bundled.js').models; +import { assert } from 'chai'; +import * as models from '@keymanapp/models-templates'; describe('Common utility functions', function() { // TODO: unit tests for other common utility functions diff --git a/common/models/templates/test/test-priority-queue.js b/common/models/templates/test/test-priority-queue.js index b004027d84a08c3701d13b626e2f669418801f4a..5fdd0caa3d4f1dab3044192845f97d563b474371 100644 --- a/common/models/templates/test/test-priority-queue.js +++ b/common/models/templates/test/test-priority-queue.js @@ -2,8 +2,8 @@ * Unit tests for the priority queue. */ -var assert = require('chai').assert; -var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue; +import { assert } from 'chai'; +import { PriorityQueue } from '@keymanapp/models-templates'; describe('Priority queue', function() { it('can act as a min-heap', function () { diff --git a/common/models/templates/test/test-quote-behavior.js b/common/models/templates/test/test-quote-behavior.js index dd5408f6a96f9e4b32a3c328d3494d3f1b4e7149..c6348d651d9de16d8075b13f2d1c4019448c7812 100644 --- a/common/models/templates/test/test-quote-behavior.js +++ b/common/models/templates/test/test-quote-behavior.js @@ -2,8 +2,8 @@ * Unit tests for quote behaviors. */ -var assert = require('chai').assert; -var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior; +import { assert } from 'chai'; +import { QuoteBehavior } from '@keymanapp/models-templates'; describe('Quote behaviors', function() { describe('Script directionality', function() { diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index a7ae0835e05daa19851ae3f7db4aafbed479066c..6c1ff746e58bf2e10ed1a3aaccb6be37bcbf217f 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -2,9 +2,9 @@ * Unit tests for common utility functions/methods. */ -var assert = require('chai').assert; -var models = require('../build/index.bundled.js').models; -var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers; +import { assert } from 'chai'; +import * as models from "@keymanapp/models-templates"; +import * as wordBreakers from "@keymanapp/models-wordbreakers"; describe('Tokenization functions', function() { describe('tokenize', function() { @@ -210,7 +210,7 @@ describe('Tokenization functions', function() { // Step 2: Allow shifting a defined 'constant' span without mutating the definition. let shiftSpan = function(span, delta) { // Avoid mutating the parameter! - shiftedSpan = { + let shiftedSpan = { text: span.text, start: span.start + delta, end: span.end + delta, diff --git a/common/models/templates/test/test-trie-model.js b/common/models/templates/test/test-trie-model.js index b48dc254ecc52740edc6c4d605df498bca1a393b..e63c66148b391f2e6cff2718d1f5dc1ca5cfdc20 100644 --- a/common/models/templates/test/test-trie-model.js +++ b/common/models/templates/test/test-trie-model.js @@ -2,8 +2,8 @@ * Unit tests for the Trie prediction model. */ -var assert = require('chai').assert; -var TrieModel = require('../build/index.bundled.js').models.TrieModel; +import { assert } from 'chai'; +import { TrieModel } from '@keymanapp/models-templates'; describe('LMLayerWorker trie model for word lists', function() { describe('instantiation', function () { diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 31f52f854a3dff84439edb108f24108ae76b4463..d3d04963165280562116aac8f1487ed8e2267561 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -2,8 +2,8 @@ * Unit tests for the Trie prediction model. */ -var assert = require('chai').assert; -var TrieModel = require('../build/index.bundled.js').models.TrieModel; +import { assert } from 'chai'; +import { TrieModel } from '@keymanapp/models-templates'; // Useful for tests related to strings with supplementary pairs. var smpForUnicode = function(code){ @@ -23,7 +23,7 @@ describe('Trie traversal abstractions', function() { let rootKeys = ['t', 'o', 'a', 'i', 'w', 'h', 'f', 'b', 'n', 'y', 's', 'm', 'u', 'c', 'd', 'l', 'e', 'j', 'p', 'g', 'v', 'k', 'r', 'q'] - for(child of rootTraversal.children()) { + for(let child of rootTraversal.children()) { let keyIndex = rootKeys.indexOf(child.char); assert.notEqual(keyIndex, -1); rootKeys.splice(keyIndex, 1); @@ -43,7 +43,7 @@ describe('Trie traversal abstractions', function() { let tSuccess = false; let hSuccess = false; let eSuccess = false; - for(child of rootTraversal.children()) { + for(let child of rootTraversal.children()) { if(child.char == 't') { tSuccess = true; let traversalInner1 = child.traversal(); @@ -51,7 +51,7 @@ describe('Trie traversal abstractions', function() { assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); - for(tChild of traversalInner1.children()) { + for(let tChild of traversalInner1.children()) { if(tChild.char == 'h') { hSuccess = true; let traversalInner2 = tChild.traversal(); @@ -59,7 +59,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(tChild.traversal().entries); assert.isArray(tChild.traversal().entries); - for(hChild of traversalInner2.children()) { + for(let hChild of traversalInner2.children()) { if(hChild.char == 'e') { eSuccess = true; let traversalInner3 = hChild.traversal(); @@ -68,7 +68,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner3.entries); assert.equal(traversalInner3.entries[0], "the"); - for(eChild of traversalInner3.children()) { + for(let eChild of traversalInner3.children()) { let keyIndex = eKeys.indexOf(eChild.char); assert.notEqual(keyIndex, -1, "Did not find char '" + eChild.char + "' in array!"); eKeys.splice(keyIndex, 1); @@ -96,21 +96,21 @@ describe('Trie traversal abstractions', function() { // 't', 'r', 'o' have siblings, but these don't. let leafChildSequence = ['u', 'b', 'l', 'e']; let eSuccess = false; - for(child of rootTraversal.children()) { + for(let child of rootTraversal.children()) { if(child.char == 't') { let traversalInner1 = child.traversal(); assert.isDefined(traversalInner1); assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); - for(tChild of traversalInner1.children()) { + for(let tChild of traversalInner1.children()) { if(tChild.char == 'r') { let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); assert.isEmpty(tChild.traversal().entries); - for(rChild of traversalInner2.children()) { + for(let rChild of traversalInner2.children()) { if(rChild.char == 'o') { let curChild = rChild; @@ -176,7 +176,7 @@ describe('Trie traversal abstractions', function() { let aSuccess = false; let pSuccess = false; let eSuccess = false; - for(child of rootTraversal.children()) { + for(let child of rootTraversal.children()) { if(child.char == smpA) { aSuccess = true; let traversalInner1 = child.traversal(); @@ -184,7 +184,7 @@ describe('Trie traversal abstractions', function() { assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); - for(aChild of traversalInner1.children()) { + for(let aChild of traversalInner1.children()) { if(aChild.char == smpP) { pSuccess = true; let traversalInner2 = aChild.traversal(); @@ -192,7 +192,7 @@ describe('Trie traversal abstractions', function() { assert.isArray(aChild.traversal().entries); assert.isEmpty(aChild.traversal().entries); - for(pChild of traversalInner2.children()) { + for(let pChild of traversalInner2.children()) { let keyIndex = pKeys.indexOf(pChild.char); assert.notEqual(keyIndex, -1, "Did not find char '" + pChild.char + "' in array!"); pKeys.splice(keyIndex, 1); diff --git a/common/models/templates/tsconfig.bundled.json b/common/models/templates/tsconfig.bundled.json deleted file mode 100644 index fd8dc5f65a532eba7b4c41dce246bc9d21ae15c6..0000000000000000000000000000000000000000 --- a/common/models/templates/tsconfig.bundled.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - // This variant of the tsconfig.json exists to create a 'leaf', 'bundled' - // version of the models/templates build product. The same reference - // cannot be prepended twice in a composite tsc build, posing problems - // for certain down-line builds if the two tsconfigs are not differentiated. - "extends": "./tsconfig.json", - "compilerOptions": { - "outFile": "build/index.bundled.js", - }, - "references": [ - { "path": "../../web/keyman-version", "prepend": true}, - { "path": "../../web/utils", "prepend": true }, - { "path": "../types" } - ], - "include": [ - "src/**/*.ts" - ], - "exclude": [ - "test" - ] -} diff --git a/common/models/templates/tsconfig.json b/common/models/templates/tsconfig.json index 8cc9c6af1c6e0d46233c99508ac359f7b121e965..7ff603ef29d85f26f774ea751fa7dfd2030b507a 100644 --- a/common/models/templates/tsconfig.json +++ b/common/models/templates/tsconfig.json @@ -1,16 +1,27 @@ { "extends": "../../../tsconfig-base.json", "compilerOptions": { + "allowJs": false, "declaration": true, - "module": "none", - "outFile": "./build/index.js", + "module": "es6", + "moduleResolution": "node16", "sourceMap": true, + "sourceRoot": "/common/models/templates/src", + "importHelpers": true, + "inlineSources": true, + "strict": true, "lib": ["es6"], + "types": ["node"], "target": "es5", + "baseUrl": "./", + "outDir": "build/obj", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" }, "references": [ + { "path": "../types" }, { "path": "../../web/utils" }, - { "path": "../types" } + { "path": "../../models/wordbreakers" } ], "include": [ "src/**/*.ts" diff --git a/common/models/wordbreakers/build-bundler.js b/common/models/wordbreakers/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..49693c0d2a66b8f5a94f96e50d78b2da9b6fae2c --- /dev/null +++ b/common/models/wordbreakers/build-bundler.js @@ -0,0 +1,33 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +await esbuild.build({ + bundle: true, + sourcemap: true, + format: "esm", + nodePaths: ['..'], + entryPoints: ['build/obj/index.js'], + outfile: "build/lib/index.mjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); + + +// Bundled CommonJS (classic Node) module version +esbuild.buildSync({ + bundle: true, + sourcemap: true, + format: "cjs", + nodePaths: ['..'], + entryPoints: ['build/obj/index.js'], + outfile: "build/lib/index.cjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/common/models/wordbreakers/build.sh b/common/models/wordbreakers/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..76e6d99d12fa1031ab025f9452a13f40614bfb96 --- /dev/null +++ b/common/models/wordbreakers/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Compile our sourcemap-path remapping module for use by Web builds, releases, etc. +# +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# This script runs from its own folder +cd "$(dirname "$THIS_SCRIPT")" + +################################ Main script ################################ + +builder_describe "Builds the predictive-text wordbreaker implementation module" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + "--ci" + +builder_describe_outputs \ + configure /node_modules \ + build build/lib/index.mjs + +builder_parse "$@" + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +### CLEAN ACTIONS + +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### BUILD ACTIONS + +if builder_start_action build; then + npm run tsc -- -b + node build-bundler.js + + # Declaration bundling. + npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/index.d.ts + + builder_finish_action success build +fi + +if builder_start_action test; then + if builder_has_option --ci; then + npm run mocha -- -reporter mocha-teamcity-reporter + else + npm run mocha + fi + + builder_finish_action success test +fi \ No newline at end of file diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index 3c2eaa1028daaba061a82048b7d33d741770dc15..15866bd8de6d10da4d693a54f79daf0f9beab057 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -14,7 +14,16 @@ ], "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", - "main": "build/index.js", + "main": "build/obj/index.js", + "types": "build/obj/index.d.ts", + "exports": { + ".": "./build/obj/index.js", + "./lib": { + "import": "./build/lib/index.mjs", + "require": "./build/lib/index.cjs" + }, + "./obj/*.js": "./build/obj/*.js" + }, "directories": { "lib": "lib", "test": "test" @@ -30,23 +39,28 @@ "url": "git+https://github.com/keymanapp/keyman.git" }, "scripts": { - "build": "tsc", + "build": "gosh ./build.sh", + "tsc": "tsc", "prepublish": "npm run build", "pretest": "npm run build", - "test": "mocha" + "test": "mocha", + "mocha": "mocha" }, "bugs": { "url": "https://github.com/keymanapp/keyman/issues" }, "devDependencies": { "@keymanapp/models-types": "*", + "@keymanapp/resources-gosh": "*", "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^10.9.1", "typescript": "^4.9.5" }, + "type": "module", "dependencies": { "@types/node": "^14.0.3" } diff --git a/common/models/wordbreakers/src/ascii.ts b/common/models/wordbreakers/src/ascii.ts index e321f387d4414824eb44c1f6592a7d8908ebcfd1..902a6e70583af7917de4667c6414d83eee51432e 100644 --- a/common/models/wordbreakers/src/ascii.ts +++ b/common/models/wordbreakers/src/ascii.ts @@ -1,40 +1,39 @@ /// -namespace wordBreakers { - /** - * Splits ASCII words. - * - * @param phrase - */ - export function ascii(phrase: string): Span[] { - let matchWord = /[A-Za-z0-9']+/g; - let words: Span[] = []; - let match: RegExpExecArray | null; - while ((match = matchWord.exec(phrase)) !== null) { - words.push(new RegExpDerivedSpan(match[0], match.index)); - } - return words; +/** + * Splits ASCII words. + * + * @param phrase + */ +export default function ascii(phrase: string): Span[] { + let matchWord = /[A-Za-z0-9']+/g; + let words: Span[] = []; + let match: RegExpExecArray | null; + while ((match = matchWord.exec(phrase)) !== null) { + words.push(new RegExpDerivedSpan(match[0], match.index)); } - /** - * A concrete span class that derives its properties from the result of - * RegExp.exec() array. - */ - class RegExpDerivedSpan implements Span { - readonly text: string; - readonly start: number; + return words; +} + +/** + * A concrete span class that derives its properties from the result of + * RegExp.exec() array. + */ +class RegExpDerivedSpan implements Span { + readonly text: string; + readonly start: number; - constructor(text: string, start: number) { - this.text = text; - this.start = start; - } + constructor(text: string, start: number) { + this.text = text; + this.start = start; + } - get length(): number { - return this.text.length; - } + get length(): number { + return this.text.length; + } - get end(): number { - return this.start + this.text.length; - } + get end(): number { + return this.start + this.text.length; } } diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index 16a4d9beb38b3c058e90cc1f225d7806521fd733..abd128f020ee41ce2f723c4def7fa6835c4046ef 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -1,6 +1,5 @@ -namespace wordBreakers { -export namespace data { // Automatically generated file. DO NOT MODIFY. + /** * Valid values for a word break property. */ @@ -1801,5 +1800,3 @@ export const WORD_BREAK_PROPERTY: [number, WordBreakProperty][] = [ [/*start*/ 0xE0100, WordBreakProperty.Extend], [/*start*/ 0xE01F0, WordBreakProperty.Other], ]; -} -} diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 715b722932c48f93d3e599f9b6f30a3cf1a4df50..20e159ed1988e81ae47cd07300ee56307c38f364 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -1,625 +1,608 @@ -// Include the word-breaking data here: -/// -namespace wordBreakers { +import { WordBreakProperty, WORD_BREAK_PROPERTY, I, propertyMap } from "./data.js"; + +/** + * A set of options used to customize and extend the behavior of the default + * Unicode wordbreaker. + */ +export interface DefaultWordBreakerOptions { /** - * A set of options used to customize and extend the behavior of the default - * Unicode wordbreaker. + * Allows addition of custom wordbreaking rules, which will be applied + * after WB1-WB4 and before all other default wordbreaking rules. + * + * @see `WordbreakerRule` */ - export interface DefaultWordBreakerOptions { - /** - * Allows addition of custom wordbreaking rules, which will be applied - * after WB1-WB4 and before all other default wordbreaking rules. - * - * @see `WordbreakerRule` - */ - rules?: WordbreakerRule[]; - - /** - * Allows assignment of characters to different word-breaking properties than - * their standard word-breaking assignment, including to custom properties - * specified within `customProperties`. - * @param char - */ - propertyMapping?(char: string): string; - - /** - * Allows definition of extra word-breaking properties for use with custom - * rules. - */ - customProperties?: string[]; - } + rules?: WordbreakerRule[]; /** - * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * - * @see http://unicode.org/reports/tr29/#Word_Boundaries - * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 + * Allows assignment of characters to different word-breaking properties than + * their standard word-breaking assignment, including to custom properties + * specified within `customProperties`. + * @param char */ - export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { - let boundaries = findBoundaries(text, options); - if (boundaries.length == 0) { - return []; - } - - // All non-empty strings have at least TWO boundaries: at the start and at the end of - // the string. - let spans = []; - for (let i = 0; i < boundaries.length - 1; i++) { - let start = boundaries[i]; - let end = boundaries[i + 1]; - let span = new LazySpan(text, start, end); - - if (isNonSpace(span.text, options)) { - spans.push(span); - // Preserve a sequence-final space if it exists. Needed to signal "end of word". - } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... - // We don't want to return the whitespace itself; the correct token is simply ''. - span = new LazySpan(text, end, end); - spans.push(span); - } + propertyMapping?(char: string): string; + + /** + * Allows definition of extra word-breaking properties for use with custom + * rules. + */ + customProperties?: string[]; +} + +/** + * Word breaker based on Unicode Standard Annex #29, Section 4.1: + * Default Word Boundary Specification. + * + * @see http://unicode.org/reports/tr29/#Word_Boundaries + * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 + */ +export default function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { + let boundaries = findBoundaries(text, options); + if (boundaries.length == 0) { + return []; + } + + // All non-empty strings have at least TWO boundaries: at the start and at the end of + // the string. + let spans = []; + for (let i = 0; i < boundaries.length - 1; i++) { + let start = boundaries[i]; + let end = boundaries[i + 1]; + let span = new LazySpan(text, start, end); + + if (isNonSpace(span.text, options)) { + spans.push(span); + // Preserve a sequence-final space if it exists. Needed to signal "end of word". + } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... + // We don't want to return the whitespace itself; the correct token is simply ''. + span = new LazySpan(text, end, end); + spans.push(span); } - return spans; + } + return spans; +} + +/** + * A span that does not cut out the substring until it absolutely has to! + */ +class LazySpan implements Span { + private _source: string; + readonly start: number; + readonly end: number; + constructor(source: string, start: number, end: number) { + this._source = source; + this.start = start; + this.end = end; + } + + get text(): string { + return this._source.substring(this.start, this.end); } - // Utilities // - import WordBreakProperty = wordBreakers.data.WordBreakProperty; - import WORD_BREAK_PROPERTY = wordBreakers.data.WORD_BREAK_PROPERTY; - import I = wordBreakers.data.I; + get length(): number { + return this.end - this.start; + } +} +/** + * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide + * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other + * default word-breaking rules and can be used to define custom rules of similar structure. + * + * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + */ +export interface WordbreakerRule { /** - * A span that does not cut out the substring until it absolutely has to! + * Indicates whether or not the rule applies in the specified context. + * @param context */ - class LazySpan implements Span { - private _source: string; - readonly start: number; - readonly end: number; - constructor(source: string, start: number, end: number) { - this._source = source; - this.start = start; - this.end = end; - } + match(context: BreakerContext): boolean; - get text(): string { - return this._source.substring(this.start, this.end); - } + /** + * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. + */ + breakIfMatch: boolean; +} - get length(): number { - return this.end - this.start; +/** + * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. + * + * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + */ +export class BreakerContext { + // Referenced by this object in order to facilitate `lookahead` maintenance. + private readonly text: string; + readonly options?: DefaultWordBreakerOptions; + + /** + * Represents the property of character immediately preceding `left`'s character. + */ + readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately preceding the potential word boundary. + */ + readonly left: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following the potential word boundary. + */ + readonly right: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following `right`'s character. + */ + readonly lookahead: WordBreakProperty; // Always initialized by constructor. + + /** + * Initializes the word-breaking context at the start of the word-breaker's boundary-detection + * algorithm. + * @param text The text to be word-broken + * @param lookaheadPos The position corresponding to `lookahead`. + */ + constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); + /** + * Used internally by the boundary-detection algorithm during context-shifting operations. + * @param text + * @param lookbehind + * @param left + * @param right + * @param lookahead + */ + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + lookbehind: WordBreakProperty, + left: WordBreakProperty, + right: WordBreakProperty, + lookahead: WordBreakProperty); + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + prop1: WordBreakProperty | number, + prop2?: WordBreakProperty, + prop3?: WordBreakProperty, + prop4?: WordBreakProperty) { + this.text = text; + this.options = options; + + if(arguments.length == 3) { + this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; + } else /*if(arguments.length == 6)*/ { + this.lookbehind = prop1 as WordBreakProperty; + this.left = prop2 as WordBreakProperty; + this.right = prop3 as WordBreakProperty; + this.lookahead = prop4 as WordBreakProperty; } } /** - * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide - * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other - * default word-breaking rules and can be used to define custom rules of similar structure. - * - * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + * The general use-case when shifting boundary-check position if WB4 is not active. + * @param lookahead The WordBreakProperty for the character to become `lookahead`. + * @returns */ - export interface WordbreakerRule { - /** - * Indicates whether or not the rule applies in the specified context. - * @param context - */ - match(context: BreakerContext): boolean; - - /** - * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. - */ - breakIfMatch: boolean; + public next(lookaheadPos: number): BreakerContext { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); } /** - * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. - * - * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + * Used for WB4: when ignoring characters before an intervening linebreak, we + * replace `right` with the current `lookahead`, without affecting `lookbehind` + * or `left`. A new `lookahead` is then needed. + * @param lookahead + * @returns */ - export class BreakerContext { - // Referenced by this object in order to facilitate `lookahead` maintenance. - private readonly text: string; - readonly options?: DefaultWordBreakerOptions; - - /** - * Represents the property of character immediately preceding `left`'s character. - */ - readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; - - /** - * Represents the property of the character immediately preceding the potential word boundary. - */ - readonly left: WordBreakProperty = WordBreakProperty.sot; - - /** - * Represents the property of the character immediately following the potential word boundary. - */ - readonly right: WordBreakProperty = WordBreakProperty.sot; - - /** - * Represents the property of the character immediately following `right`'s character. - */ - readonly lookahead: WordBreakProperty; // Always initialized by constructor. - - /** - * Initializes the word-breaking context at the start of the word-breaker's boundary-detection - * algorithm. - * @param text The text to be word-broken - * @param lookaheadPos The position corresponding to `lookahead`. - */ - constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); - /** - * Used internally by the boundary-detection algorithm during context-shifting operations. - * @param text - * @param lookbehind - * @param left - * @param right - * @param lookahead - */ - constructor(text: string, - options: DefaultWordBreakerOptions | undefined, - lookbehind: WordBreakProperty, - left: WordBreakProperty, - right: WordBreakProperty, - lookahead: WordBreakProperty); - constructor(text: string, - options: DefaultWordBreakerOptions | undefined, - prop1: WordBreakProperty | number, - prop2?: WordBreakProperty, - prop3?: WordBreakProperty, - prop4?: WordBreakProperty) { - this.text = text; - this.options = options; - - if(arguments.length == 3) { - this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; - } else /*if(arguments.length == 6)*/ { - this.lookbehind = prop1 as WordBreakProperty; - this.left = prop2 as WordBreakProperty; - this.right = prop3 as WordBreakProperty; - this.lookahead = prop4 as WordBreakProperty; - } - } + public ignoringRight(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); + } - /** - * The general use-case when shifting boundary-check position if WB4 is not active. - * @param lookahead The WordBreakProperty for the character to become `lookahead`. - * @returns - */ - public next(lookaheadPos: number): BreakerContext { - let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); - } - - /** - * Used for WB4: when ignoring characters before an intervening linebreak, we - * replace `right` with the current `lookahead`, without affecting `lookbehind` - * or `left`. A new `lookahead` is then needed. - * @param lookahead - * @returns - */ - public ignoringRight(lookaheadPos: number) { - let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); - } - - /** - * Used for WB4: when ignoring characters after an intervening linebreak, it's - * `lookahead` that gets replaced without shifting the other tracked properties. - * @param lookahead - * @returns - */ - public ignoringLookahead(lookaheadPos: number) { - let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); - } - - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - private wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= this.text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(this.text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(this.text[pos] + this.text[pos + 1]); - } - return property(this.text[pos], this.options); - } - - /** - * Returns `true` if and only if each member of the context has a property included within - * its corresponding set (when specified). Any set may be replaced with null to disable - * a check against its corresponding property. - * @param lookbehindSet - * @param leftSet - * @param rightSet - * @param lookaheadSet - */ - public match(lookbehindSet: WordBreakProperty[] | null, - leftSet: WordBreakProperty[] | null, - rightSet: WordBreakProperty[] | null, - lookaheadSet: WordBreakProperty[] | null) : boolean { - let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; - result = result && (leftSet?.includes(this.left) ?? true); - result = result && (rightSet?.includes(this.right) ?? true); - return result && (lookaheadSet?.includes(this.lookahead) ?? true); - } - - /** - * Returns `true` if and only if each member of the context has a property included within - * its corresponding set (when specified). Any set may be replaced with null to disable - * a check against its corresponding property. - * - * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules - * or defined in the word-breaker customization options; matching is case-insensitive. - * Also includes two extra properties: - * - `sot` - start of text - * - `eot` - end of text - * @param lookbehindSet - * @param leftSet - * @param rightSet - * @param lookaheadSet - */ - public propertyMatch(lookbehindSet: string[] | null, - leftSet: string[] | null, - rightSet: string[] | null, - lookaheadSet: string[] | null) : boolean { - const propMapper = (name: string) => propertyVal(name, this.options); - return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, - leftSet?.map(propMapper) as WordBreakProperty[] | null, - rightSet?.map(propMapper) as WordBreakProperty[] | null, - lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); - } + /** + * Used for WB4: when ignoring characters after an intervening linebreak, it's + * `lookahead` that gets replaced without shifting the other tracked properties. + * @param lookahead + * @returns + */ + public ignoringLookahead(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + private wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= this.text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(this.text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(this.text[pos] + this.text[pos + 1]); } + return property(this.text[pos], this.options); + } /** - * Returns true when the chunk does not solely consist of whitespace. - * - * @param chunk a chunk of text. Starts and ends at word boundaries. + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet */ - function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { - return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( - wb === WordBreakProperty.CR || - wb === WordBreakProperty.LF || - wb === WordBreakProperty.Newline || - wb === WordBreakProperty.WSegSpace - )); + public match(lookbehindSet: WordBreakProperty[] | null, + leftSet: WordBreakProperty[] | null, + rightSet: WordBreakProperty[] | null, + lookaheadSet: WordBreakProperty[] | null) : boolean { + let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; + result = result && (leftSet?.includes(this.left) ?? true); + result = result && (rightSet?.includes(this.right) ?? true); + return result && (lookaheadSet?.includes(this.lookahead) ?? true); } /** - * Yields a series of string indices where a word break should - * occur. That is, there should be a break BEFORE each string - * index yielded by this generator. + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. * - * @param text Text to find word boundaries in. + * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules + * or defined in the word-breaker customization options; matching is case-insensitive. + * Also includes two extra properties: + * - `sot` - start of text + * - `eot` - end of text + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet */ - function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { - // WB1 and WB2: no boundaries if given an empty string. - if (text.length === 0) { - // There are no boundaries in an empty string! - return []; - } - - if(options && !options.rules) { - options.rules = []; - } - - // This algorithm works by maintaining a sliding window of four SCALAR VALUES. - // - // - Scalar values? JavaScript strings are NOT actually a string of - // Unicode code points; some characters are made up of TWO - // JavaScript indices. e.g., - // "💩".length === 2; - // "💩"[0] === '\uD83D'; - // "💩"[1] === '\uDCA9'; - // - // These characters that are represented by TWO indices are - // called "surrogate pairs". Since we don't want to be in the - // "middle" of a character, make sure we're always advancing - // by scalar values, and NOT indices. That means, we sometimes - // need to advance by TWO indices, not just one. - // - Four values? Some rules look at what's to the left of - // left, and some look at what's to the right of right. So - // keep track of this! - - let boundaries = []; - - let rightPos: number; - let lookaheadPos = 0; // lookahead, one scalar value to the right of right. - // Before the start of the string is also the start of the string. - let state = new BreakerContext(text, options, lookaheadPos); - // Count RIs to make sure we're not splitting emoji flags: - let nConsecutiveRegionalIndicators = 0; - - do { - // Shift all positions, one scalar value to the right. - rightPos = lookaheadPos; - lookaheadPos = positionAfter(lookaheadPos); - // Shift all properties, one scalar value to the right. - state = state.next(lookaheadPos); + public propertyMatch(lookbehindSet: string[] | null, + leftSet: string[] | null, + rightSet: string[] | null, + lookaheadSet: string[] | null) : boolean { + const propMapper = (name: string) => propertyVal(name, this.options); + return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, + leftSet?.map(propMapper) as WordBreakProperty[] | null, + rightSet?.map(propMapper) as WordBreakProperty[] | null, + lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); + } + } - // Break at the start and end of text, unless the text is empty. - // WB1: Break at start of text... - if (state.match(null, [WordBreakProperty.sot], null, null)) { - boundaries.push(rightPos); - continue; - } - // WB2: Break at the end of text... - if (state.match(null, null, [WordBreakProperty.eot], null)) { - boundaries.push(rightPos); - break; // Reached the end of the string. We're done! - } - // WB3: Do not break within CRLF: - if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { - continue; - } +/** + * Returns true when the chunk does not solely consist of whitespace. + * + * @param chunk a chunk of text. Starts and ends at word boundaries. + */ +function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { + return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( + wb === WordBreakProperty.CR || + wb === WordBreakProperty.LF || + wb === WordBreakProperty.Newline || + wb === WordBreakProperty.WSegSpace + )); +} - // WB3b: Otherwise, break after... - const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; - if(state.match(null, NEWLINE_SET, null, null)) { - boundaries.push(rightPos); - continue; - } - // WB3a: ...and before newlines - if (state.match(null, null, NEWLINE_SET, null)) { - boundaries.push(rightPos); - continue; - } +/** + * Yields a series of string indices where a word break should + * occur. That is, there should be a break BEFORE each string + * index yielded by this generator. + * + * @param text Text to find word boundaries in. + */ +function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { + // WB1 and WB2: no boundaries if given an empty string. + if (text.length === 0) { + // There are no boundaries in an empty string! + return []; + } - // TODO: WB3c is not implemented, due to its complex, error-prone - // implementation, requiring a ginormous regexp, and the fact that - // the only thing it does is prevent big emoji sequences from being - // split up, like 🧚ðŸ¼â€â™‚ï¸ - // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt + if(options && !options.rules) { + options.rules = []; + } - // WB3d: Keep horizontal whitespace together - if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { - continue; - } + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. + // + // - Scalar values? JavaScript strings are NOT actually a string of + // Unicode code points; some characters are made up of TWO + // JavaScript indices. e.g., + // "💩".length === 2; + // "💩"[0] === '\uD83D'; + // "💩"[1] === '\uDCA9'; + // + // These characters that are represented by TWO indices are + // called "surrogate pairs". Since we don't want to be in the + // "middle" of a character, make sure we're always advancing + // by scalar values, and NOT indices. That means, we sometimes + // need to advance by TWO indices, not just one. + // - Four values? Some rules look at what's to the left of + // left, and some look at what's to the right of right. So + // keep track of this! + + let boundaries = []; + + let rightPos: number; + let lookaheadPos = 0; // lookahead, one scalar value to the right of right. + // Before the start of the string is also the start of the string. + let state = new BreakerContext(text, options, lookaheadPos); + // Count RIs to make sure we're not splitting emoji flags: + let nConsecutiveRegionalIndicators = 0; + + do { + // Shift all positions, one scalar value to the right. + rightPos = lookaheadPos; + lookaheadPos = positionAfter(lookaheadPos); + // Shift all properties, one scalar value to the right. + state = state.next(lookaheadPos); + + // Break at the start and end of text, unless the text is empty. + // WB1: Break at start of text... + if (state.match(null, [WordBreakProperty.sot], null, null)) { + boundaries.push(rightPos); + continue; + } + // WB2: Break at the end of text... + if (state.match(null, null, [WordBreakProperty.eot], null)) { + boundaries.push(rightPos); + break; // Reached the end of the string. We're done! + } + // WB3: Do not break within CRLF: + if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { + continue; + } - // WB4: Ignore format and extend characters - // This is to keep grapheme clusters together! - // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules - // N.B.: The rule about "except after sot, CR, LF, and - // Newline" already been by WB1, WB2, WB3a, and WB3b above. - const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; - while (state.match(null, null, SET_WB4_IGNORE, null)) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and - // lookbehind however! - [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - state = state.ignoringRight(lookaheadPos); - } - // In ignoring the characters in the previous loop, we could - // have fallen off the end of the string, so end the loop - // prematurely if that happens! - if (state.right === WordBreakProperty.eot) { - boundaries.push(rightPos); - break; - } - // WB4 (continued): Lookahead must ALSO ignore these format, - // extend, ZWJ characters! - while (state.match(null, null, null, SET_WB4_IGNORE)) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and right, - // however! - lookaheadPos = positionAfter(lookaheadPos); - state = state.ignoringLookahead(lookaheadPos); - } + // WB3b: Otherwise, break after... + const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; + if(state.match(null, NEWLINE_SET, null, null)) { + boundaries.push(rightPos); + continue; + } + // WB3a: ...and before newlines + if (state.match(null, null, NEWLINE_SET, null)) { + boundaries.push(rightPos); + continue; + } - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; - const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; - - // Custom rules may override the base ruleset aside from the first few fundamental ones. - if(options?.rules) { - let customMatch: boolean = false; - for(const rule of options.rules) { - customMatch = rule.match(state); - if(customMatch) { - if(rule.breakIfMatch) { - boundaries.push(rightPos); - } - break; // as customMatch == true here, this will trigger the `continue` that follows. - } - } + // TODO: WB3c is not implemented, due to its complex, error-prone + // implementation, requiring a ginormous regexp, and the fact that + // the only thing it does is prevent big emoji sequences from being + // split up, like 🧚ðŸ¼â€â™‚ï¸ + // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt + + // WB3d: Keep horizontal whitespace together + if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { + continue; + } + + // WB4: Ignore format and extend characters + // This is to keep grapheme clusters together! + // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules + // N.B.: The rule about "except after sot, CR, LF, and + // Newline" already been by WB1, WB2, WB3a, and WB3b above. + const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; + while (state.match(null, null, SET_WB4_IGNORE, null)) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and + // lookbehind however! + [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; + state = state.ignoringRight(lookaheadPos); + } + // In ignoring the characters in the previous loop, we could + // have fallen off the end of the string, so end the loop + // prematurely if that happens! + if (state.right === WordBreakProperty.eot) { + boundaries.push(rightPos); + break; + } + // WB4 (continued): Lookahead must ALSO ignore these format, + // extend, ZWJ characters! + while (state.match(null, null, null, SET_WB4_IGNORE)) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and right, + // however! + lookaheadPos = positionAfter(lookaheadPos); + state = state.ignoringLookahead(lookaheadPos); + } + + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; + const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + + // Custom rules may override the base ruleset aside from the first few fundamental ones. + if(options?.rules) { + let customMatch: boolean = false; + for(const rule of options.rules) { + customMatch = rule.match(state); if(customMatch) { - continue; + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. } } - - // WB5: Do not break between most letters. - // if (isAHLetter(state.left) && isAHLetter(state.right)) - if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { - continue; - } - // Do not break across certain punctuation - // WB6: (Don't break before apostrophes in contractions) - const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; - if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { - continue; - } - // WB7: (Don't break after apostrophes in contractions) - if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { + if(customMatch) { continue; } + } - // WB7a - if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { - continue; - } - // WB7b - if(state.match(null, - [WordBreakProperty.Hebrew_Letter], - [WordBreakProperty.Double_Quote], - [WordBreakProperty.Hebrew_Letter])) { - continue; - } - // WB7c - if(state.match([WordBreakProperty.Hebrew_Letter], - [WordBreakProperty.Double_Quote], - [WordBreakProperty.Hebrew_Letter], - null)) { - continue; - } - // Do not break within sequences of digits, or digits adjacent to letters. - // e.g., "3a" or "A3" - // WB8 - if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { - continue; - } - // WB9 - if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { - continue; - } - // WB10 - if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { - continue; - } - // Do not break within sequences, such as 3.2, 3,456.789 - // WB11 - const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; - if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { - continue; - } - // WB12 - if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { - continue; - } - // WB13: Do not break between Katakana - if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { - continue; - } - // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) - // WB13a - const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, - WordBreakProperty.Numeric, - ...SET_AHLETTER]; - if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { - continue; - } - if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { - continue; - } - // WB13b - if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { - continue; - } + // WB5: Do not break between most letters. + // if (isAHLetter(state.left) && isAHLetter(state.right)) + if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { + continue; + } + // Do not break across certain punctuation + // WB6: (Don't break before apostrophes in contractions) + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter].concat(SET_MIDNUMLETQ); + if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { + continue; + } + // WB7: (Don't break after apostrophes in contractions) + if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { + continue; + } - // WB15 & WB16: - // Do not break within emoji flag sequences. That is, do not break between - // regional indicator (RI) symbols if there is an odd number of RI - // characters before the break point. - if (state.right === WordBreakProperty.Regional_Indicator) { - // Emoji flags are actually composed of TWO scalar values, each being a - // "regional indicator". These indicators correspond to Latin letters. Put - // two of them together, and they spell out an ISO 3166-1-alpha-2 country - // code. Since these always come in pairs, NEVER split the pairs! So, if - // we happen to be inside the middle of an odd numbered of - // Regional_Indicators, DON'T SPLIT IT! - nConsecutiveRegionalIndicators += 1; - if ((nConsecutiveRegionalIndicators % 2) == 1) { - continue; - } - } else { - nConsecutiveRegionalIndicators = 0; - } - // WB999: Otherwise, break EVERYWHERE (including around ideographs) - boundaries.push(rightPos); - } while (rightPos < text.length); - - return boundaries; - - ///// Internal utility functions ///// - - /** - * Returns the position of the start of the next scalar value. This jumps - * over surrogate pairs. - * - * If asked for the character AFTER the end of the string, this always - * returns the length of the string. - */ - function positionAfter(pos: number): number { - if (pos >= text.length) { - return text.length; - } else if (isStartOfSurrogatePair(text[pos])) { - return pos + 2; + // WB7a + if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { + continue; + } + // WB7b + if(state.match(null, + [WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter])) { + continue; + } + // WB7c + if(state.match([WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter], + null)) { + continue; + } + // Do not break within sequences of digits, or digits adjacent to letters. + // e.g., "3a" or "A3" + // WB8 + if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { + continue; + } + // WB9 + if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { + continue; + } + // WB10 + if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { + continue; + } + // Do not break within sequences, such as 3.2, 3,456.789 + // WB11 + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum].concat(SET_MIDNUMLETQ); + if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { + continue; + } + // WB12 + if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { + continue; + } + // WB13: Do not break between Katakana + if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { + continue; + } + // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) + // WB13a + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, WordBreakProperty.Numeric].concat(SET_AHLETTER); + if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { + continue; + } + if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { + continue; + } + // WB13b + if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { + continue; + } + + // WB15 & WB16: + // Do not break within emoji flag sequences. That is, do not break between + // regional indicator (RI) symbols if there is an odd number of RI + // characters before the break point. + if (state.right === WordBreakProperty.Regional_Indicator) { + // Emoji flags are actually composed of TWO scalar values, each being a + // "regional indicator". These indicators correspond to Latin letters. Put + // two of them together, and they spell out an ISO 3166-1-alpha-2 country + // code. Since these always come in pairs, NEVER split the pairs! So, if + // we happen to be inside the middle of an odd numbered of + // Regional_Indicators, DON'T SPLIT IT! + nConsecutiveRegionalIndicators += 1; + if ((nConsecutiveRegionalIndicators % 2) == 1) { + continue; } - return pos + 1; + } else { + nConsecutiveRegionalIndicators = 0; } - } + // WB999: Otherwise, break EVERYWHERE (including around ideographs) + boundaries.push(rightPos); + } while (rightPos < text.length); - function isStartOfSurrogatePair(character: string) { - let codeUnit = character.charCodeAt(0); - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; - } + return boundaries; + + ///// Internal utility functions ///// /** - * Return the Word_Break property value for a character. - * Note that - * @param character a scalar value + * Returns the position of the start of the next scalar value. This jumps + * over surrogate pairs. + * + * If asked for the character AFTER the end of the string, this always + * returns the length of the string. */ - function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { - // If there is a customized mapping for the character, prioritize that. - if(options?.propertyMapping) { - let propName = options.propertyMapping(character); - if(propName) { - return propertyVal(propName, options); - } + function positionAfter(pos: number): number { + if (pos >= text.length) { + return text.length; + } else if (isStartOfSurrogatePair(text[pos])) { + return pos + 2; } + return pos + 1; + } +} - // This MUST be a scalar value. - // TODO: remove dependence on character.codepointAt()? - let codepoint = character.codePointAt(0) as number; +function isStartOfSurrogatePair(character: string) { + let codeUnit = character.charCodeAt(0); + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; +} - return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); +/** + * Return the Word_Break property value for a character. + * Note that + * @param character a scalar value + */ +function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { + // If there is a customized mapping for the character, prioritize that. + if(options?.propertyMapping) { + let propName = options.propertyMapping(character); + if(propName) { + return propertyVal(propName, options); + } } - function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { - const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() + // This MUST be a scalar value. + // TODO: remove dependence on character.codepointAt()? + let codepoint = character.codePointAt(0) as number; - const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; - return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); - } + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); +} - /** - * Binary search for the word break property of a given CODE POINT. - * - * The auto-generated data.ts master array defines a **character range** - * lookup table. If a character's codepoint is equal to or greater than - * the I.Start value for an entry and exclusively less than the next entry, - * it falls in the first entry's range bucket and is classified accordingly - * by this method. - */ - function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { - // All items that are not found in the array are assigned the 'Other' property. - if (right < left) { - return WordBreakProperty.Other; - } +function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { + const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() - let midpoint = left + ~~((right - left) / 2); - let candidate = WORD_BREAK_PROPERTY[midpoint]; + const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; + return customIndex != -1 ? -customIndex - 1 : propertyMap.findIndex(matcher); +} - let nextRange = WORD_BREAK_PROPERTY[midpoint + 1]; - let startOfNextRange = nextRange ? nextRange[I.Start] : Infinity; +/** + * Binary search for the word break property of a given CODE POINT. + * + * The auto-generated data.ts master array defines a **character range** + * lookup table. If a character's codepoint is equal to or greater than + * the I.Start value for an entry and exclusively less than the next entry, + * it falls in the first entry's range bucket and is classified accordingly + * by this method. + */ +function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { + // All items that are not found in the array are assigned the 'Other' property. + if (right < left) { + return WordBreakProperty.Other; + } - if (codePoint < candidate[I.Start]) { - return searchForProperty(codePoint, left, midpoint - 1); - } else if (codePoint >= startOfNextRange) { - return searchForProperty(codePoint, midpoint + 1, right); - } + let midpoint = left + ~~((right - left) / 2); + let candidate = WORD_BREAK_PROPERTY[midpoint]; - // We found it! - return candidate[I.Value]; + let nextRange = WORD_BREAK_PROPERTY[midpoint + 1]; + let startOfNextRange = nextRange ? nextRange[I.Start] : Infinity; + + if (codePoint < candidate[I.Start]) { + return searchForProperty(codePoint, left, midpoint - 1); + } else if (codePoint >= startOfNextRange) { + return searchForProperty(codePoint, midpoint + 1, right); } -} -// We cannot export a member whose name is a reserved word when -// implementing a namespace, BUT we can manually make the -// assignment and **declare** it as part of the namespace. -wordBreakers['default'] = wordBreakers.default_; -declare namespace wordBreakers { - export { default_ as default }; + // We found it! + return candidate[I.Value]; } diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index 2140ae6bf4cf0157cdb93dff26d5dabcc4cb2a46..7f1682fca27c7599385e983f6d5eda72ca6688c3 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,8 +1,5 @@ -/// -/// -/// +import placeholder from "./placeholder.js"; +import ascii from "./ascii.js"; +import default_ from "./default/index.js"; -// Let the word breakers be available both in the browser and in Node. -if (typeof module !== "undefined" && (module as any).exports) { - (module as any).exports['wordBreakers'] = wordBreakers; -} +export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker }; \ No newline at end of file diff --git a/common/models/wordbreakers/src/placeholder.ts b/common/models/wordbreakers/src/placeholder.ts index 1105b89fb99b56e4a6182d5237aa9ac8b88c561c..87832505ed046761bda5301d3b4a63ceb2dac1ad 100644 --- a/common/models/wordbreakers/src/placeholder.ts +++ b/common/models/wordbreakers/src/placeholder.ts @@ -1,24 +1,21 @@ -namespace wordBreakers { - - /** - * A **VERY** dumb word breaker that simply splits at words. Do not use this - * word breaker! - * - * @param phrase The phrase in which to break words. - * @deprecated Use a word breaker tailored to your language instead! - */ - export function placeholder(phrase: string): Span[] { - let nextStart = 0; - return phrase.split(/\s+/).map(utterance => { - // XXX: The indices are NOT accurate to the original phrase! - let span = { - start: nextStart, - end: nextStart + utterance.length, - text: utterance, - length: utterance.length - }; - nextStart = span.end; - return span; - }); - } +/** + * A **VERY** dumb word breaker that simply splits at words. Do not use this + * word breaker! + * + * @param phrase The phrase in which to break words. + * @deprecated Use a word breaker tailored to your language instead! + */ +export default function placeholder(phrase: string): Span[] { + let nextStart = 0; + return phrase.split(/\s+/).map(utterance => { + // XXX: The indices are NOT accurate to the original phrase! + let span = { + start: nextStart, + end: nextStart + utterance.length, + text: utterance, + length: utterance.length + }; + nextStart = span.end; + return span; + }); } diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.js b/common/models/wordbreakers/test/test-ascii-word-breaker.js index 55b9c3b8ba0f5f440e266955eee39b04d1f214cd..843df8c002aeb050814bfded663c2b796d6be285 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.js +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.js @@ -1,5 +1,5 @@ -var assert = require('chai').assert; -var breakASCIIWords = require('../build').wordBreakers['ascii']; +import { assert } from 'chai'; +import { ascii as breakASCIIWords } from '@keymanapp/models-wordbreakers'; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 85129b836ccd3673c69cd7f8efb62c37d5934e7d..17c03cc4b8a0259161923893d7c0b29b6b4b4d49 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -2,8 +2,8 @@ * Smoke-test the default */ -const assert = require('chai').assert; -const breakWords = require('../build').wordBreakers['default']; +import { assert } from 'chai'; +import { default as breakWords } from '@keymanapp/models-wordbreakers'; const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.js b/common/models/wordbreakers/test/test-placeholder-word-breaker.js index dbc1aef34e61714e9d127d85d6f0519b52902a28..0991315cd25805ee8f754c03059241c15d193fbc 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.js +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -1,6 +1,6 @@ -const assert = require('chai').assert; -const breakWords = require('../build').wordBreakers['placeholder']; +import { assert } from 'chai'; +import { placeholder as breakWords } from '@keymanapp/models-wordbreakers'; describe('The placeholder word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/models/wordbreakers/tsconfig.json b/common/models/wordbreakers/tsconfig.json index 6575d9e7c568e7880dd0594afb78d80a303de226..549b7410d6e99f994c68f7b8590adc3a0c9a1cd9 100644 --- a/common/models/wordbreakers/tsconfig.json +++ b/common/models/wordbreakers/tsconfig.json @@ -2,14 +2,21 @@ "extends": "../../../tsconfig-base.json", "compilerOptions": { - "module": "none", + "module": "es6", "lib": ["es6"], "target": "es5", + "moduleResolution": "Node16", "declaration": true, "sourceMap": true, - "outFile": "./build/index.js", + "importHelpers": true, + "inlineSources": true, + "sourceRoot": "/common/models/wordbreakers/src", "strict": true, "allowJs": false, + "baseUrl": "./", + "outDir": "build/obj", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" }, "references": [ { "path": "../types" } diff --git a/common/predictive-text/.gitignore b/common/predictive-text/.gitignore index 924ca9595eb0d21772177fe976896e9e3f57abbe..b4aef947b4ad67df7cebb579736918eb28bcd390 100644 --- a/common/predictive-text/.gitignore +++ b/common/predictive-text/.gitignore @@ -68,6 +68,7 @@ typings/ **/*.js # Intentional JavaScript files. +!build-bundler.js !testing/**/*.js !unit_tests/**/*.js !polyfills/**/*.js diff --git a/common/predictive-text/browser.tsconfig.json b/common/predictive-text/browser.tsconfig.json deleted file mode 100644 index 929924f9f16ea7b81c496526f03c658068b6ff7f..0000000000000000000000000000000000000000 --- a/common/predictive-text/browser.tsconfig.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "extends": "../../tsconfig-base.json", - - "compilerOptions": { - "allowJs": true, - "module": "none", - "outFile": "./build/index.js", - "inlineSources": true, - "sourceMap": true, - "target": "es5", - "lib": ["es6", "dom"] - }, - "include" : ["./*.ts"], - "references": [ - { "path": "../models/types" }, - { "path": "../models/wordbreakers" }, - { "path": "../models/templates" }, - { "path": "../web/lm-message-types" } - ], - "exclude" : [ - "node-defaultWorker.ts", - "virtualizedWorker.ts", - "index.d.ts", - "worker-interface.d.ts", - "node_modules", - "testing" - ], - "files": [ - "../web/lm-worker/build/index.wrapped-for-bundle.ts" - ] -} diff --git a/common/predictive-text/build-bundler.js b/common/predictive-text/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..a666e008fd6270fb454670f49027b07cb74acb9c --- /dev/null +++ b/common/predictive-text/build-bundler.js @@ -0,0 +1,41 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +await esbuild.build({ + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + format: "esm", + nodePaths: ['../../node_modules'], + entryPoints: { + 'index': 'build/obj/web/index.js', + }, + external: ['fs', 'vm'], + outdir: 'build/lib/web', + outExtension: { '.js': '.mjs' }, + tsconfig: 'src/web/tsconfig.json', + target: "es5" +}); + +// // Direct-use version +// esbuild.buildSync({ +// bundle: true, +// sourcemap: true, +// format: "iife", +// nodePaths: ['..'], +// entryPoints: { +// 'worker-main': 'build/obj/worker-main.js' +// }, +// outdir: 'build/lib', +// tsconfig: 'tsconfig.json', +// target: "es5" +// }); \ No newline at end of file diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 617eaa379b7707b07e138f7fe402c6c57a51902e..4fe5a744cb5fc87e126955f9cf2e8900cf8c0bd7 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -22,28 +22,21 @@ cd "$(dirname "$THIS_SCRIPT")" ################################ Main script ################################ -# TODO: once these modules are builder-based, reference here too: -# "@../models/templates" \ -# "@../models/types" \ -# "@../models/wordbreakers" +# "@../models/types" \ # is just a .d.ts, so there's nothing to actually BUILD. builder_describe "Builds the lm-layer module" \ "@/common/web/keyman-version" \ + "@/common/web/tslib" \ "@/common/web/lm-worker" \ "clean" \ "configure" \ "build" \ "test" \ - ":headless A headless, Node-oriented version of the module useful for unit tests" \ - ":browser The standard version of the module for in-browser use" \ "--ci Sets $(builder_term test) action to use CI-based test configurations & reporting" builder_describe_outputs \ - configure /node_modules \ - configure:headless /node_modules \ - configure:browser /node_modules \ - build:headless /common/predictive-text/build/headless.js \ - build:browser /common/predictive-text/build/index.js + configure /node_modules \ + build /common/predictive-text/build/lib/web/index.mjs # is built by the final step. builder_parse "$@" @@ -63,18 +56,14 @@ fi ### BUILD ACTIONS -# Builds the top-level JavaScript file for use in browsers -if builder_start_action build:browser; then - npm run tsc -- -b ./browser.tsconfig.json - - builder_finish_action success build:browser -fi - # Builds the top-level JavaScript file for use on Node -if builder_start_action build:headless; then - npm run tsc -- -b ./tsconfig.json +if builder_start_action build; then + npm run tsc -- -b ./tsconfig.all.json + + # esbuild-bundled products at this level are not intended to be used for anything but testing. + node build-bundler.js - builder_finish_action success build:headless + builder_finish_action success build fi ### TEST ACTIONS @@ -86,16 +75,10 @@ if builder_has_option --ci; then TEST_OPTIONS=--ci fi -if builder_start_action test:headless; then +if builder_start_action test; then # We'll test the included libraries here for now, at least until we have # converted their builds to builder scripts - ./unit_tests/test.sh test:libraries test:headless $TEST_OPTIONS + ./unit_tests/test.sh test:libraries test:headless test:browser $TEST_OPTIONS - builder_finish_action success test:headless -fi - -if builder_start_action test:browser; then - ./unit_tests/test.sh test:browser $TEST_OPTIONS - - builder_finish_action success test:browser -fi + builder_finish_action success test +fi \ No newline at end of file diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts deleted file mode 100644 index 3dd8c4b1f47d3bc625a73426ff2b20ab0266a618..0000000000000000000000000000000000000000 --- a/common/predictive-text/index.d.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Defines TS method typing for the LMLayer's public API. - * - * Unfortunately, it is not possible to do this programmatically at this time, as TS won't output declarations - * if any JS files are allowed in compilation. We can't convert the embedded worker's intermediate file to TS - * safely due to DedicatedWorkerGlobalScope and a few other typing issues. - */ - -/// -/// - -declare namespace com.keyman.text.prediction { - class LMLayer { - - /** - * Construct the top-level LMLayer interface. This also starts the underlying Worker. - * Make sure to call .load() when using the default Worker. - * - * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: - * or file: URI. If uri is not provided, this will start the default Worker. - */ - constructor(capabilities: Capabilities, worker?: Worker); - - /** - * Initializes the LMLayer worker with the keyboard/platform's capabilities, - * as well as a description of the model required. - */ - loadModel(model: string): Promise; - - /** - * Prepares the LMLayer for reinitialization with a different model/capability set. - */ - unloadModel(); - - predict(transform: Transform, context: Context): Promise; - - // TODO: asynchronous close() method. - // Worker code must recognize message and call self.close(). - - private onMessage(event: MessageEvent): void; - - /** - * Given a function, this utility returns the source code within it, as a string. - * This is intended to unwrap the "wrapped" source code created in the LMLayerWorker - * build process. - * - * @param fn The function whose body will be returned. - */ - static unwrap(fn: Function): string; - - /** - * Converts the INSIDE of a function into a blob URI that can - * be passed as a valid URI for a Worker. - * @param fn Function whose body will be referenced by a URI. - * - * This function makes the following possible: - * - * let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () { - * postMessage('inside Web Worker') - * function onmessage(event) { - * // handle message inside Web Worker. - * } - * })); - */ - static asBlobURI(fn: Function): string; - - /** - * Clears out any computational resources in use by the LMLayer, including shutting - * down any internal WebWorkers. - */ - public shutdown(): void; - } -} \ No newline at end of file diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts deleted file mode 100644 index 887f71c2ce9f60f80c83b0ff11b23cf544896afc..0000000000000000000000000000000000000000 --- a/common/predictive-text/index.ts +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) - * Copyright (c) 2018 SIL International - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/// -/// -/// - -/** - * Top-level interface to the Language Modelling layer, or "LMLayer" for short. - * - * The Language Modelling layer provides a way for keyboards to offer prediction and - * correction functionalities. The LMLayer proper runs within a Web Worker, however, - * this class is intended to run in the main thread, and automatically spawn a Web - * Worker, capable of offering predictions. - * - * Since the Worker runs in a different thread, the public methods of this class are - * asynchronous. Methods of note include: - * - * - #loadModel() -- loads a specified model file - * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for - * the input event - * - #unloadModel() -- unloads the LMLayer's currently loaded model, preparing it to - * receive (load) a new model - * - * The top-level LMLayer will automatically starts up its own Web Worker. - */ - -namespace com.keyman.text.prediction { - export class LMLayer { - /** - * The underlying worker instance. By default, this is the LMLayerWorker. - */ - private _worker: Worker; - /** Call this when the LMLayer has sent us the 'ready' message! */ - private _declareLMLayerReady: (conf: Configuration) => void; - private _predictPromises: PromiseStore; - private _wordbreakPromises: PromiseStore; - private _acceptPromises: PromiseStore; - private _revertPromises: PromiseStore; - private _nextToken: number; - private capabilities: Capabilities; - - /** - * Construct the top-level LMLayer interface. This also starts the underlying Worker. - * - * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: - * or file: URI. If uri is not provided, this will start the default Worker. - */ - constructor(capabilities: Capabilities, worker?: Worker, testMode?: boolean) { - // Either use the given worker, or instantiate the default worker. - this._worker = worker || DefaultWorker.constructInstance(); - this._worker.onmessage = this.onMessage.bind(this) - this._declareLMLayerReady = null; - this._predictPromises = new PromiseStore(); - this._wordbreakPromises = new PromiseStore(); - this._acceptPromises = new PromiseStore(); - this._revertPromises = new PromiseStore(); - this._nextToken = Number.MIN_SAFE_INTEGER; - - this.sendConfig(capabilities, !!testMode); - } - - /** - * Initializes the LMLayer worker with the host platform's capability set. - * - * @param capabilities The host platform's capability spec - a model cannot assume access to more context - * than specified by this parameter. - */ - private sendConfig(capabilities: Capabilities, testMode: boolean) { - this._worker.postMessage({ - message: 'config', - capabilities: capabilities, - testMode: testMode - }); - } - - /** - * Initializes the LMLayer worker with a path to the desired model file. - */ - loadModel(modelSource: string, loadType: 'file' | 'raw' = 'file'): Promise { - return new Promise((resolve, _reject) => { - // Sets up so the promise is resolved in the onMessage() callback, when it receives - // the 'ready' message. - this._declareLMLayerReady = resolve; - - let modelSourceSpec: any = { - type: loadType - }; - - if(loadType == 'file') { - modelSourceSpec.file = modelSource; - } else { - modelSourceSpec.code = modelSource; - } - - this._worker.postMessage({ - message: 'load', - source: modelSourceSpec - }); - }); - } - - /** - * Unloads the previously-active model from memory, resetting the LMLayer to prep - * for transition to use of a new model. - */ - public unloadModel() { - this._worker.postMessage({ - message: 'unload' - }); - } - - predict(transform: Transform | Distribution, context: Context): Promise { - let token = this._nextToken++; - return new Promise((resolve, reject) => { - this._predictPromises.make(token, resolve, reject); - this._worker.postMessage({ - message: 'predict', - token: token, - transform: transform, - context: context, - }); - }); - } - - wordbreak(context: Context): Promise { - let token = this._nextToken++; - return new Promise((resolve, reject) => { - this._wordbreakPromises.make(token, resolve, reject); - this._worker.postMessage({ - message: 'wordbreak', - token: token, - context: context - }) - }); - } - - acceptSuggestion(suggestion: Suggestion, context: Context, postTransform: Transform): Promise { - let token = this._nextToken++; - return new Promise((resolve, reject) => { - this._acceptPromises.make(token, resolve, reject); - this._worker.postMessage({ - message: 'accept', - token: token, - suggestion: suggestion, - context: context, - postTransform: postTransform - }); - }); - } - - revertSuggestion(reversion: Reversion, context: Context): Promise { - let token = this._nextToken++; - return new Promise((resolve, reject) => { - this._revertPromises.make(token, resolve, reject); - this._worker.postMessage({ - message: 'revert', - token: token, - reversion: reversion, - context: context - }) - }); - } - - resetContext(context: Context) { - this._worker.postMessage({ - message: 'reset-context', - context: context - }); - } - - // TODO: asynchronous close() method. - // Worker code must recognize message and call self.close(). - - private onMessage(event: MessageEvent): void { - let payload: OutgoingMessage = event.data; - if (payload.message === 'error') { - console.error(payload.log); - if(payload.error) { - console.error(payload.error); - } - } - else if (payload.message === 'ready') { - this._declareLMLayerReady(event.data.configuration); - } else if (payload.message === 'suggestions') { - this._predictPromises.keep(payload.token, payload.suggestions); - } else if (payload.message === 'currentword') { - this._wordbreakPromises.keep(payload.token, payload.word); - } else if (payload.message === 'postaccept') { - this._acceptPromises.keep(payload.token, payload.reversion); - } else if (payload.message === 'postrevert') { - this._revertPromises.keep(payload.token, payload.suggestions); - } else { - // This branch should never execute, but just in case... - //@ts-ignore - throw new Error(`Message not implemented: ${payload.message}`); - } - } - - /** - * Clears out any computational resources in use by the LMLayer, including shutting - * down any internal WebWorkers. - */ - public shutdown() { - this._worker.terminate(); - } - - /** - * Given a function, this utility returns the source code within it, as a string. - * This is intended to unwrap the "wrapped" source code created in the LMLayerWorker - * build process. - * - * @param fn The function whose body will be returned. - */ - static unwrap(fn: Function): string { - let wrapper = fn.toString(); - let match = wrapper.match(/function[^{]+{((?:.|\r|\n)+)}[^}]*$/); - return match[1]; - } - } -} - -(function () { - let ns = com.keyman.text.prediction; - - // Let LMLayer be available both in the browser and in Node. - if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { - module.exports = ns.LMLayer; - //@ts-ignore - ns.LMLayer.PromiseStore = ns.PromiseStore; - } else { - //@ts-ignore - window.LMLayer = ns.LMLayer; - } -}()); \ No newline at end of file diff --git a/common/predictive-text/node-defaultWorker.ts b/common/predictive-text/node-defaultWorker.ts deleted file mode 100644 index 3cf24dcf53714808e99f097176cafe201386dee9..0000000000000000000000000000000000000000 --- a/common/predictive-text/node-defaultWorker.ts +++ /dev/null @@ -1,12 +0,0 @@ -/// - -namespace com.keyman.text.prediction { - export class DefaultWorker { - static constructInstance(): Worker { - let scriptStr = LMLayer.unwrap(LMLayerWorkerCode); - let worker = new VirtualizedWorker(scriptStr); - - return worker; - } - } -} \ No newline at end of file diff --git a/common/predictive-text/package.json b/common/predictive-text/package.json index fda9b5bfb231fa4493d686762e2cdba571b72525..2e251fa90d2400faa1b5e383bafd1060648d832a 100644 --- a/common/predictive-text/package.json +++ b/common/predictive-text/package.json @@ -1,7 +1,29 @@ { "name": "@keymanapp/lexical-model-layer", "description": "Keyman/Predictive Text integration layer", - "main": "build/index.js", + "main": "build/obj/node/index.js", + "exports": { + ".": { + "node": { + "types": "./build/obj/node/index.d.ts", + "import": "./build/obj/index.js" + }, + "default": { + "types": "./build/obj/web/index.d.ts", + "import": "./build/obj/web/index.js" + } + }, + "./node": "./build/obj/node/index.js", + "./web": "./build/obj/web/index.js", + "./build/obj/node/index.js": "./build/obj/node/index.js", + "./build/obj/web/index.js": "./build/obj/web/index.js" + }, + "imports": { + "#./*.js": "./build/obj/*.js", + "#./node/*.js": "./build/obj/node/*.js", + "#./web/*.js": "./build/obj/web/*.js" + }, + "type": "module", "scripts": { "karma": "karma", "mocha": "mocha", diff --git a/common/predictive-text/promise-store.ts b/common/predictive-text/promise-store.ts deleted file mode 100644 index 4ef29ad7c013a827be3f9c56a23a88e6455a26b9..0000000000000000000000000000000000000000 --- a/common/predictive-text/promise-store.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) - * Copyright (c) 2018 SIL International - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - - -type Resolve = (value?: T | PromiseLike) => void; -type Reject = (reason?: any) => void; -interface PromiseCallbacks { - resolve: Resolve; - reject: Reject; -} - -namespace com.keyman.text.prediction { - /** - * Associate tokens with promises. - * - * First, .make() a promise -- associate a token with resolve/reject callbacks. - * - * You can either .keep() a promise -- resolve() and forget it; - * Or you may also .break() a promise -- reject() and forget it. - * - * is the type of resolved value (value yielded successfully by promise). - */ - export class PromiseStore { - // IE11 offers partial support for new Map(). - // Assume only .get(), .set(), .has(), .delete(), and .size work. - // See: http://kangax.github.io/compat-table/es6/#test-Map - private _promises: Map>; - constructor() { - this._promises = new Map(); - } - /** - * How many promises are currently being tracked? - */ - get length(): number { - return this._promises.size; - } - /** - * Associate a token with its respective resolve and reject callbacks. - */ - make(token: Token, resolve: Resolve, reject: Reject): void { - if (this._promises.has(token)) { - return reject(`Existing request with token ${token}`); - } - this._promises.set(token, { reject, resolve }); - } - /** - * Resolve the promise associated with a token (with a value!). - * Once the promise is resolved, the token is removed.. - */ - keep(token: Token, value: T) { - let callbacks = this._promises.get(token); - if (!callbacks) { - throw new Error(`No promise associated with token: ${token}`); - } - let accept = callbacks.resolve; - this._promises.delete(token); - return accept(value); - } - /** - * Instantly reject and forget a promise associated with the token. - */ - break(token: Token, reason?: any): void { - let callbacks = this._promises.get(token); - if (!callbacks) { - throw new Error(`No promise associated with token: ${token}`); - } - this._promises.delete(token); - callbacks.reject(reason); - } - } -} \ No newline at end of file diff --git a/common/predictive-text/src/index.ts b/common/predictive-text/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..e087e6957dc1e9cb0b35bfe66d1cf9ff836658f8 --- /dev/null +++ b/common/predictive-text/src/index.ts @@ -0,0 +1,3 @@ +export { default as LMLayer } from './lmlayer.js'; +export { default as NodeWorker } from './node/worker.js'; +export { default as WebWorker } from './web/worker.js'; \ No newline at end of file diff --git a/common/predictive-text/src/lmlayer.ts b/common/predictive-text/src/lmlayer.ts new file mode 100644 index 0000000000000000000000000000000000000000..f331273e657202f768a9c831bc5c6389c96fa702 --- /dev/null +++ b/common/predictive-text/src/lmlayer.ts @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) + * Copyright (c) 2018 SIL International + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +import PromiseStore from "./promise-store.js"; + +/// +/// + +/** + * Top-level interface to the Language Modelling layer, or "LMLayer" for short. + * + * The Language Modelling layer provides a way for keyboards to offer prediction and + * correction functionalities. The LMLayer proper runs within a Web Worker, however, + * this class is intended to run in the main thread, and automatically spawn a Web + * Worker, capable of offering predictions. + * + * Since the Worker runs in a different thread, the public methods of this class are + * asynchronous. Methods of note include: + * + * - #loadModel() -- loads a specified model file + * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for + * the input event + * - #unloadModel() -- unloads the LMLayer's currently loaded model, preparing it to + * receive (load) a new model + * + * The top-level LMLayer will automatically starts up its own Web Worker. + */ + +export default class LMLayer { + /** + * The underlying worker instance. By default, this is the LMLayerWorker. + */ + private _worker: Worker; + /** Call this when the LMLayer has sent us the 'ready' message! */ + private _declareLMLayerReady: (conf: Configuration) => void; + private _predictPromises: PromiseStore; + private _wordbreakPromises: PromiseStore; + private _acceptPromises: PromiseStore; + private _revertPromises: PromiseStore; + private _nextToken: number; + private capabilities: Capabilities; + + /** + * Construct the top-level LMLayer interface. This also starts the underlying Worker. + * + * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: + * or file: URI. If uri is not provided, this will start the default Worker. + */ + constructor(capabilities: Capabilities, worker: Worker, testMode?: boolean) { + // Either use the given worker, or instantiate the default worker. + this._worker = worker; + this._worker.onmessage = this.onMessage.bind(this) + this._declareLMLayerReady = null; + this._predictPromises = new PromiseStore(); + this._wordbreakPromises = new PromiseStore(); + this._acceptPromises = new PromiseStore(); + this._revertPromises = new PromiseStore(); + this._nextToken = Number.MIN_SAFE_INTEGER; + + this.sendConfig(capabilities, !!testMode); + } + + /** + * Initializes the LMLayer worker with the host platform's capability set. + * + * @param capabilities The host platform's capability spec - a model cannot assume access to more context + * than specified by this parameter. + */ + private sendConfig(capabilities: Capabilities, testMode: boolean) { + this._worker.postMessage({ + message: 'config', + capabilities: capabilities, + testMode: testMode + }); + } + + /** + * Initializes the LMLayer worker with a path to the desired model file. + */ + loadModel(modelSource: string, loadType: 'file' | 'raw' = 'file'): Promise { + return new Promise((resolve, _reject) => { + // Sets up so the promise is resolved in the onMessage() callback, when it receives + // the 'ready' message. + this._declareLMLayerReady = resolve; + + let modelSourceSpec: any = { + type: loadType + }; + + if(loadType == 'file') { + modelSourceSpec.file = modelSource; + } else { + modelSourceSpec.code = modelSource; + } + + this._worker.postMessage({ + message: 'load', + source: modelSourceSpec + }); + }); + } + + /** + * Unloads the previously-active model from memory, resetting the LMLayer to prep + * for transition to use of a new model. + */ + public unloadModel() { + this._worker.postMessage({ + message: 'unload' + }); + } + + predict(transform: Transform | Distribution, context: Context): Promise { + let token = this._nextToken++; + return new Promise((resolve, reject) => { + this._predictPromises.make(token, resolve, reject); + this._worker.postMessage({ + message: 'predict', + token: token, + transform: transform, + context: context, + }); + }); + } + + wordbreak(context: Context): Promise { + let token = this._nextToken++; + return new Promise((resolve, reject) => { + this._wordbreakPromises.make(token, resolve, reject); + this._worker.postMessage({ + message: 'wordbreak', + token: token, + context: context + }) + }); + } + + acceptSuggestion(suggestion: Suggestion, context: Context, postTransform: Transform): Promise { + let token = this._nextToken++; + return new Promise((resolve, reject) => { + this._acceptPromises.make(token, resolve, reject); + this._worker.postMessage({ + message: 'accept', + token: token, + suggestion: suggestion, + context: context, + postTransform: postTransform + }); + }); + } + + revertSuggestion(reversion: Reversion, context: Context): Promise { + let token = this._nextToken++; + return new Promise((resolve, reject) => { + this._revertPromises.make(token, resolve, reject); + this._worker.postMessage({ + message: 'revert', + token: token, + reversion: reversion, + context: context + }) + }); + } + + resetContext(context: Context) { + this._worker.postMessage({ + message: 'reset-context', + context: context + }); + } + + // TODO: asynchronous close() method. + // Worker code must recognize message and call self.close(). + + private onMessage(event: MessageEvent): void { + let payload: OutgoingMessage = event.data; + if (payload.message === 'error') { + console.error(payload.log); + if(payload.error) { + console.error(payload.error); + } + } + else if (payload.message === 'ready') { + this._declareLMLayerReady(event.data.configuration); + } else if (payload.message === 'suggestions') { + this._predictPromises.keep(payload.token, payload.suggestions); + } else if (payload.message === 'currentword') { + this._wordbreakPromises.keep(payload.token, payload.word); + } else if (payload.message === 'postaccept') { + this._acceptPromises.keep(payload.token, payload.reversion); + } else if (payload.message === 'postrevert') { + this._revertPromises.keep(payload.token, payload.suggestions); + } else { + // This branch should never execute, but just in case... + //@ts-ignore + throw new Error(`Message not implemented: ${payload.message}`); + } + } + + /** + * Clears out any computational resources in use by the LMLayer, including shutting + * down any internal WebWorkers. + */ + public shutdown() { + this._worker.terminate(); + } +} diff --git a/common/predictive-text/src/node/index.ts b/common/predictive-text/src/node/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..1a70a48bf30393310e4a59815ced628fd69baa36 --- /dev/null +++ b/common/predictive-text/src/node/index.ts @@ -0,0 +1,3 @@ +export { default as LMLayer } from '../lmlayer.js'; +export { default as Worker } from './worker.js'; +export { default as SourcemappedWorker } from './sourcemappedWorker.js'; \ No newline at end of file diff --git a/common/predictive-text/src/node/mappedWorker.ts b/common/predictive-text/src/node/mappedWorker.ts new file mode 100644 index 0000000000000000000000000000000000000000..aafbba52323d6f516f40d867b449cd5e07377aac --- /dev/null +++ b/common/predictive-text/src/node/mappedWorker.ts @@ -0,0 +1,88 @@ +// We use a subset of the Worker interface here; compiling directly against the true +// WebWorker type definitions would require us to implement more methods than we do. +/// + +// Defines types related to Node workers. +import * as worker from 'worker_threads'; +import { Buffer } from 'buffer'; +import { URL } from 'url'; + +/** + * Defines mappings from Node Worker signatures to WebWorker signatures + */ +const nodeWorkerToWebWorkerMappingSource = ` +import { parentPort } from 'worker_threads'; +import fs from 'fs'; +import vm from 'vm'; + +function postMessage(...args) { + parentPort.postMessage.call(parentPort, args); +} + +parentPort.on('message', (ev) => { + onmessage({data: ev}); +}); + +function importScripts(...args) { + function loadScriptInContext(scriptPath) { + let scriptStr = fs.readFileSync(scriptPath); + var script = new vm.Script(scriptStr, { filename: scriptPath }); + script.runInThisContext(); + } + + for(let arg of args) { + loadScriptInContext(arg); + } +} + +/* + * You'd think the method signature mapping would be implied from the first line, + * but all three lines must be explicitly specified or the emulation will fail. + */ +const self = globalThis; +self.postMessage = postMessage; +self.importScripts = importScripts; +`; + +/** + * Uses the Node version of Workers to provide proper, authentic separate-thread + * 'sandboxing'. Also intercepts and interprets certain WebWorker method signatures + * necessary to run the WebWorker-oriented worker code. + * + * Alternatively, only after writing this did I discover this package: + * https://github.com/developit/web-worker. They also ran one notable issue I did: + * Node 18.x, at least, does not support use of Node Blobs for construction of a + * Worker: https://github.com/developit/web-worker/pull/32... unlike Web Workers. + * + * So... Base64-encoded Data URLs it is. + * + * What we have here is perfectly fine for now, but if we need more complicated + * cross-platform Worker support in the future, it may be wise to swap to use of + * that package. + */ +export default class MappedWorker extends worker.Worker implements Worker { + constructor(scriptStr: string) { + const concatenatedScript = ` + ${nodeWorkerToWebWorkerMappingSource} + + ${scriptStr} + `; + const buffer = Buffer.from(concatenatedScript); + const dataSrc = "data:text/javascript;base64," + buffer.toString('base64'); + //@ts-ignore + super(new URL(dataSrc)); + + // WebWorkers have a defined `onmessage` function, rather than this.on('message', ...) + this.on('message', (ev) => { + if(this.onmessage) { + this.onmessage({data: ev[0]}); + } + }); + } + + /** + * Accepts a callback function that will receive messages sent from the `VirtualizedWorker`'s `postMessage` function, + * much like the standard `Worker.onmessage`. + */ + onmessage: (this: Worker, ev: MessageEvent) => any; +} \ No newline at end of file diff --git a/common/predictive-text/src/node/sourcemappedWorker.ts b/common/predictive-text/src/node/sourcemappedWorker.ts new file mode 100644 index 0000000000000000000000000000000000000000..aaaf79e0f6fcc1f1d42207dd767b4663f4f09e95 --- /dev/null +++ b/common/predictive-text/src/node/sourcemappedWorker.ts @@ -0,0 +1,19 @@ +import MappedWorker from "./mappedWorker.js"; +import unwrap from '../unwrap.js'; + +import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.js"; + +export default class SourcemappedWorker { + static constructInstance(): Worker { + let scriptStr = unwrap(LMLayerWorkerCode); + + // If this is definitively set to either true or false, tree-shaking can take effect. + // An imported const variable doesn't seem to do it, though. + // if(false) { + scriptStr += '\n' + LMLayerWorkerSourcemapComment; + // } + let worker = new MappedWorker(scriptStr); + + return worker as any as Worker; + } +} \ No newline at end of file diff --git a/common/predictive-text/src/node/tsconfig.json b/common/predictive-text/src/node/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..dacf350335ead81ed5db44aea20fa1d1277d73d5 --- /dev/null +++ b/common/predictive-text/src/node/tsconfig.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": false, + "declaration": true, + "module": "es6", + "moduleResolution": "Node16", + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman", + // This build-configuration only targets Node and exists solely for unit tests. + // Also, we need ES6 in order to properly inherit from the Node Worker-type. + "target": "ES6", + "lib": ["es6"], + "types": ["node"], + "baseUrl": "../", + "outDir": "../../build/obj", + "tsBuildInfoFile": "../../build/obj/node/tsconfig.tsbuildinfo", + "rootDir": "../" + }, + "include": [ + "*.ts" + ], + "references": [ + { "path": "../tsconfig.json" }, + { "path": "../../../web/lm-worker" } + ] +} \ No newline at end of file diff --git a/common/predictive-text/src/node/worker.ts b/common/predictive-text/src/node/worker.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ad04e55d0be08b11b34073ef60d72b1e25981b7 --- /dev/null +++ b/common/predictive-text/src/node/worker.ts @@ -0,0 +1,15 @@ +import MappedWorker from "./mappedWorker.js"; +import unwrap from '../unwrap.js'; + +import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.min.js"; + +export default class Worker { + static constructInstance(): Worker { + let scriptStr = unwrap(LMLayerWorkerCode); + + scriptStr += '\n' + LMLayerWorkerSourcemapComment; + let worker = new MappedWorker(scriptStr); + + return worker as any as Worker; + } +} \ No newline at end of file diff --git a/common/predictive-text/src/promise-store.ts b/common/predictive-text/src/promise-store.ts new file mode 100644 index 0000000000000000000000000000000000000000..890ba5923929a57ffa6fdc0ffe223527b0f590e2 --- /dev/null +++ b/common/predictive-text/src/promise-store.ts @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) + * Copyright (c) 2018 SIL International + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/// + +type Resolve = (value?: T | PromiseLike) => void; +type Reject = (reason?: any) => void; +interface PromiseCallbacks { + resolve: Resolve; + reject: Reject; +} + + +/** + * Associate tokens with promises. + * + * First, .make() a promise -- associate a token with resolve/reject callbacks. + * + * You can either .keep() a promise -- resolve() and forget it; + * Or you may also .break() a promise -- reject() and forget it. + * + * is the type of resolved value (value yielded successfully by promise). + */ +export default class PromiseStore { + // IE11 offers partial support for new Map(). + // Assume only .get(), .set(), .has(), .delete(), and .size work. + // See: http://kangax.github.io/compat-table/es6/#test-Map + private _promises: Map>; + constructor() { + this._promises = new Map(); + } + /** + * How many promises are currently being tracked? + */ + get length(): number { + return this._promises.size; + } + /** + * Associate a token with its respective resolve and reject callbacks. + */ + make(token: Token, resolve: Resolve, reject: Reject): void { + if (this._promises.has(token)) { + return reject(`Existing request with token ${token}`); + } + this._promises.set(token, { reject, resolve }); + } + /** + * Resolve the promise associated with a token (with a value!). + * Once the promise is resolved, the token is removed.. + */ + keep(token: Token, value: T) { + let callbacks = this._promises.get(token); + if (!callbacks) { + throw new Error(`No promise associated with token: ${token}`); + } + let accept = callbacks.resolve; + this._promises.delete(token); + return accept(value); + } + /** + * Instantly reject and forget a promise associated with the token. + */ + break(token: Token, reason?: any): void { + let callbacks = this._promises.get(token); + if (!callbacks) { + throw new Error(`No promise associated with token: ${token}`); + } + this._promises.delete(token); + callbacks.reject(reason); + } +} \ No newline at end of file diff --git a/common/predictive-text/src/tsconfig.json b/common/predictive-text/src/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..f6fe671043ff00b8c01edca82b3eefbdcbbda656 --- /dev/null +++ b/common/predictive-text/src/tsconfig.json @@ -0,0 +1,34 @@ +{ + "extends": "../../../tsconfig-base.json", + + "compilerOptions": { + "allowJs": false, + "declaration": true, + "module": "es6", + "moduleResolution": "Node16", + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman", + "target": "es5", + "lib": ["es6"], + "types": ["node"], + "baseUrl": "./", + "outDir": "../build/obj", + "tsBuildInfoFile": "../build/obj/tsconfig.tsbuildinfo", + "rootDir": "./" + }, + "references": [ + { "path": "../../web/utils" }, + { "path": "../../models/types"}, + { "path": "../../models/wordbreakers"}, + { "path": "../../web/lm-message-types"}, + { "path": "../../models/templates"} + ], + "include" : [ "./*.ts" ], + "exclude" : [ + "node", + "web", + "index.ts" // It's handled by the final layer. + ] +} diff --git a/common/predictive-text/src/unwrap.ts b/common/predictive-text/src/unwrap.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc0769e5bd112e3738169f0cc9de63bcf8b920af --- /dev/null +++ b/common/predictive-text/src/unwrap.ts @@ -0,0 +1,11 @@ +/** + * Given a function, this utility returns the source code within it, as a string. + * This is intended to unwrap the "wrapped" source code created in the LMLayerWorker + * build process. + * + * @param fn The function whose body will be returned. + */ +export default function unwrap(encodedSrc: string): string { + // There used to be more to this, but now it's a pretty simple passthrough! + return encodedSrc; +} \ No newline at end of file diff --git a/common/predictive-text/src/web/index.ts b/common/predictive-text/src/web/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..1a70a48bf30393310e4a59815ced628fd69baa36 --- /dev/null +++ b/common/predictive-text/src/web/index.ts @@ -0,0 +1,3 @@ +export { default as LMLayer } from '../lmlayer.js'; +export { default as Worker } from './worker.js'; +export { default as SourcemappedWorker } from './sourcemappedWorker.js'; \ No newline at end of file diff --git a/common/predictive-text/src/web/sourcemappedWorker.ts b/common/predictive-text/src/web/sourcemappedWorker.ts new file mode 100644 index 0000000000000000000000000000000000000000..da7ed77eb96393f01e5737404a6a26c8256e98e5 --- /dev/null +++ b/common/predictive-text/src/web/sourcemappedWorker.ts @@ -0,0 +1,34 @@ +import unwrap from '../unwrap.js'; +import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.js"; + +export default class SourcemappedWorker { + static constructInstance(): Worker { + return new Worker(this.asBlobURI(LMLayerWorkerCode)); + } + + /** + * Converts the INSIDE of a function into a blob URI that can + * be passed as a valid URI for a Worker. + * @param fn Function whose body will be referenced by a URI. + * + * This function makes the following possible: + * + * let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () { + * postMessage('inside Web Worker') + * function onmessage(event) { + * // handle message inside Web Worker. + * } + * })); + */ + static asBlobURI(encodedSrc: string): string { + let code = unwrap(encodedSrc); + + // If this is definitively set to either true or false, tree-shaking can take effect. + // An imported const variable doesn't seem to do it, though. + // if(false) { + code += '\n' + LMLayerWorkerSourcemapComment; + // } + let blob = new Blob([code], { type: 'text/javascript' }); + return URL.createObjectURL(blob); + } +} \ No newline at end of file diff --git a/common/predictive-text/src/web/tsconfig.json b/common/predictive-text/src/web/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..bcb94fca1060674a915cf39762dd09712f8c92dd --- /dev/null +++ b/common/predictive-text/src/web/tsconfig.json @@ -0,0 +1,26 @@ +{ + "extends": "../../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": false, + "declaration": true, + "module": "es6", + "moduleResolution": "Node16", + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman", + "target": "es5", + "lib": ["es6", "DOM"], + "types": ["node"], + "baseUrl": "../", + "outDir": "../../build/obj", + "tsBuildInfoFile": "../../build/obj/web/tsconfig.tsbuildinfo", + "rootDir": "../" + }, + "include": [ + "*.ts" + ], + "references": [ + { "path": "../tsconfig.json" } + ] +} \ No newline at end of file diff --git a/common/predictive-text/src/web/worker.ts b/common/predictive-text/src/web/worker.ts new file mode 100644 index 0000000000000000000000000000000000000000..372887fcddf22c499ef7bc54b356acbd668a5c87 --- /dev/null +++ b/common/predictive-text/src/web/worker.ts @@ -0,0 +1,34 @@ +import unwrap from '../unwrap.js'; +import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.min.js"; + +export default class DefaultWorker { + static constructInstance(): Worker { + return new Worker(this.asBlobURI(LMLayerWorkerCode)); + } + + /** + * Converts the INSIDE of a function into a blob URI that can + * be passed as a valid URI for a Worker. + * @param fn Function whose body will be referenced by a URI. + * + * This function makes the following possible: + * + * let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () { + * postMessage('inside Web Worker') + * function onmessage(event) { + * // handle message inside Web Worker. + * } + * })); + */ + static asBlobURI(encodedSrc: string): string { + let code = unwrap(encodedSrc); + + // If this is definitively set to either true or false, tree-shaking can take effect. + // An imported const variable doesn't seem to do it, though. + // if(false) { + code += '\n' + LMLayerWorkerSourcemapComment; + // } + let blob = new Blob([code], { type: 'text/javascript' }); + return URL.createObjectURL(blob); + } +} \ No newline at end of file diff --git a/common/predictive-text/worker-interface.d.ts b/common/predictive-text/src/worker-interface.d.ts similarity index 100% rename from common/predictive-text/worker-interface.d.ts rename to common/predictive-text/src/worker-interface.d.ts diff --git a/common/predictive-text/tsconfig.all.json b/common/predictive-text/tsconfig.all.json new file mode 100644 index 0000000000000000000000000000000000000000..5cb3fdf96bfec7ce985c0b17641328dd897a747e --- /dev/null +++ b/common/predictive-text/tsconfig.all.json @@ -0,0 +1,35 @@ +{ + // Due to the heterogenous target mix supported by the LMLayer, the two different "worker" types + // require separate sub-tsconfigs, both of which require common references. The multi-config + // setup in this project and its subfolders ensure all modules can compile cleanly and in the + // necessary compilation order. + "extends": "../../tsconfig-base.json", + + "compilerOptions": { + "allowJs": false, + "declaration": true, + "module": "es6", + "moduleResolution": "Node16", + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman", + "target": "es5", + "lib": ["es6"], + "types": ["node"], + "baseUrl": "./", + "outDir": "build/obj", + "tsBuildInfoFile": "build/obj/tsconfig.all.tsbuildinfo", + "rootDir": "./src" + }, + "files": [], + "references": [ + { "path": "../web/utils" }, + { "path": "../models/types"}, + { "path": "../models/wordbreakers"}, + { "path": "../web/lm-message-types"}, + { "path": "../models/templates"}, + { "path": "src/node" }, + { "path": "src/web" } + ], +} diff --git a/common/predictive-text/tsconfig.json b/common/predictive-text/tsconfig.json deleted file mode 100644 index c364f699a12ef53fcf06a33ed92b6ca28554a255..0000000000000000000000000000000000000000 --- a/common/predictive-text/tsconfig.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "extends": "../../tsconfig-base.json", - - "compilerOptions": { - "allowJs": true, - "outFile": "build/headless.js", - "module": "none", - "inlineSources": true, - "sourceMap": true, - "target": "es5", - "lib": ["es6"] - }, - "references": [ - { "path": "../web/utils" }, - { "path": "../models/types"}, - { "path": "../models/wordbreakers"}, - { "path": "../web/lm-message-types"}, - { "path": "../models/templates"}, - ], - "include" : [ "./*.ts" ], - "exclude" : [ - "web-defaultWorker.ts", - "node_modules", - "testing" - ], - "files": [ - // By renaming the output JS as TS, and including the @ts-nocheck directive, we can - // bypass wrapped-worker typing issues while still bundling the file as part of the - // tsc build! - "../web/lm-worker/build/index.wrapped-for-bundle.ts" - ] -} diff --git a/common/predictive-text/unit_tests/headless/promise-store.js b/common/predictive-text/unit_tests/headless/promise-store.js index f14b9c8555626377f7aa4eb57fdb7bb96d59f256..0809b215ce71386e0b8002d5d5a1212570afc529 100644 --- a/common/predictive-text/unit_tests/headless/promise-store.js +++ b/common/predictive-text/unit_tests/headless/promise-store.js @@ -1,7 +1,8 @@ -var assert = require('chai').assert; -var sinon = require('sinon'); +import { assert } from 'chai'; +import sinon from 'sinon'; -let PromiseStore = require('../../build/headless').PromiseStore; +import PromiseStore from '#./promise-store.js'; +import { randomToken } from '@keymanapp/common-test-resources/model-helpers.mjs'; describe('PromiseStore', function () { describe('.make()', function () { diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index f4b891c8e088049888b113b57a690fc6dc0009ba..78b071e3c427ddaa07bdfeb1559ddd790d363c01 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -1,11 +1,29 @@ -var assert = require('chai').assert; -var sinon = require('sinon'); +import { assert } from 'chai'; +import sinon from 'sinon'; -let LMLayer = require('../../build/headless'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { LMLayer } from '#./node/index.js'; +import { capabilities } from '@keymanapp/common-test-resources/model-helpers.mjs'; // Test the top-level LMLayer interface. // Note: these tests can only be run after BOTH stages of compilation are completed. describe('LMLayer', function() { + /** + * Returns an object implementing *enough* of the Worker + * interface to fool the LMLayer into thinking it's + * communicating with a bona fide Web Worker. + * + * @returns {Worker} an object with sinon.fake() instances. + */ + function createFakeWorker(postMessage) { + return { + postMessage: postMessage ? sinon.fake(postMessage) : sinon.fake(), + onmessage: null + }; + } + describe('[[constructor]]', function () { it('should accept a Worker to instantiate', function () { new LMLayer(capabilities(), createFakeWorker()); @@ -29,7 +47,7 @@ describe('LMLayer', function() { let fakeWorker = createFakeWorker(); let lmLayer = new LMLayer(capabilities(), fakeWorker); - lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); + lmLayer.loadModel(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")); assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); }); @@ -37,7 +55,7 @@ describe('LMLayer', function() { it('should send the `load` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); let lmLayer = new LMLayer(capabilities(), fakeWorker); - let configuration = await lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); + let configuration = await lmLayer.loadModel(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")); assert.propertyVal(fakeWorker.postMessage, 'callCount', 2); // In the "Worker", assert the message looks right and @@ -95,34 +113,6 @@ describe('LMLayer', function() { }) }); - // Since the Blob API is limited to browsers, look for those - // tests for .asBlobURI() in the in_browser tests. - describe('.unwrap', function () { - it('should return the inner code of a function', function () { - // Create a multi-line function body we can match in a RegExp. - let text = LMLayer.unwrap(function hello() { - var hello; - var world; - }); - // Unwrap should give us back ONLY the body. Whitespace isn't really important. - assert.match(text, /^\s*var\s+hello;\s*var\s+world;\s*$/); - }); - }); - - /** - * Returns an object implementing *enough* of the Worker - * interface to fool the LMLayer into thinking it's - * communicating with a bona fide Web Worker. - * - * @returns {Worker} an object with sinon.fake() instances. - */ - function createFakeWorker(postMessage) { - return { - postMessage: postMessage ? sinon.fake(postMessage) : sinon.fake(), - onmessage: null - }; - } - /** * Call a function in the future, i.e., later in the event loop. * The call does NOT block the current execution. diff --git a/common/predictive-text/unit_tests/headless/worker-dummy-integration.js b/common/predictive-text/unit_tests/headless/worker-dummy-integration.js index f1d8c375d40e3dc07bcd80f8335c808c34ae8ea0..d546edb70403a4e4e6e1a11f126f3de3d41a2197 100644 --- a/common/predictive-text/unit_tests/headless/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/headless/worker-dummy-integration.js @@ -1,9 +1,11 @@ -var assert = require('chai').assert; -var fs = require('fs'); -let LMLayer = require('../../build/headless'); +import { assert } from 'chai'; +import fs from 'fs'; -// Load the LMLayerWorkerCode function into this context -require('node:vm').runInThisContext(fs.readFileSync(__dirname + '/../../../web/lm-worker/build/index.js', 'utf-8')); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { LMLayer, SourcemappedWorker as Worker } from '#./node/index.js'; +import { capabilities, iGotDistractedByHazel } from '@keymanapp/common-test-resources/model-helpers.mjs'; /* * Shows off the LMLayer API, using the full prediction interface. @@ -14,10 +16,23 @@ require('node:vm').runInThisContext(fs.readFileSync(__dirname + '/../../../web/l * of suggestions when loaded and return them sequentially. */ describe('LMLayer using dummy model', function () { + let lmLayer; + let worker; + + beforeEach(function() { + worker = Worker.constructInstance(); + lmLayer = new LMLayer(capabilities(), worker); + }); + + afterEach(function () { + // As we're using Node worker threads here, failure to terminate them will cause the + // headless test run to hang after completion. + lmLayer.shutdown(); + worker.terminate(); // should be covered by the former, but just in case... for CI stability. + }); + describe('Prediction', function () { it('will predict future suggestions (loaded from file)', function () { - var lmLayer = new LMLayer(capabilities()); - var stripIDs = function(suggestions) { suggestions.forEach(function(suggestion) { delete suggestion.id; @@ -29,7 +44,7 @@ describe('LMLayer using dummy model', function () { // Not done yet, as this test case is a slightly-edited copy of the in-browser version. return lmLayer.loadModel( // We're running headlessly, so the path can be relative to the npm root directory. - "unit_tests/in_browser/resources/models/simple-dummy.js" + require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js") ).then(function (actualConfiguration) { return Promise.resolve(); }).then(function () { @@ -49,14 +64,11 @@ describe('LMLayer using dummy model', function () { }).then(function (suggestions) { stripIDs(suggestions); assert.deepEqual(suggestions, iGotDistractedByHazel()[3]); - lmLayer.shutdown(); return Promise.resolve(); }); }); it('will predict future suggestions (loaded from raw source)', function () { - var lmLayer = new LMLayer(capabilities()); - var stripIDs = function(suggestions) { suggestions.forEach(function(suggestion) { delete suggestion.id; @@ -64,7 +76,7 @@ describe('LMLayer using dummy model', function () { }; // We're running headlessly, so the path can be relative to the npm root directory. - let modelCode = fs.readFileSync("./unit_tests/in_browser/resources/models/simple-dummy.js").toString(); + let modelCode = fs.readFileSync(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")).toString(); // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax. @@ -90,7 +102,6 @@ describe('LMLayer using dummy model', function () { }).then(function (suggestions) { stripIDs(suggestions); assert.deepEqual(suggestions, iGotDistractedByHazel()[3]); - lmLayer.shutdown(); return Promise.resolve(); }); }); @@ -98,14 +109,12 @@ describe('LMLayer using dummy model', function () { describe('Wordbreaking', function () { it('will perform (default) wordbreaking and return word at caret', function () { - var lmLayer = new LMLayer(capabilities()); - // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax. // Not done yet, as this test case is a slightly-edited copy of the in-browser version. return lmLayer.loadModel( // We're running headlessly, so the path can be relative to the npm root directory. - "unit_tests/in_browser/resources/models/simple-dummy.js" + require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js") ).then(function (actualConfiguration) { return Promise.resolve(); }).then(function () { diff --git a/common/predictive-text/unit_tests/headless/worker-trie-integration.js b/common/predictive-text/unit_tests/headless/worker-trie-integration.js index ca543301a20e9337d8b209fcfe0ceb491c21f727..8cd257878d1491e56937ba5f01bbcb41267343a9 100644 --- a/common/predictive-text/unit_tests/headless/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/headless/worker-trie-integration.js @@ -1,21 +1,39 @@ -var assert = require('chai').assert; -let LMLayer = require('../../build/headless'); +import { assert } from 'chai'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { LMLayer, SourcemappedWorker as Worker } from '#./node/index.js'; +import { capabilities } from '@keymanapp/common-test-resources/model-helpers.mjs'; /* * How to run the worlist */ describe('LMLayer using the trie model', function () { + let lmLayer; + let worker; + + beforeEach(function() { + worker = Worker.constructInstance(); + lmLayer = new LMLayer(capabilities(), worker); + }); + + afterEach(function () { + // As we're using Node worker threads here, failure to terminate them will cause the + // headless test run to hang after completion. + lmLayer.shutdown(); + worker.terminate(); // should be covered by the former, but just in case... for CI stability. + }); + describe('Prediction', function () { var EXPECTED_SUGGESTIONS = 3; it('will predict an empty buffer', function () { - var lmLayer = new LMLayer(capabilities()); - // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax. // Not done yet, as this test case is a slightly-edited copy of the in-browser version. return lmLayer.loadModel( // We're running headlessly, so the path can be relative to the npm root directory. - "unit_tests/in_browser/resources/models/simple-trie.js" + require.resolve("@keymanapp/common-test-resources/models/simple-trie.js") ).then(function (_actualConfiguration) { return Promise.resolve(); }).then(function () { @@ -50,11 +68,9 @@ describe('LMLayer using the trie model', function () { // // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 it('should use the default searchTermToKey()', function () { - var lmLayer = new LMLayer(capabilities()); - return lmLayer.loadModel( // We're running headlessly, so the path can be relative to the npm root directory. - "unit_tests/in_browser/resources/models/naive-trie.js" + require.resolve("@keymanapp/common-test-resources/models/naive-trie.js") ).then(function (_actualConfiguration) { return Promise.resolve(); }).then(function () { diff --git a/common/predictive-text/unit_tests/helpers.js b/common/predictive-text/unit_tests/helpers.js deleted file mode 100644 index 1ece33428bca1d7372bbc1e8f723d54bec11b7ca..0000000000000000000000000000000000000000 --- a/common/predictive-text/unit_tests/helpers.js +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @file helpers - * - * Globally-defined helper functions for use in in Mocha tests. - */ - -var fs = require("fs"); -var vm = require("vm"); - -// Choose the appropriate global object. Either `global` in -// Node, or `window` in browsers. -var _ = global || window; - -/** - * Creates a MessageEvent (for inter-worker communication), with the given data payload. - * - * @param {*} data - */ -_.createMessageEventWithData = function createMessageEventWithData(data) { - return { data }; -} - -/** - * Creates a simple, default capabilities object for standard-case LMLayer init. - */ -_.capabilities = function capabilities() { - return { - maxLeftContextCodePoints: 64 - } -} - -/** - * Mimics a message from the outer LMLayer shell with a simple, default config object. - * Used for Worker tests. - */ -_.configWorker = function configWorker(worker) { - worker.onMessage(createMessageEventWithData({ - message: 'config', - capabilities: _.capabilities() - })); -} - -/** - * A valid model that suggests exactly what you want it to suggest. - * - * @returns {ModelDescription} - */ -_.dummyModel = function dummyModel(futureSuggestions) { - return { - type: 'dummy', - futureSuggestions: futureSuggestions || [] - }; -} -/** - * Capabilities of a keyboard that will ONLY send left-sided capabilities. - * The keyboard does not support deleting to the right. - * - * @returns {Capabilities} - */ -_.defaultCapabilities = function defaultCapabilities() { - return { - maxLeftContextCodeUnits: 64 - }; -} - -/** - * Returns the Context of an empty buffer; no text, at both the start and - * end of the buffer. - * - * @returns {Context} - */ -_.emptyContext = function emptyContext() { - return { - left: '', - startOfBuffer: true, - endOfBuffer: true - }; -} - -/** - * Returns a Transform that, when applied, makes no changes to the buffer. - * - * @returns {Transform} - */ -_.zeroTransform = function zeroTransform() { - return { - insert: '', - deleteLeft: 0, - }; -} - -/** - * Returns a random token. NOT guaranteed to be unique. - * - * @returns {Token} - */ -_.randomToken = function randomToken() { - var range = Number.MAX_SAFE_INTEGER - Number.MIN_SAFE_INTEGER; - return Math.random() * range + Number.MIN_SAFE_INTEGER; -} - -// Use fixtures used in browser tests IN NODE! -if (typeof require === 'function') { - _.iGotDistractedByHazel = function () { - return jsonFixture('future_suggestions/i_got_distracted_by_hazel'); - } - - /** - * Return a JSON fixture - */ - _.jsonFixture = function (name) { - // Assuming this file structure: - // . - // ├── helpers.js - // └── in_browser - // └── json - // ├── future_suggestions - // │ └── ... - // └── wordlists - // └── ... - return require('./in_browser/json/' + name); - } - - // This worker-global function does not exist by default in Node! - _.importScriptsWith = function(context) { - return function() { // the constructed context's importScripts method. - - /* Use of vm.createContext and script.runInContext allow us to avoid - * polluting the global scope with imports. When we throw away the - * context object, imported scripts will be automatically GC'd. - */ - for(var i=0; i < arguments.length; i++) { - context = vm.createContext(context); - var script = new vm.Script(fs.readFileSync(arguments[i])); - script.runInContext(context); - } - } - } -} diff --git a/common/predictive-text/unit_tests/in_browser/CI.conf.cjs b/common/predictive-text/unit_tests/in_browser/CI.conf.cjs new file mode 100644 index 0000000000000000000000000000000000000000..eaedb4a1ca6d9190c49f0ef4b1347593ec46773d --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/CI.conf.cjs @@ -0,0 +1,4 @@ +var BASE_CONFIG = require("./base.conf.cjs"); +var ci_config_adapter = require("../../../test/resources/karma-browserstack-config.cjs"); + +module.exports = ci_config_adapter(BASE_CONFIG, "full"); \ No newline at end of file diff --git a/common/predictive-text/unit_tests/in_browser/CI.conf.js b/common/predictive-text/unit_tests/in_browser/CI.conf.js deleted file mode 100644 index d3d2b02e400591819170741716f02af73dce9277..0000000000000000000000000000000000000000 --- a/common/predictive-text/unit_tests/in_browser/CI.conf.js +++ /dev/null @@ -1,4 +0,0 @@ -var BASE_CONFIG = require("./base.conf.js"); -var ci_config_adapter = require("../../../../common/test/resources/karma-browserstack-config.js"); - -module.exports = ci_config_adapter(BASE_CONFIG, "full"); \ No newline at end of file diff --git a/common/predictive-text/unit_tests/in_browser/base.conf.js b/common/predictive-text/unit_tests/in_browser/base.conf.cjs similarity index 55% rename from common/predictive-text/unit_tests/in_browser/base.conf.js rename to common/predictive-text/unit_tests/in_browser/base.conf.cjs index 2c62af18549764d6fb37f87ac5aa838cd1abc103..6fa9de9f22ddf4045dd12506d91b5bac618751ed 100644 --- a/common/predictive-text/unit_tests/in_browser/base.conf.js +++ b/common/predictive-text/unit_tests/in_browser/base.conf.cjs @@ -4,7 +4,7 @@ module.exports = { // base path that will be used to resolve all patterns (eg. files, exclude) - basePath: './', + basePath: '../../../../', // Root directory of the repository(!) client: { /* `client.args` here is passed to the test runner page as `__karma__.config.args`. @@ -31,16 +31,27 @@ module.exports = { // list of files / patterns to load in the browser files: [ - // Include the generated worker code. Make sure it's linked before any of the test cases. - '../../../web/lm-worker/build/index.js', - '../../build/index.js', - '../../../../common/test/resources/timeout-adapter.js', - 'helpers.js', // Provides utility helpers and objects for tests. - 'cases/**/*.js', // Where the tests actually reside. + // { pattern: 'node_modules/**/*.js', served: true, watched: false, included: false }, + // { pattern: 'node_modules/**/*.mjs', served: true, watched: false, included: false, type: "module" }, + // { pattern: 'node_modules/**/*.cjs', served: true, watched: false, included: false }, + + // Provides utility helpers and objects for tests. + { pattern: 'common/predictive-text/unit_tests/in_browser/helpers.mjs', served: true, watched: true, included: false}, + + // Where the tests actually reside. + { pattern: 'common/predictive-text/unit_tests/in_browser/cases/**/*.js', type: 'module' }, + + 'common/test/resources/*.js', + 'common/test/resources/json/models/**/*.json', + { pattern: 'common/web/lm-worker/build/lib/*.js', watched: true, served: true, included: false}, + { pattern: 'common/web/lm-worker/build/lib/*.js.map', watched: true, served: true, included: false}, + { pattern: 'common/predictive-text/build/obj/**/*.*', watched: true, served: true, included: false }, + { pattern: 'common/predictive-text/build/obj/**/*.js.map', watched: true, served: true, included: false }, + { pattern: 'common/predictive-text/build/lib/**/*.*', watched: true, served: true, included: false }, + { pattern: 'common/predictive-text/build/lib/**/*.js.map', watched: true, served: true, included: false }, // We don't have anything in these locations... yet. But they'll be useful for test resources. - 'json/**/*.json', // Where pre-loaded JSON resides. - {pattern: 'resources/**/*.*', watched: true, served: true, included: false}, // General testing resources. + {pattern: 'common/test/resources/**/*.*', watched: true, served: true, included: false}, // General testing resources. ], // list of files / patterns to exclude @@ -50,17 +61,19 @@ module.exports = { // preprocess matching files before serving them to the browser // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { - 'json/**/*.json' : ['json_fixtures'] + 'common/test/resources/json/models/**/*.json' : ['json_fixtures'] }, // Settings to properly configure how JSON fixtures are automatically loaded by Karma. jsonFixturesPreprocessor: { - stripPrefix: 'json/', + stripPrefix: 'common/test/resources/json/', variableName: '__json__' }, proxies: { - "/resources/": "/base/resources/" + "/resources/": "/base/common/test/resources/", + // "/node_modules/": "/base/node_modules/", + // "/@keymanapp/lm-worker/": "/base/node_modules/@keymanapp/lm-worker/" }, // web server port diff --git a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js index 12c0c2aff136aa65677fd7eca621ce253874b1c4..d2481e522cac955c4f60a10684938dd286bf4eb7 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js @@ -1,22 +1,23 @@ var assert = chai.assert; -var LMLayer = com.keyman.text.prediction.LMLayer; -var DefaultWorker = com.keyman.text.prediction.DefaultWorker; + +import { LMLayer, Worker as WorkerBuilder } from "../../../build/lib/web/index.mjs"; +import * as helpers from "../helpers.mjs"; describe('LMLayer', function () { this.timeout(testconfig.timeouts.standard); describe('[[constructor]]', function () { it('should construct with a single argument', function () { - let lmLayer = new LMLayer(helpers.defaultCapabilities, null, true); + let lmLayer = new LMLayer(helpers.defaultCapabilities, WorkerBuilder.constructInstance(), true); assert.instanceOf(lmLayer, LMLayer); lmLayer.shutdown(); }); }); - describe('#asBlobURI()', function () { + describe.skip('#asBlobURI()', function () { // #asBlobURI() requires browser APIs, hence why it cannot be tested headless in Node. it('should take a function and convert it into a blob function', function (done) { - let uri = DefaultWorker.asBlobURI(function dummyHandler() { + let uri = WorkerBuilder.asBlobURI(function dummyHandler() { // Post something weird, so we can be reasonably certain the Web Worker is... // well, working. // WARNING: Do NOT refactor this string as a variable. It **MUST** remain a string diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index 993975dbfeac9ec27f9a28e98ba878cd3b37c29b..b3c8ec3e9c164c05c4c1eeeef354f2614518bebc 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -1,5 +1,7 @@ var assert = chai.assert; -var LMLayer = com.keyman.text.prediction.LMLayer; + +import { LMLayer, Worker } from "../../../build/lib/web/index.mjs"; +import * as helpers from "../helpers.mjs"; /* * Shows off the LMLayer API, using the full prediction interface. @@ -17,7 +19,7 @@ describe('LMLayer using dummy model', function () { this.timeout(testconfig.timeouts.standard * 3); // This one makes multiple subsequent calls across // the WebWorker boundary, so we should be generous here. - var lmLayer = new LMLayer(helpers.defaultCapabilities, null, true); + var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance(), true); var stripIDs = function(suggestions) { suggestions.forEach(function(suggestion) { @@ -60,7 +62,7 @@ describe('LMLayer using dummy model', function () { it('will perform (default) wordbreaking and return word at caret', function () { this.timeout(testconfig.timeouts.standard * 3); // This one makes multiple subsequent calls across // the WebWorker boundary, so we should be generous here. - var lmLayer = new LMLayer(helpers.defaultCapabilities); + var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance()); // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but @@ -94,6 +96,6 @@ describe('LMLayer using dummy model', function () { } function iGotDistractedByHazel() { - return __json__['future_suggestions/i_got_distracted_by_hazel']; + return __json__['models/future_suggestions/i_got_distracted_by_hazel']; } }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index 833cda2aeae3d49c36bae0a83d13766675253c79..c068daf8f1a919dbc14f6f97d9a9bedcb597f814 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -1,5 +1,7 @@ var assert = chai.assert; -var LMLayer = com.keyman.text.prediction.LMLayer; + +import { LMLayer, Worker } from "../../../build/lib/web/index.mjs"; +import * as helpers from "../helpers.mjs"; /* * How to run the worlist @@ -16,7 +18,7 @@ describe('LMLayer using the trie model', function () { // Parameter 3 = true: enables 'test mode', disables correction-search timeout. // This helps prevent the correction-search timeout from flaking out periodically during unit tests in // CI, since remote servers / devices are involved. - var lmLayer = new LMLayer(helpers.defaultCapabilities, null, true); + var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance(), true); // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but @@ -58,7 +60,7 @@ describe('LMLayer using the trie model', function () { // // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 it('should use the default searchTermToKey()', function () { - var lmLayer = new LMLayer(helpers.defaultCapabilities); + var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance()); return lmLayer.loadModel( // We need to provide an absolute path since the worker is based within a blob. diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index fdf85dfcf16493d708138cb42a7ae77ff8fe2392..4bcb5bdc1f344311de48f614d889383fa09d6fa2 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -1,6 +1,8 @@ var assert = chai.assert; -var LMLayer = com.keyman.text.prediction.LMLayer; -var DefaultWorker = com.keyman.text.prediction.DefaultWorker; + +import { Worker as WorkerBuilder } from "../../../build/lib/web/index.mjs"; +import { LMLayerWorkerCode } from "/base/common/web/lm-worker/build/lib/worker-main.wrapped.min.js"; +import * as helpers from "../helpers.mjs"; describe('LMLayerWorker', function () { // This one makes multiple subsequent calls across the WebWorker boundary, so we should be generous here. @@ -8,15 +10,16 @@ describe('LMLayerWorker', function () { describe('LMLayerWorkerCode', function() { it('should exist!', function() { - assert.isFunction(LMLayerWorkerCode, - 'Could not find LMLayerWorkerCode! Does embedded_worker.js exist?' - ); + // assert.isFunction(LMLayerWorkerCode, + // 'Could not find LMLayerWorkerCode! Does embedded_worker.js exist?' + // ); + assert.isString(LMLayerWorkerCode); }); }); describe('Usage within a Web Worker', function () { it('should install itself in the worker context', function (done) { - let uri = DefaultWorker.asBlobURI(LMLayerWorkerCode); + let uri = WorkerBuilder.asBlobURI(LMLayerWorkerCode); let worker = new Worker(uri); worker.onmessage = function thisShouldBeCalled(message) { done(); diff --git a/common/predictive-text/unit_tests/in_browser/helpers.js b/common/predictive-text/unit_tests/in_browser/helpers.js deleted file mode 100644 index a2d8ab127ed4c9c11d3c60d7fd07fcdbaa0e41ef..0000000000000000000000000000000000000000 --- a/common/predictive-text/unit_tests/in_browser/helpers.js +++ /dev/null @@ -1,8 +0,0 @@ -var helpers; - -// Establishes the equivalent of a TS namespace. -(function(helpers){ - helpers.defaultCapabilities = { - maxLeftContextCodeUnits: 64 - }; -})(helpers || (helpers = {})); \ No newline at end of file diff --git a/common/predictive-text/unit_tests/in_browser/helpers.mjs b/common/predictive-text/unit_tests/in_browser/helpers.mjs new file mode 100644 index 0000000000000000000000000000000000000000..385379bde64fadf5d3cde19cf820cc73eb205273 --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/helpers.mjs @@ -0,0 +1,3 @@ +export let defaultCapabilities = { + maxLeftContextCodeUnits: 64 +}; \ No newline at end of file diff --git a/common/predictive-text/unit_tests/in_browser/manual.conf.js b/common/predictive-text/unit_tests/in_browser/manual.conf.cjs similarity index 89% rename from common/predictive-text/unit_tests/in_browser/manual.conf.js rename to common/predictive-text/unit_tests/in_browser/manual.conf.cjs index f1e3ed48652bad04529b3854401feda4e047d6e1..738f2b7713d597b9cf047fc53af14c9aba5d3d36 100644 --- a/common/predictive-text/unit_tests/in_browser/manual.conf.js +++ b/common/predictive-text/unit_tests/in_browser/manual.conf.cjs @@ -1,5 +1,5 @@ module.exports = function(config) { - var base = require("./base.conf.js"); + var base = require("./base.conf.cjs"); var specifics = { // test results reporter to use @@ -19,7 +19,7 @@ module.exports = function(config) { // level of logging // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG - // Can't set in base.conf.js b/c of constant's definition style. + // Can't set in base.conf.cjs b/c of constant's definition style. logLevel: config.LOG_INFO }; diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index f1b49635f62081e6b61958dad39cfebed4ed809e..58d13b530d1358c062f7152faf2e9b0c97172200 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -15,7 +15,7 @@ cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ # Defaults -FLAGS="--require ./unit_tests/helpers" +FLAGS="" builder_describe "Runs all tests for the language-modeling / predictive-text layer module" \ "configure" \ @@ -29,8 +29,11 @@ builder_describe "Runs all tests for the language-modeling / predictive-text lay builder_parse "$@" +TEST_OPTS= if builder_has_option --ci && builder_is_debug_build; then builder_die "Options --ci and --debug are incompatible." +elif builder_has_option --ci; then + TEST_OPTS=--ci fi if builder_start_action configure; then @@ -44,26 +47,14 @@ if builder_start_action test:libraries; then # They do not have builder-based scripts, being run directly via npm package script. # So, for now, we add a text header to clarify what is running at each stage, in # addition to fair bit of `pushd` and `popd`. - pushd "$KEYMAN_ROOT/common/models/wordbreakers" echo echo "### Running $(builder_term common/models/wordbreakers) tests" - # NPM doesn't seem to parse the post `--` part if specified via script variable. - # So... a simple if-else will do the job for now. - if builder_has_option --ci; then - npm run test -- -reporter mocha-teamcity-reporter - else - npm run test - fi - popd + "$KEYMAN_ROOT/common/models/wordbreakers/build.sh" test $TEST_OPTS pushd "$KEYMAN_ROOT/common/models/templates" echo echo "### Running $(builder_term common/models/templates) tests" - if builder_has_option --ci; then - npm run test -- -reporter mocha-teamcity-reporter - else - npm run test - fi + "$KEYMAN_ROOT/common/models/templates/build.sh" test $TEST_OPTS popd pushd "$KEYMAN_ROOT/common/models/types" @@ -74,6 +65,12 @@ if builder_start_action test:libraries; then npm run test popd + pushd "$KEYMAN_ROOT/common/web/lm-worker" + echo + echo "### Running ${BUILDER_TERM_START}common/web/lm-worker${BUILDER_TERM_END} tests" + ./build.sh test $TEST_OPTS + popd + builder_finish_action success test:libraries fi @@ -122,13 +119,13 @@ if builder_start_action test:browser; then if builder_has_option --ci; then KARMA_FLAGS="$KARMA_FLAGS --reporters teamcity,BrowserStack" - KARMA_CONFIG="CI.conf.js" + KARMA_CONFIG="CI.conf.cjs" KARMA_INFO_LEVEL="--log-level=debug" else - KARMA_CONFIG="manual.conf.js" + KARMA_CONFIG="manual.conf.cjs" if builder_is_debug_build; then KARMA_FLAGS="$KARMA_FLAGS --no-single-run" - KARMA_CONFIG="manual.conf.js" + KARMA_CONFIG="manual.conf.cjs" KARMA_INFO_LEVEL="--log-level=debug" echo @@ -137,7 +134,7 @@ if builder_start_action test:browser; then fi fi - if [[ KARMA_CONFIG == "manual.conf.js" ]]; then + if [[ KARMA_CONFIG == "manual.conf.cjs" ]]; then get_browser_set_for_OS else BROWSERS= diff --git a/common/predictive-text/virtualizedWorker.ts b/common/predictive-text/virtualizedWorker.ts deleted file mode 100644 index ea2233d1ab768b8fffae4f7a19f3ec6cf64fbb52..0000000000000000000000000000000000000000 --- a/common/predictive-text/virtualizedWorker.ts +++ /dev/null @@ -1,88 +0,0 @@ -/// -/// - -namespace com.keyman.text.prediction { - // Unfortunately, importing like this loses all type information for the modules. - // It's that, or massively overhaul lm-layer to be module-based instead of - // namespace-based. - var fs = require("fs"); - var vm = require("vm"); - - class VirtualizedWorkerContext { - // The LMLayerWorker installs itself to 'self', the expected Worker global, so we provide an alias. - self: VirtualizedWorkerContext; - - constructor() { - this.self = this; - } - - postMessage: (message: any) => void; - - importScripts(...scriptNames: string[]) { - /* Use of vm.createContext and script.runInContext allow us to avoid - * polluting the global scope with imports. When we throw away the - * context object, imported scripts will be automatically GC'd. - */ - for(let script of scriptNames) { - this.__importScriptString(fs.readFileSync(script, "UTF-8")); - } - } - - __importScriptString(scriptStr: string) { - let context = vm.createContext(this); - var script = new vm.Script(scriptStr); - script.runInContext(context); - } - } - - /** - * Note: this does not create an actual Worker, separate process, or thread. Everything will - * be executed in-line on a virtualized context. - * - * In the future, it might be nice to use Node's Worker Threads implementation. - */ - export class VirtualizedWorker implements Worker { - private _workerContext: VirtualizedWorkerContext; - - constructor(scriptStr: string) { - this._workerContext = new VirtualizedWorkerContext(); - // Needs to exist before setting up the worker; must exist by `.install()`. - this._workerContext.postMessage = this.workerPostMessage.bind(this); - - // Initialize the "worker". - this._workerContext.__importScriptString(scriptStr); - } - - // Sends the worker's postMessage messages to the appropriate `onmessage` handler. - private workerPostMessage(message: unknown) { - if(this.onmessage) { - this.onmessage({data: message} as any as MessageEvent); - } - } - - /** - * Accepts a callback function that will receive messages sent from the `VirtualizedWorker`'s `postMessage` function, - * much like the standard `Worker.onmessage`. - */ - onmessage: (this: Worker, ev: MessageEvent) => any; - - postMessage(message: any) { - let msgObj = {data: message}; - let msgJSON = JSON.stringify(msgObj); - - /* - * Execute the command within the virtualized worker's scope. The worker's returned - * `postMessage` calls will still reach outside, as they have a reference to `this` via - * `postMessage` (which we've set to a bound `this.workerPostMessage`). - * - * Among other things, this will allow the worker to use its internal namespaces without issue. - */ - let msgCommand = "onmessage(" + msgJSON + ")"; - this._workerContext.__importScriptString(msgCommand); - } - - terminate(): void { - this._workerContext = null; - } - } -} \ No newline at end of file diff --git a/common/predictive-text/web-defaultWorker.ts b/common/predictive-text/web-defaultWorker.ts deleted file mode 100644 index 6a4f527c138e286edbf245b8d7e2dc7e047ad723..0000000000000000000000000000000000000000 --- a/common/predictive-text/web-defaultWorker.ts +++ /dev/null @@ -1,27 +0,0 @@ -namespace com.keyman.text.prediction { - export class DefaultWorker { - static constructInstance(): Worker { - return new Worker(this.asBlobURI(LMLayerWorkerCode)); - } - - /** - * Converts the INSIDE of a function into a blob URI that can - * be passed as a valid URI for a Worker. - * @param fn Function whose body will be referenced by a URI. - * - * This function makes the following possible: - * - * let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () { - * postMessage('inside Web Worker') - * function onmessage(event) { - * // handle message inside Web Worker. - * } - * })); - */ - static asBlobURI(fn: Function): string { - let code = LMLayer.unwrap(fn); - let blob = new Blob([code], { type: 'text/javascript' }); - return URL.createObjectURL(blob); - } - } -} \ No newline at end of file diff --git a/common/test/resources/index.mjs b/common/test/resources/index.mjs new file mode 100644 index 0000000000000000000000000000000000000000..3152cd4584d6746ebfffff38a4abea6809068e4f --- /dev/null +++ b/common/test/resources/index.mjs @@ -0,0 +1,3 @@ +// Exists to facilitate relative path resolution via Node's require.resolve function. + +export { }; \ No newline at end of file diff --git a/common/test/resources/json/engine_tests/chirality.json b/common/test/resources/json/engine_tests/chirality.json index f46ab5399ddb4f94005f7245f063d929fe907a3b..4f3153c2758fc2fc484e405068397e1dba6a8972 100644 --- a/common/test/resources/json/engine_tests/chirality.json +++ b/common/test/resources/json/engine_tests/chirality.json @@ -4,7 +4,13 @@ "id": "test_chirality", "name": "KeymanWeb Chirality Tester", "filename": "resources/keyboards/test_chirality.js", - "languages": {} + "languages": [ + { + "id": "en", + "name": "English", + "region": "World" + } + ] }, "inputTestSets": [ { diff --git a/common/test/resources/json/engine_tests/deadkeys.json b/common/test/resources/json/engine_tests/deadkeys.json index 1764915605bda0c7fe2b229d4dc992d69265f233..aec422f0cc6e23a6b91760a68ec43b696ce3e849 100644 --- a/common/test/resources/json/engine_tests/deadkeys.json +++ b/common/test/resources/json/engine_tests/deadkeys.json @@ -4,7 +4,13 @@ "id": "test_deadkeys", "name": "Keyman Deadkey Stress-Tester", "filename": "resources/keyboards/test_deadkeys.js", - "languages": {} + "languages": [ + { + "id": "en", + "name": "English", + "region": "World" + } + ] }, "inputTestSets": [ { diff --git a/common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json b/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json similarity index 100% rename from common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json rename to common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json diff --git a/common/predictive-text/unit_tests/in_browser/json/tries/accented.json b/common/test/resources/json/models/tries/accented.json similarity index 100% rename from common/predictive-text/unit_tests/in_browser/json/tries/accented.json rename to common/test/resources/json/models/tries/accented.json diff --git a/common/predictive-text/unit_tests/in_browser/json/tries/english-1000.json b/common/test/resources/json/models/tries/english-1000.json similarity index 100% rename from common/predictive-text/unit_tests/in_browser/json/tries/english-1000.json rename to common/test/resources/json/models/tries/english-1000.json diff --git a/common/test/resources/karma-browserstack-config.js b/common/test/resources/karma-browserstack-config.cjs similarity index 100% rename from common/test/resources/karma-browserstack-config.js rename to common/test/resources/karma-browserstack-config.cjs diff --git a/common/test/resources/keyboards/khmer_angkor.js b/common/test/resources/keyboards/khmer_angkor.js index 7b797e7e6922b7e8ca8d5a6e7fd502f9de58b29a..93cdec4df5e081088bd2bc64aea27923565fd430 100644 --- a/common/test/resources/keyboards/khmer_angkor.js +++ b/common/test/resources/keyboards/khmer_angkor.js @@ -6,8 +6,8 @@ KeymanWeb.KR(new Keyboard_khmer_angkor()); } function Keyboard_khmer_angkor() { - var modCodes = com.keyman.text.Codes.modifierCodes; - var keyCodes = com.keyman.text.Codes.keyCodes; + var modCodes = keyman.osk.modifierCodes; + var keyCodes = keyman.osk.keyCodes; this.KI="Keyboard_khmer_angkor"; this.KN="Khmer Angkor"; @@ -2804,7 +2804,7 @@ function Keyboard_khmer_angkor() k.KO(-1,t,"»"); } if(m) { - + k.KDC(-1,t); r=this.g_normalise(t,e); } diff --git a/common/test/resources/keyboards/test_deadkeys.js b/common/test/resources/keyboards/test_deadkeys.js index c55165b8f60b27cec70da4043142e9c06b25ca22..697fb65b5461e96be6b4ef42d81d63b014849e29 100644 --- a/common/test/resources/keyboards/test_deadkeys.js +++ b/common/test/resources/keyboards/test_deadkeys.js @@ -6,8 +6,8 @@ KeymanWeb.KR(new Keyboard_test_deadkeys()); } function Keyboard_test_deadkeys() { - var modCodes = com.keyman.text.Codes.modifierCodes; - var keyCodes = com.keyman.text.Codes.keyCodes; + var modCodes = keyman.osk.modifierCodes; + var keyCodes = keyman.osk.keyCodes; this.KI="Keyboard_test_deadkeys"; this.KN="Keyman Deadkey Stress-Tester"; @@ -325,7 +325,7 @@ function Keyboard_test_deadkeys() k.KDO(-1,t,17); } if(m) { - + k.KDC(-1,t); r=this.g_dead_reorder(t,e); } diff --git a/common/test/resources/model-helpers.mjs b/common/test/resources/model-helpers.mjs new file mode 100644 index 0000000000000000000000000000000000000000..1de10bdb15e48e5b6826c2cb40864f508d477c3c --- /dev/null +++ b/common/test/resources/model-helpers.mjs @@ -0,0 +1,160 @@ +import path from 'path'; +import { assert } from 'chai'; +import fs from 'fs'; +import vm from 'vm'; + +/** + * @file model-helpers.mjs + * + * Defines a common set of helper functions for use with predictive-text-related + * unit tests. + */ + +// Ensure that we can successfully load the module & apply kmwLength, as it's +// needed for some of the unit tests. + +// // Verify that the KMW string extensions are loaded via side-effect. + +import { extendString } from '@keymanapp/web-utils'; + +import { createRequire } from "module"; +import { fileURLToPath } from 'url'; + +// Ensure that our KMW string-extensions activate. +extendString(); +assert.ok('💩'.kmwLength); + +/** + * Creates a MessageEvent (for inter-worker communication), with the given data payload. + * + * @param {*} data + */ +export function createMessageEventWithData(data) { + return { data }; +} + +/** + * Creates a simple, default capabilities object for standard-case LMLayer init. + */ +export function capabilities() { + return { + maxLeftContextCodePoints: 64 + } +} + +/** + * Mimics a message from the outer LMLayer shell with a simple, default config object. + * Used for Worker tests. + */ +export function configWorker(worker) { + worker.onMessage(createMessageEventWithData({ + message: 'config', + capabilities: capabilities() + })); +} + +/** + * A valid model that suggests exactly what you want it to suggest. + * + * @returns {ModelDescription} + */ +export function dummyModel(futureSuggestions) { + return { + type: 'dummy', + futureSuggestions: futureSuggestions || [] + }; +} +/** + * Capabilities of a keyboard that will ONLY send left-sided capabilities. + * The keyboard does not support deleting to the right. + * + * @returns {Capabilities} + */ +export function defaultCapabilities() { + return { + maxLeftContextCodeUnits: 64 + }; +} + +/** + * Returns the Context of an empty buffer; no text, at both the start and + * end of the buffer. + * + * @returns {Context} + */ +export function emptyContext() { + return { + left: '', + startOfBuffer: true, + endOfBuffer: true + }; +} + +/** + * Returns a Transform that, when applied, makes no changes to the buffer. + * + * @returns {Transform} + */ +export function zeroTransform() { + return { + insert: '', + deleteLeft: 0, + }; +} + +/** + * Returns a random token. NOT guaranteed to be unique. + * + * @returns {Token} + */ +export function randomToken() { + var range = Number.MAX_SAFE_INTEGER - Number.MIN_SAFE_INTEGER; + return Math.random() * range + Number.MIN_SAFE_INTEGER; +} + +export function iGotDistractedByHazel() { + return jsonFixture('models/future_suggestions/i_got_distracted_by_hazel'); +} + +export function jsonFixture(name, root, import_root) { + // Assuming this file structure: + // root + // └── json + // ├── future_suggestions + // │ └── ... + // └── wordlists + // └── ... + + // Default root: this folder. + if(!root) { + root = 'json'; + } + + if(!import_root) { + import_root = import.meta.url; + } + + // The most straight-forward way... is to use CommonJS-style require to load JSON. + // Fortunately, Node provides the tools needed to recreate it. + const require = createRequire(import_root); + + // ES-module mode also leaves out `__dirname`, so we rebuild that too. + const __filename = fileURLToPath(import_root); + const __dirname = path.dirname(__filename); + return require(path.join(__dirname, root, `${name}.json`)); +} + +export function importScriptsWith(context) { + return function() { // the constructed context's importScripts method. + + /* Use of vm.createContext and script.runInContext allow us to avoid + * polluting the global scope with imports. When we throw away the + * context object, imported scripts will be automatically GC'd. + */ + for(var i=0; i < arguments.length; i++) { + context = vm.createContext(context); + var script = new vm.Script(fs.readFileSync(arguments[i])); + script.runInContext(context); + } + } +} diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js b/common/test/resources/models/naive-trie.js similarity index 100% rename from common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js rename to common/test/resources/models/naive-trie.js diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/test/resources/models/simple-dummy.js similarity index 100% rename from common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js rename to common/test/resources/models/simple-dummy.js diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js b/common/test/resources/models/simple-trie.js similarity index 100% rename from common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js rename to common/test/resources/models/simple-trie.js diff --git a/common/test/resources/package.json b/common/test/resources/package.json new file mode 100644 index 0000000000000000000000000000000000000000..03f8de3cb4f63223fb0b043d2650ab238bf6f9aa --- /dev/null +++ b/common/test/resources/package.json @@ -0,0 +1,12 @@ +{ + "name": "@keymanapp/common-test-resources", + "description": "Common test resources for Keyman's JS & TS-based components & products", + "type": "module", + "license": "MIT", + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "@types/node": "^10.17.21", + "chai": "^4.3.4", + "typescript": "^4.9.5" + } +} diff --git a/common/test/resources/timeout-adapter.js b/common/test/resources/timeout-adapter.js index b8141a6a418858330d1eec338c2bed95a5a58957..0e93ec94a9558a46789cb672178ce15ac8029bd3 100644 --- a/common/test/resources/timeout-adapter.js +++ b/common/test/resources/timeout-adapter.js @@ -1,6 +1,6 @@ // Preprocessing of the Karma configuration's client.args parameter. -com = com || {}; +var com = com || {}; com.keyman = com.keyman || {}; com.keyman.karma = com.keyman.karma || {}; diff --git a/common/tools/sourcemap-path-remapper/build.sh b/common/tools/sourcemap-path-remapper/build.sh index 16c5c76a673b9c23167cc51a63984cab1a87b6c1..d917bca89bc6557b73a1430c588cc8b9296d3633 100755 --- a/common/tools/sourcemap-path-remapper/build.sh +++ b/common/tools/sourcemap-path-remapper/build.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash # -# Compile our sourcemap-path remapping module for use by Web builds, releases, etc. +# Compiles the JS sourcemap remapper tool used by some of our TS projects when complications +# arise with sourcemap handling. # + +# Exit on command failure and when using unset variables: set -eu ## START STANDARD BUILD SCRIPT INCLUDE @@ -17,9 +20,6 @@ cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ -# TODO: for predictive-text, we only need :headless, perhaps we should be splitting modules? -# TODO: remove :tools once kmlmc is a dependency for test:module - builder_describe "Builds a sourcemap manipulation ES module for use in Web-related builds" \ "clean" \ "configure" \ @@ -31,13 +31,6 @@ builder_describe_outputs \ builder_parse "$@" -### CONFIGURE ACTIONS - -if builder_start_action configure; then - verify_npm_setup - builder_finish_action success configure -fi - ### CLEAN ACTIONS if builder_start_action clean; then @@ -45,6 +38,13 @@ if builder_start_action clean; then builder_finish_action success clean fi +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + ### BUILD ACTIONS if builder_start_action build; then diff --git a/common/tools/sourcemap-path-remapper/package.json b/common/tools/sourcemap-path-remapper/package.json index 552d60989b39bfd14b02e345a13b68e650eee9d6..2a468977393aa9caf51b62de1753aa9780d90fe8 100644 --- a/common/tools/sourcemap-path-remapper/package.json +++ b/common/tools/sourcemap-path-remapper/package.json @@ -10,7 +10,7 @@ "type": "git", "url": "git+https://github.com/keymanapp/keyman.git" }, - "author": "Joshua A. Horton ", + "author": "Joshua A. Horton ", "license": "MIT", "bugs": { "url": "https://github.com/keymanapp/keyman/issues" diff --git a/common/tools/sourcemap-path-remapper/src/index.ts b/common/tools/sourcemap-path-remapper/src/index.ts index de54a3499b4a428b0e60e85b4ba8681c23e1145f..7d26e3c8a73d1fbe6ed924b1b598ed4ea69ff54b 100644 --- a/common/tools/sourcemap-path-remapper/src/index.ts +++ b/common/tools/sourcemap-path-remapper/src/index.ts @@ -1,5 +1,5 @@ import * as fs from 'fs'; -import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats. +import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats. // Base64, stringified-JSON, end-of-file comment... // Because we're compiling the main project based on its TS build outputs, and our cross-module references @@ -16,33 +16,33 @@ export interface Mapping { */ class RemappingState { constructor(srcmap: Object) { - this.sourcemap = srcmap; + this.sourceMap = srcmap; this.resetChangedSourcepathTracking(); } - public readonly sourcemap: any; - private _unchangedSourcepaths: string[] = []; + public readonly sourceMap: any; + private _unchangedSourcePaths: string[] = []; public get unchangedSourcepaths() { - return this._unchangedSourcepaths; + return this._unchangedSourcePaths; } public resetChangedSourcepathTracking() { - this._unchangedSourcepaths = [...this.sourcemap.sources]; + this._unchangedSourcePaths = [...this.sourceMap.sources]; } /** * The sourcemap's `sourceRoot` property. */ public get sourceRoot() { - return this.sourcemap.sourceRoot; + return this.sourceMap.sourceRoot; } /** * The sourcemap's `sourceRoot` property. */ public set sourceRoot(path: string) { - this.sourcemap.sourceRoot = path; + this.sourceMap.sourceRoot = path; } /** @@ -59,7 +59,7 @@ class RemappingState { const unusedTransforms = [...transforms]; - const sourcePaths = this.sourcemap.sources; + const sourcePaths = this.sourceMap.sources; for(let i = 0; i < sourcePaths.length; i++) { for(const transform of transforms) { if(sourcePaths[i].match(transform.from)) { @@ -101,7 +101,7 @@ class RemappingState { * @param file */ public toFile(file: fs.PathLike) { - fs.writeFileSync(file, convertSourcemap.fromObject(this.sourcemap).toJSON()); + fs.writeFileSync(file, convertSourceMap.fromObject(this.sourceMap).toJSON()); } } @@ -132,6 +132,6 @@ export default class SourcemapRemapper { * @returns */ public static fromBuffer(buffer: Buffer) { - return this.fromObject(convertSourcemap.fromJSON(buffer).toObject()); + return this.fromObject(convertSourceMap.fromJSON(buffer).toObject()); } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/build.sh b/common/web/gesture-recognizer/build.sh index ea84de386344f367a939a564be54164b9bd6d830..43088489f0af107870602f6a9718e4c22a8094ae 100755 --- a/common/web/gesture-recognizer/build.sh +++ b/common/web/gesture-recognizer/build.sh @@ -1,10 +1,4 @@ #!/usr/bin/env bash -# -# Builds the include script for the current Keyman version. -# - -# Exit on command failure and when using unset variables: -set -eu ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary @@ -26,7 +20,7 @@ builder_describe "Builds the gesture-recognition model for Web-based on-screen k "test" \ ":module" \ ":tools tools for testing & developing test resources for this module" \ - "--ci sets the --ci option for child scripts (i.e, the `test` action)" + "--ci sets the --ci option for child scripts (i.e, the $(builder_term test) action)" builder_describe_outputs \ configure:module /node_modules \ @@ -71,5 +65,5 @@ if builder_start_action test:module; then fi if builder_has_action test:tools && ! builder_has_action test:module; then - echo "The ${BUILDER_TERM_START}test:tools${BUILDER_TERM_END} action is currently a no-op." + echo "The $(builder_term test:tools) action is currently a no-op." fi \ No newline at end of file diff --git a/common/web/input-processor/build-bundler.js b/common/web/input-processor/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..f16b207454a72af0866e5f8e88021e0dba4d065d --- /dev/null +++ b/common/web/input-processor/build-bundler.js @@ -0,0 +1,51 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +// Bundled ES module version +esbuild.buildSync({ + entryPoints: ['build/obj/index.js'], + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + external: ['fs', 'vm'], + format: "esm", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version and utils imports to resolve. + // + // We also need to point it at the nested build output folder to resolve in-project + // imports when compiled - esbuild doesn't seem to pick up on the shifted base. + nodePaths: ['..', "build/obj"], + outfile: "build/lib/index.mjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); + +// Bundled CommonJS (classic Node) module version +esbuild.buildSync({ + entryPoints: ['build/obj/index.js'], + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + external: ['fs', 'vm'], + format: "cjs", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version and utils imports to resolve. + // + // We also need to point it at the nested build output folder to resolve in-project + // imports when compiled - esbuild doesn't seem to pick up on the shifted base. + nodePaths: ['..', "build/obj"], + outfile: "build/lib/index.cjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh index 11efb33fbaa300f0329f019a0e54732efc7f4dbc..ef1b35c56f1db8dba622aeaca0b7e96743305577 100755 --- a/common/web/input-processor/build.sh +++ b/common/web/input-processor/build.sh @@ -30,7 +30,7 @@ builder_describe "Builds the standalone, headless form of Keyman Engine for Web' builder_describe_outputs \ configure /node_modules \ - build /common/web/input-processor/build/index.js + build /common/web/input-processor/build/lib/index.mjs \ builder_parse "$@" @@ -51,7 +51,12 @@ fi ### BUILD ACTIONS if builder_start_action build; then - tsc --build src/tsconfig.json + tsc -b ./tsconfig.json + node build-bundler.js + + # Declaration bundling. + tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts + builder_finish_action success build fi @@ -63,9 +68,6 @@ if builder_start_action test; then FLAGS="--reporter mocha-teamcity-reporter" fi - # Build the leaf-style, bundled version of input-processor for use in testing. - tsc -b src/tsconfig.bundled.json - mocha --recursive $FLAGS ./tests/cases/ builder_finish_action success test diff --git a/common/web/input-processor/package.json b/common/web/input-processor/package.json index 6675574334dfd432dd20912a1eda3eb222a27849..a4d86d5be4f2194b56db1ca3116ddae1eb831e62 100644 --- a/common/web/input-processor/package.json +++ b/common/web/input-processor/package.json @@ -38,5 +38,20 @@ "@keymanapp/keyman-version": "*", "@keymanapp/web-utils": "*", "eventemitter3": "^4.0.0" + }, + "type": "module", + "imports": { + "#./*.js": "./build/obj/*.js" + }, + "main": "./build/obj/index.js", + "types": "./build/obj/index.d.ts", + "exports": { + ".": "./build/obj/index.js", + "./lib": { + "types": "./build/lib/index.d.ts", + "import": "./build/lib/index.mjs", + "require": "./build/lib/index.cjs" + }, + "./obj/*.js": "./build/obj/*.js" } } diff --git a/common/web/input-processor/src/includes/events.ts b/common/web/input-processor/src/includes/events.ts deleted file mode 100644 index 2893c0ccee9f181956ecc0dfa6695a37cce0d6c7..0000000000000000000000000000000000000000 --- a/common/web/input-processor/src/includes/events.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Implements Node's EventEmitter class and related module components in a near -// browser-compatible way. (Just requires a blank 'module' object on the window.) -/// - -// Unfortunately, I can't get it to recognize type information properly -// because we can't use require statements. So, a small-scale manual definition. -declare class EventEmitter { - /** Add a listener for a given event */ - on(event: string, func: (...args: any[]) => boolean, context?: any); - /** Add a one-time listener for a given event */ - once(event: string, func: (...args: any[]) => boolean, context?: any); - removeListener(event: string, func: (...args: any[]) => boolean, context?: any, once?: boolean); - - // Defines their alternately-themed aliases. - addListener: typeof EventEmitter.prototype.on; - off: typeof EventEmitter.prototype.removeListener; - - // Defines the actual event-raising function. - emit(eventName: string, ...args: any[]); -} \ No newline at end of file diff --git a/common/web/input-processor/src/index.ts b/common/web/input-processor/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..360a66010834b73c5ccf54c2cdaee26144607063 --- /dev/null +++ b/common/web/input-processor/src/index.ts @@ -0,0 +1,7 @@ +/// + +export { default as InputProcessor } from './text/inputProcessor.js'; +export { default as ContextWindow } from './text/contextWindow.js'; +export { default as ModelSpec } from './text/prediction/modelSpec.js'; +export { default as LanguageProcessor, StateChangeEnum } from './text/prediction/languageProcessor.js'; +export { default as PredictionContext } from './text/prediction/predictionContext.js'; \ No newline at end of file diff --git a/common/web/input-processor/src/text/contextWindow.ts b/common/web/input-processor/src/text/contextWindow.ts index 69b83a58dbb96a1bb63d7e41f60b84aed8459168..2adb4f0b6a6a0b11c5bc05b8edf3a1b71937f740 100644 --- a/common/web/input-processor/src/text/contextWindow.ts +++ b/common/web/input-processor/src/text/contextWindow.ts @@ -1,44 +1,44 @@ -namespace com.keyman.text { - export class ContextWindow implements Context { - // Used to limit the range of context replicated for use of keyboard rules within - // the engine, as used for fat-finger prep / `Alternate` generation. - public static readonly ENGINE_RULE_WINDOW: Configuration = { - leftContextCodePoints: 64, - rightContextCodePoints: 32 - }; - - left: string; - right?: string; - - startOfBuffer: boolean; - endOfBuffer: boolean; - - casingForm?: CasingForm; - - constructor(mock: Mock, config: Configuration, layerId: string) { - this.left = mock.getTextBeforeCaret(); - this.startOfBuffer = this.left._kmwLength() <= config.leftContextCodePoints; - if(!this.startOfBuffer) { - // Our custom substring version will return the last n characters if param #1 is given -n. - this.left = this.left._kmwSubstr(-config.leftContextCodePoints); - } - - this.right = mock.getTextAfterCaret(); - this.endOfBuffer = this.right._kmwLength() <= config.rightContextCodePoints; - if(!this.endOfBuffer) { - this.right = this.right._kmwSubstr(0, config.rightContextCodePoints); - } - - this.casingForm = - layerId == 'shift' ? 'initial' : - layerId == 'caps' ? 'upper' : - null; +import { Mock } from "@keymanapp/keyboard-processor"; + +export default class ContextWindow implements Context { + // Used to limit the range of context replicated for use of keyboard rules within + // the engine, as used for fat-finger prep / `Alternate` generation. + public static readonly ENGINE_RULE_WINDOW: Configuration = { + leftContextCodePoints: 64, + rightContextCodePoints: 32 + }; + + left: string; + right?: string; + + startOfBuffer: boolean; + endOfBuffer: boolean; + + casingForm?: CasingForm; + + constructor(mock: Mock, config: Configuration, layerId: string) { + this.left = mock.getTextBeforeCaret(); + this.startOfBuffer = this.left._kmwLength() <= config.leftContextCodePoints; + if(!this.startOfBuffer) { + // Our custom substring version will return the last n characters if param #1 is given -n. + this.left = this.left._kmwSubstr(-config.leftContextCodePoints); } - public toMock(): Mock { - let caretPos = this.left._kmwLength(); - - return new Mock(this.left + (this.right || ""), caretPos); + this.right = mock.getTextAfterCaret(); + this.endOfBuffer = this.right._kmwLength() <= config.rightContextCodePoints; + if(!this.endOfBuffer) { + this.right = this.right._kmwSubstr(0, config.rightContextCodePoints); } + + this.casingForm = + layerId == 'shift' ? 'initial' : + layerId == 'caps' ? 'upper' : + null; + } + + public toMock(): Mock { + let caretPos = this.left._kmwLength(); + + return new Mock(this.left + (this.right || ""), caretPos); } } \ No newline at end of file diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index c2cf2aeed866753262c1db3865bc0319aa613afe..51516882a4e34b01d8a6fdf258f12a105f288e0e 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -1,373 +1,359 @@ // Defines a 'polyfill' of sorts for NPM's events module -/// -/// -/// -/// - -namespace com.keyman.text { - export class InputProcessor { - public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = { - baseLayout: 'us' - } - /** - * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` - * entry points. - */ - private contextDevice: utils.DeviceSpec; - private kbdProcessor: KeyboardProcessor; - private lngProcessor: prediction.LanguageProcessor; - - constructor(device: utils.DeviceSpec, options?: ProcessorInitOptions) { - if(!device) { - throw new Error('device must be defined'); - } +/// - if(!options) { - options = InputProcessor.DEFAULT_OPTIONS; - } +import ContextWindow from "./contextWindow.js"; +import LanguageProcessor from "./prediction/languageProcessor.js"; +import type ModelSpec from "./prediction/modelSpec.js"; +import { globalObject, DeviceSpec } from "@keymanapp/web-utils"; + +import { + type Alternate, + Codes, + isEmptyTransform, + type Keyboard, + KeyboardInterface, + KeyboardProcessor, + type KeyEvent, + Mock, + type OutputTarget, + type ProcessorInitOptions, + RuleBehavior, + SystemStoreIDs, + type TextTransform +} from "@keymanapp/keyboard-processor"; + +export default class InputProcessor { + public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = { + baseLayout: 'us' + } - this.contextDevice = device; - this.kbdProcessor = new KeyboardProcessor(device, options); - this.lngProcessor = new prediction.LanguageProcessor(); + /** + * Indicates the device (platform) to be used for non-keystroke events, + * such as those sent to `begin postkeystroke` and `begin newcontext` + * entry points. + */ + private contextDevice: DeviceSpec; + private kbdProcessor: KeyboardProcessor; + private lngProcessor: LanguageProcessor; + + constructor(device: DeviceSpec, predictiveTextWorker: Worker, options?: ProcessorInitOptions) { + if(!device) { + throw new Error('device must be defined'); } - public get languageProcessor(): prediction.LanguageProcessor { - return this.lngProcessor; + if(!options) { + options = InputProcessor.DEFAULT_OPTIONS; } - public get keyboardProcessor(): KeyboardProcessor { - return this.kbdProcessor; - } + this.contextDevice = device; + this.kbdProcessor = new KeyboardProcessor(device, options); + this.lngProcessor = new LanguageProcessor(predictiveTextWorker); + } - public get keyboardInterface(): text.KeyboardInterface { - return this.keyboardProcessor.keyboardInterface; - } + public get languageProcessor(): LanguageProcessor { + return this.lngProcessor; + } - public get activeKeyboard(): keyboards.Keyboard { - return this.keyboardInterface.activeKeyboard; - } + public get keyboardProcessor(): KeyboardProcessor { + return this.kbdProcessor; + } + + public get keyboardInterface(): KeyboardInterface { + return this.keyboardProcessor.keyboardInterface; + } - public set activeKeyboard(keyboard: keyboards.Keyboard) { - this.keyboardInterface.activeKeyboard = keyboard; + public get activeKeyboard(): Keyboard { + return this.keyboardInterface.activeKeyboard; + } - // All old deadkeys and keyboard-specific cache should immediately be invalidated - // on a keyboard change. - this.resetContext(); - } + public set activeKeyboard(keyboard: Keyboard) { + this.keyboardInterface.activeKeyboard = keyboard; - public get activeModel(): prediction.ModelSpec { - return this.languageProcessor.activeModel; - } + // All old deadkeys and keyboard-specific cache should immediately be invalidated + // on a keyboard change. + this.resetContext(); + } - /** - * Tell the currently active keyboard that a new context has been selected, - * e.g. by focus change, selection change, keyboard change, etc. - * - * @param {Object} outputTarget The OutputTarget that has focus - * @returns {Object} A RuleBehavior object describing the cumulative effects of - * all matched keyboard rules - */ - processNewContextEvent(outputTarget: OutputTarget): RuleBehavior { - const ruleBehavior = this.keyboardProcessor.processNewContextEvent(this.contextDevice, outputTarget); - - if(ruleBehavior) { - ruleBehavior.finalize(this.keyboardProcessor, outputTarget, true); - } - return ruleBehavior; - } + public get activeModel(): ModelSpec { + return this.languageProcessor.activeModel; + } - /** - * Simulate a keystroke according to the touched keyboard button element - * - * Handles default output and keyboard processing for both OSK and physical keystrokes. - * - * @param {Object} keyEvent The abstracted KeyEvent to use for keystroke processing - * @param {Object} outputTarget The OutputTarget receiving the KeyEvent - * @returns {Object} A RuleBehavior object describing the cumulative effects of - * all matched keyboard rules. - */ - processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { - const kbdMismatch = keyEvent.srcKeyboard && this.activeKeyboard != keyEvent.srcKeyboard; - const trueActiveKeyboard = this.activeKeyboard; - - try { - if(kbdMismatch) { - // Avoid force-reset of context per our setter above. - this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard; - } + /** + * Simulate a keystroke according to the touched keyboard button element + * + * Handles default output and keyboard processing for both OSK and physical keystrokes. + * + * @param {Object} keyEvent The abstracted KeyEvent to use for keystroke processing + * @param {Object} outputTarget The OutputTarget receiving the KeyEvent + * @returns {Object} A RuleBehavior object describing the cumulative effects of + * all matched keyboard rules. + */ + processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { + const kbdMismatch = keyEvent.srcKeyboard && this.activeKeyboard != keyEvent.srcKeyboard; + const trueActiveKeyboard = this.activeKeyboard; + + try { + if(kbdMismatch) { + // Avoid force-reset of context per our setter above. + this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard; + } - return this._processKeyEvent(keyEvent, outputTarget); - } finally { - if(kbdMismatch) { - // Restore our "current" activeKeyboard to its setting before the mismatching KeyEvent. - this.keyboardInterface.activeKeyboard = trueActiveKeyboard; - } + return this._processKeyEvent(keyEvent, outputTarget); + } finally { + if(kbdMismatch) { + // Restore our "current" activeKeyboard to its setting before the mismatching KeyEvent. + this.keyboardInterface.activeKeyboard = trueActiveKeyboard; } } + } - /** - * Acts as the core of `processKeyEvent` once we're comfortable asserting that the incoming - * keystroke matches the current `activeKeyboard`. - * @param keyEvent - * @param outputTarget - * @returns - */ - private _processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { - let formFactor = keyEvent.device.formFactor; - let fromOSK = keyEvent.isSynthetic; - - // The default OSK layout for desktop devices does not include nextlayer info, relying on modifier detection here. - // It's the OSK equivalent to doModifierPress on 'desktop' form factors. - if((formFactor == utils.FormFactor.Desktop || !this.activeKeyboard || this.activeKeyboard.usesDesktopLayoutOnDevice(keyEvent.device)) && fromOSK) { - // If it's a desktop OSK style and this triggers a layer change, - // a modifier key was clicked. No output expected, so it's safe to instantly exit. - if(this.keyboardProcessor.selectLayer(keyEvent)) { - return new RuleBehavior(); - } + /** + * Acts as the core of `processKeyEvent` once we're comfortable asserting that the incoming + * keystroke matches the current `activeKeyboard`. + * @param keyEvent + * @param outputTarget + * @returns + */ + private _processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { + let formFactor = keyEvent.device.formFactor; + let fromOSK = keyEvent.isSynthetic; + + // The default OSK layout for desktop devices does not include nextlayer info, relying on modifier detection here. + // It's the OSK equivalent to doModifierPress on 'desktop' form factors. + if((formFactor == DeviceSpec.FormFactor.Desktop || !this.activeKeyboard || this.activeKeyboard.usesDesktopLayoutOnDevice(keyEvent.device)) && fromOSK) { + // If it's a desktop OSK style and this triggers a layer change, + // a modifier key was clicked. No output expected, so it's safe to instantly exit. + if(this.keyboardProcessor.selectLayer(keyEvent)) { + return new RuleBehavior(); } + } - // Will handle keystroke-based non-layer change modifier & state keys, mapping them through the physical keyboard's version - // of state management. `doModifierPress` must always run. - if(this.keyboardProcessor.doModifierPress(keyEvent, outputTarget, !fromOSK)) { - // If run on a desktop platform, we know that modifier & state key presses may not - // produce output, so we may make an immediate return safely. - if(!fromOSK) { - return new RuleBehavior(); - } + // Will handle keystroke-based non-layer change modifier & state keys, mapping them through the physical keyboard's version + // of state management. `doModifierPress` must always run. + if(this.keyboardProcessor.doModifierPress(keyEvent, outputTarget, !fromOSK)) { + // If run on a desktop platform, we know that modifier & state key presses may not + // produce output, so we may make an immediate return safely. + if(!fromOSK) { + return new RuleBehavior(); } + } - // If suggestions exist AND space is pressed, accept the suggestion and do not process the keystroke. - // If a suggestion was just accepted AND backspace is pressed, revert the change and do not process the backspace. - // We check the first condition here, while the prediction UI handles the second through the try__() methods below. - if(this.languageProcessor.isActive) { - // The following code relies on JS's logical operator "short-circuit" properties to prevent unwanted triggering of the second condition. - - // Can the suggestion UI revert a recent suggestion? If so, do that and swallow the backspace. - if((keyEvent.kName == "K_BKSP" || keyEvent.Lcode == Codes.keyCodes["K_BKSP"]) && this.languageProcessor.tryRevertSuggestion()) { - return new RuleBehavior(); - // Can the suggestion UI accept an existing suggestion? If so, do that and swallow the space character. - } else if((keyEvent.kName == "K_SPACE" || keyEvent.Lcode == Codes.keyCodes["K_SPACE"]) && this.languageProcessor.tryAcceptSuggestion('space')) { - return new RuleBehavior(); - } + // If suggestions exist AND space is pressed, accept the suggestion and do not process the keystroke. + // If a suggestion was just accepted AND backspace is pressed, revert the change and do not process the backspace. + // We check the first condition here, while the prediction UI handles the second through the try__() methods below. + if(this.languageProcessor.isActive) { + // The following code relies on JS's logical operator "short-circuit" properties to prevent unwanted triggering of the second condition. + + // Can the suggestion UI revert a recent suggestion? If so, do that and swallow the backspace. + if((keyEvent.kName == "K_BKSP" || keyEvent.Lcode == Codes.keyCodes["K_BKSP"]) && this.languageProcessor.tryRevertSuggestion()) { + return new RuleBehavior(); + // Can the suggestion UI accept an existing suggestion? If so, do that and swallow the space character. + } else if((keyEvent.kName == "K_SPACE" || keyEvent.Lcode == Codes.keyCodes["K_SPACE"]) && this.languageProcessor.tryAcceptSuggestion('space')) { + return new RuleBehavior(); } + } - // // ...end I3363 (Build 301) + // // ...end I3363 (Build 301) - // Create a "mock" backup of the current outputTarget in its pre-input state. - // Current, long-existing assumption - it's DOM-backed. - let preInputMock = Mock.from(outputTarget, true); + // Create a "mock" backup of the current outputTarget in its pre-input state. + // Current, long-existing assumption - it's DOM-backed. + let preInputMock = Mock.from(outputTarget, true); - const startingLayerId = this.keyboardProcessor.layerId; + const startingLayerId = this.keyboardProcessor.layerId; - // We presently need the true keystroke to run on the FULL context. That index is still - // needed for some indexing operations when comparing two different output targets. - let ruleBehavior = this.keyboardProcessor.processKeystroke(keyEvent, outputTarget); + // We presently need the true keystroke to run on the FULL context. That index is still + // needed for some indexing operations when comparing two different output targets. + let ruleBehavior = this.keyboardProcessor.processKeystroke(keyEvent, outputTarget); - // Swap layer as appropriate. - if(keyEvent.kNextLayer) { - this.keyboardProcessor.selectLayer(keyEvent); - } + // Swap layer as appropriate. + if(keyEvent.kNextLayer) { + this.keyboardProcessor.selectLayer(keyEvent); + } - // If it's a key that we 'optimize out' of our fat-finger correction algorithm, - // we MUST NOT trigger it for this keystroke. - let isOnlyLayerSwitchKey = text.Codes.isKnownOSKModifierKey(keyEvent.kName); - - // Best-guess stopgap for possible custom modifier keys. - // If a key (1) does not affect the context and (2) shifts the active layer, - // we assume it's a modifier key. (Touch keyboards may define custom modifier keys.) - // - // Note: this will mean we won't generate alternates in the niche scenario where: - // 1. Keypress does not alter the actual context - // 2. It DOES emit a deadkey with an earlier processing rule. - // 3. The FINAL processing rule does not match. - // 4. The key ALSO signals a layer shift. - // If any of the four above conditions aren't met - no problem! - // So it's a pretty niche scenario. - if((ruleBehavior?.transcription?.transform as TextTransform)?.isNoOp() && keyEvent.kNextLayer) { - isOnlyLayerSwitchKey = true; - } + // If it's a key that we 'optimize out' of our fat-finger correction algorithm, + // we MUST NOT trigger it for this keystroke. + let isOnlyLayerSwitchKey = Codes.isKnownOSKModifierKey(keyEvent.kName); + + // Best-guess stopgap for possible custom modifier keys. + // If a key (1) does not affect the context and (2) shifts the active layer, + // we assume it's a modifier key. (Touch keyboards may define custom modifier keys.) + // + // Note: this will mean we won't generate alternates in the niche scenario where: + // 1. Keypress does not alter the actual context + // 2. It DOES emit a deadkey with an earlier processing rule. + // 3. The FINAL processing rule does not match. + // 4. The key ALSO signals a layer shift. + // If any of the four above conditions aren't met - no problem! + // So it's a pretty niche scenario. + + if(isEmptyTransform(ruleBehavior?.transcription?.transform) && keyEvent.kNextLayer) { + isOnlyLayerSwitchKey = true; + } - const keepRuleBehavior = ruleBehavior != null; - // Should we swallow any further processing of keystroke events for this keydown-keypress sequence? - if(keepRuleBehavior) { - // alternates are our fat-finger alternate outputs. We don't build these for keys we detect as - // layer switch keys - let alternates = isOnlyLayerSwitchKey ? null : this.buildAlternates(ruleBehavior, keyEvent, preInputMock); + const keepRuleBehavior = ruleBehavior != null; + // Should we swallow any further processing of keystroke events for this keydown-keypress sequence? + if(keepRuleBehavior) { + // alternates are our fat-finger alternate outputs. We don't build these for keys we detect as + // layer switch keys + let alternates = isOnlyLayerSwitchKey ? null : this.buildAlternates(ruleBehavior, keyEvent, preInputMock); - // Now that we've done all the keystroke processing needed, ensure any extra effects triggered - // by the actual keystroke occur. - ruleBehavior.finalize(this.keyboardProcessor, outputTarget, false); + // Now that we've done all the keystroke processing needed, ensure any extra effects triggered + // by the actual keystroke occur. + ruleBehavior.finalize(this.keyboardProcessor, outputTarget, false); - // -- All keystroke (and 'alternate') processing is now complete. Time to finalize everything! -- + // -- All keystroke (and 'alternate') processing is now complete. Time to finalize everything! -- - // Notify the ModelManager of new input - it's predictive text time! - if(alternates && alternates.length > 0) { - ruleBehavior.transcription.alternates = alternates; - } - } else { - // We need a dummy RuleBehavior for keys which have no output (e.g. Shift) - ruleBehavior = new RuleBehavior(); - ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); - ruleBehavior.triggersDefaultCommand = true; + // Notify the ModelManager of new input - it's predictive text time! + if(alternates && alternates.length > 0) { + ruleBehavior.transcription.alternates = alternates; } + } else { + // We need a dummy RuleBehavior for keys which have no output (e.g. Shift) + ruleBehavior = new RuleBehavior(); + ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); + ruleBehavior.triggersDefaultCommand = true; + } - // The keyboard may want to take an action after all other keystroke processing is - // finished, for example to switch layers. This action may not have any output - // but may change system store or variable store values. Given this, we don't need to - // save anything about the post behavior, after finalizing it + // The keyboard may want to take an action after all other keystroke processing is + // finished, for example to switch layers. This action may not have any output + // but may change system store or variable store values. Given this, we don't need to + // save anything about the post behavior, after finalizing it - // We need to tell the keyboard if the layer has been changed, either by a keyboard rule itself, - // or by the touch layout 'nextlayer' control. - const hasLayerChanged = ruleBehavior.setStore[KeyboardInterface.TSS_LAYER] || keyEvent.kNextLayer; - this.keyboardProcessor.newLayerStore.set(hasLayerChanged ? this.keyboardProcessor.layerId : ''); - this.keyboardProcessor.oldLayerStore.set(hasLayerChanged ? startingLayerId : ''); + // We need to tell the keyboard if the layer has been changed, either by a keyboard rule itself, + // or by the touch layout 'nextlayer' control. + const hasLayerChanged = ruleBehavior.setStore[SystemStoreIDs.TSS_LAYER] || keyEvent.kNextLayer; + this.keyboardProcessor.newLayerStore.set(hasLayerChanged ? this.keyboardProcessor.layerId : ''); + this.keyboardProcessor.oldLayerStore.set(hasLayerChanged ? startingLayerId : ''); - let postRuleBehavior = this.keyboardProcessor.processPostKeystroke(this.contextDevice, outputTarget); - if(postRuleBehavior) { - postRuleBehavior.finalize(this.keyboardProcessor, outputTarget, true); - } - - // Yes, even for ruleBehavior.triggersDefaultCommand. Those tend to change the context. - ruleBehavior.predictionPromise = this.languageProcessor.predict(ruleBehavior.transcription, this.keyboardProcessor.layerId); + let postRuleBehavior = this.keyboardProcessor.processPostKeystroke(this.contextDevice, outputTarget); + if(postRuleBehavior) { + postRuleBehavior.finalize(this.keyboardProcessor, outputTarget, true); + } - // Text did not change (thus, no text "input") if we tabbed or merely moved the caret. - if(!ruleBehavior.triggersDefaultCommand) { - // For DOM-aware targets, this will trigger a DOM event page designers may listen for. - outputTarget.doInputEvent(); - } + // Yes, even for ruleBehavior.triggersDefaultCommand. Those tend to change the context. + ruleBehavior.predictionPromise = this.languageProcessor.predict(ruleBehavior.transcription, this.keyboardProcessor.layerId); - return keepRuleBehavior ? ruleBehavior : null; + // Text did not change (thus, no text "input") if we tabbed or merely moved the caret. + if(!ruleBehavior.triggersDefaultCommand) { + // For DOM-aware targets, this will trigger a DOM event page designers may listen for. + outputTarget.doInputEvent(); } - private buildAlternates(ruleBehavior: RuleBehavior, keyEvent: KeyEvent, preInputMock: Mock): Alternate[] { - let alternates: Alternate[]; - - // If we're performing a 'default command', it's not a standard 'typing' event - don't do fat-finger stuff. - // Also, don't do fat-finger stuff if predictive text isn't enabled. - if(this.languageProcessor.isActive && !ruleBehavior.triggersDefaultCommand) { - let keyDistribution = keyEvent.keyDistribution; - - // We don't need to track absolute indexing during alternate-generation; - // only position-relative, so it's better to use a sliding window for context - // when making alternates. (Slightly worse for short text, matters greatly - // for long text.) - let contextWindow = new ContextWindow(preInputMock, ContextWindow.ENGINE_RULE_WINDOW, this.keyboardProcessor.layerId); - let windowedMock = contextWindow.toMock(); - - // Note - we don't yet do fat-fingering with longpress keys. - if(keyDistribution && keyEvent.kbdLayer) { - // Tracks a 'deadline' for fat-finger ops, just in case both context is long enough - // and device is slow enough that the calculation takes too long. - // - // Consider use of https://developer.mozilla.org/en-US/docs/Web/API/Performance/now instead? - // Would allow finer-tuned control. - let TIMEOUT_THRESHOLD: number = Number.MAX_VALUE; - let _globalThis = com.keyman.utils.getGlobalObject(); - let timer: () => number; - - // Available by default on `window` in browsers, but _not_ on `global` in Node, - // surprisingly. Since we can't use code dependent on `require` statements - // at present, we have to condition upon it actually existing. - if(_globalThis['performance'] && _globalThis['performance']['now']) { - timer = function() { - return _globalThis['performance']['now'](); - }; - - TIMEOUT_THRESHOLD = timer() + 16; // + 16ms. - } // else { - // We _could_ just use Date.now() as a backup... but that (probably) only matters - // when unit testing. So... we actually don't _need_ time thresholding when in - // a Node environment. - // } - - // Tracks a minimum probability for keystroke probability. Anything less will not be - // included in alternate calculations. - // - // Seek to match SearchSpace.EDIT_DISTANCE_COST_SCALE from the predictive-text engine. - // Reasoning for the selected value may be seen there. Short version - keystrokes - // that _appear_ very precise may otherwise not even consider directly-neighboring keys. - let KEYSTROKE_EPSILON = Math.exp(-5); - - // Sort the distribution into probability-descending order. - keyDistribution.sort((a, b) => b.p - a.p); - - let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor); - alternates = []; - - let totalMass = 0; // Tracks sum of non-error probabilities. - for(let pair of keyDistribution) { - if(pair.p < KEYSTROKE_EPSILON) { - totalMass += pair.p; - break; - } else if(timer && timer() >= TIMEOUT_THRESHOLD) { - // Note: it's always possible that the thread _executing_ our JS - // got paused by the OS, even if JS itself is single-threaded. - // - // The case where `alternates` is initialized (line 167) but empty - // (because of net-zero loop iterations) MUST be handled. - break; - } - - let mock = Mock.from(windowedMock, false); - - let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId); - if(!altKey) { - console.warn("Potential fat-finger key could not be found in layer!"); - continue; - } - - let altEvent = altKey.constructKeyEvent(this.keyboardProcessor, keyEvent.device); - let alternateBehavior = this.keyboardProcessor.processKeystroke(altEvent, mock); - - // If alternateBehavior.beep == true, ignore it. It's a disallowed key sequence, - // so we expect users to never intend their use. + return keepRuleBehavior ? ruleBehavior : null; + } + + private buildAlternates(ruleBehavior: RuleBehavior, keyEvent: KeyEvent, preInputMock: Mock): Alternate[] { + let alternates: Alternate[]; + + // If we're performing a 'default command', it's not a standard 'typing' event - don't do fat-finger stuff. + // Also, don't do fat-finger stuff if predictive text isn't enabled. + if(this.languageProcessor.isActive && !ruleBehavior.triggersDefaultCommand) { + let keyDistribution = keyEvent.keyDistribution; + + // We don't need to track absolute indexing during alternate-generation; + // only position-relative, so it's better to use a sliding window for context + // when making alternates. (Slightly worse for short text, matters greatly + // for long text.) + let contextWindow = new ContextWindow(preInputMock, ContextWindow.ENGINE_RULE_WINDOW, this.keyboardProcessor.layerId); + let windowedMock = contextWindow.toMock(); + + // Note - we don't yet do fat-fingering with longpress keys. + if(this.languageProcessor.isActive && keyDistribution && keyEvent.kbdLayer) { + // Tracks a 'deadline' for fat-finger ops, just in case both context is long enough + // and device is slow enough that the calculation takes too long. + // + // Consider use of https://developer.mozilla.org/en-US/docs/Web/API/Performance/now instead? + // Would allow finer-tuned control. + let TIMEOUT_THRESHOLD: number = Number.MAX_VALUE; + let _globalThis = globalObject(); + let timer: () => number; + + // Available by default on `window` in browsers, but _not_ on `global` in Node, + // surprisingly. Since we can't use code dependent on `require` statements + // at present, we have to condition upon it actually existing. + if(_globalThis['performance'] && _globalThis['performance']['now']) { + timer = function() { + return _globalThis['performance']['now'](); + }; + + TIMEOUT_THRESHOLD = timer() + 16; // + 16ms. + } // else { + // We _could_ just use Date.now() as a backup... but that (probably) only matters + // when unit testing. So... we actually don't _need_ time thresholding when in + // a Node environment. + // } + + // Tracks a minimum probability for keystroke probability. Anything less will not be + // included in alternate calculations. + // + // Seek to match SearchSpace.EDIT_DISTANCE_COST_SCALE from the predictive-text engine. + // Reasoning for the selected value may be seen there. Short version - keystrokes + // that _appear_ very precise may otherwise not even consider directly-neighboring keys. + let KEYSTROKE_EPSILON = Math.exp(-5); + + // Sort the distribution into probability-descending order. + keyDistribution.sort((a, b) => b.p - a.p); + + let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor); + alternates = []; + + let totalMass = 0; // Tracks sum of non-error probabilities. + for(let pair of keyDistribution) { + if(pair.p < KEYSTROKE_EPSILON) { + totalMass += pair.p; + break; + } else if(timer && timer() >= TIMEOUT_THRESHOLD) { + // Note: it's always possible that the thread _executing_ our JS + // got paused by the OS, even if JS itself is single-threaded. // - // Also possible that this set of conditions fail for all evaluated alternates. - if(alternateBehavior && !alternateBehavior.beep && pair.p > 0) { - let transform: Transform = alternateBehavior.transcription.transform; - - // Ensure that the alternate's token id matches that of the current keystroke, as we only - // record the matched rule's context (since they match) - transform.id = ruleBehavior.transcription.token; - alternates.push({sample: transform, 'p': pair.p}); - totalMass += pair.p; - } + // The case where `alternates` is initialized (line 167) but empty + // (because of net-zero loop iterations) MUST be handled. + break; } - // Renormalizes the distribution, as any error (beep) results - // will result in a distribution that doesn't sum to 1 otherwise. - // All `.p` values are strictly positive, so totalMass is - // guaranteed to be > 0 if the array has entries. - alternates.forEach(function(alt) { - alt.p /= totalMass; - }); - } - } - return alternates; - } + let mock = Mock.from(windowedMock, false); - public resetContext(outputTarget?: OutputTarget) { - this.keyboardProcessor.resetContext(); - this.languageProcessor.invalidateContext(outputTarget, this.keyboardProcessor.layerId); + let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId); + if(!altKey) { + console.warn("Potential fat-finger key could not be found in layer!"); + continue; + } + + let altEvent = this.keyboardProcessor.activeKeyboard.constructKeyEvent(altKey, keyEvent.device, this.keyboardProcessor.stateKeys); + let alternateBehavior = this.keyboardProcessor.processKeystroke(altEvent, mock); - // Let the keyboard do its initial group processing - //console.log('processNewContextEvent called from resetContext'); - if(outputTarget) { - this.processNewContextEvent(outputTarget); + // If alternateBehavior.beep == true, ignore it. It's a disallowed key sequence, + // so we expect users to never intend their use. + // + // Also possible that this set of conditions fail for all evaluated alternates. + if(alternateBehavior && !alternateBehavior.beep && pair.p > 0) { + let transform: Transform = alternateBehavior.transcription.transform; + + // Ensure that the alternate's token id matches that of the current keystroke, as we only + // record the matched rule's context (since they match) + transform.id = ruleBehavior.transcription.token; + alternates.push({sample: transform, 'p': pair.p}); + totalMass += pair.p; + } + } + + // Renormalizes the distribution, as any error (beep) results + // will result in a distribution that doesn't sum to 1 otherwise. + // All `.p` values are strictly positive, so totalMass is + // guaranteed to be > 0 if the array has entries. + alternates.forEach(function(alt) { + alt.p /= totalMass; + }); } } + return alternates; } -} - -(function () { - let ns = com.keyman.text; - // Let the InputProcessor be available both in the browser and in Node. - if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { - module.exports = ns.InputProcessor; - //@ts-ignore - ns.InputProcessor.com = com; // Export the root namespace so that all InputProcessor classes are accessible by unit tests. + public resetContext(outputTarget?: OutputTarget) { + // Also handles new-context events, which may modify the layer + this.keyboardProcessor.resetContext(outputTarget); + // With the layer now set, we trigger new predictions. + this.languageProcessor.invalidateContext(outputTarget, this.keyboardProcessor.layerId); } -}()); \ No newline at end of file +} \ No newline at end of file diff --git a/common/web/input-processor/src/text/prediction/languageProcessor.ts b/common/web/input-processor/src/text/prediction/languageProcessor.ts index 12511cfb9f75f245b387ce1eb63b6a7bfd9a52f3..58c6e4299327f3d7f1217c15b83c000993097261 100644 --- a/common/web/input-processor/src/text/prediction/languageProcessor.ts +++ b/common/web/input-processor/src/text/prediction/languageProcessor.ts @@ -1,288 +1,225 @@ -/// -/// - -namespace com.keyman.text.prediction { - export interface ModelSpec { - /** - * The model's unique identifier. - */ - id: string; - - /** - * The list of supported BCP-47 language codes. Only one language should be supported, - * although multiple variants based on region code (at min) may be specified. - */ - languages: string[]; - - /** - * The path/URL to the file that defines the model. If both `path` and `raw` are specified, - * `path` takes precedence. - */ - path: string; - - /** - * The raw JS script defining the model. Only used if `path` is not specified. - */ - code: string; +import EventEmitter from "eventemitter3"; +import { LMLayer } from "@keymanapp/lexical-model-layer/web"; +import { OutputTarget, Transcription, Mock } from "@keymanapp/keyboard-processor"; +import ContextWindow from "../contextWindow.js"; +import ModelSpec from "./modelSpec.js" + +/** + * Corresponds to the 'suggestionsready' LanguageProcessor event. + */ +export type ReadySuggestionsHandler = (prediction: ReadySuggestions) => boolean; + +export type StateChangeEnum = 'active'|'configured'|'inactive'; +/** + * Corresponds to the 'statechange' LanguageProcessor event. + */ +export type StateChangeHandler = (state: StateChangeEnum) => any; + +/** + * Covers 'tryaccept' events. + */ +export type TryUIHandler = (source: string) => boolean; + +export type InvalidateSourceEnum = 'new'|'context'; + +/** + * Corresponds to the 'invalidatesuggestions' LanguageProcessor event. + */ +export type InvalidateSuggestionsHandler = (source: InvalidateSourceEnum) => boolean; + +export class ReadySuggestions { + suggestions: Suggestion[]; + transcriptionID: number; + + constructor(suggestions: Suggestion[], id: number) { + this.suggestions = suggestions; + this.transcriptionID = id; } +} - /** - * Corresponds to the 'suggestionsready' LanguageProcessor event. - */ - export type ReadySuggestionsHandler = (prediction: ReadySuggestions) => boolean; +interface LanguageProcessorEventMap { + 'suggestionsready': ReadySuggestionsHandler, + 'invalidatesuggestions': InvalidateSuggestionsHandler, + 'statechange': StateChangeHandler, + 'tryaccept': TryUIHandler, + 'tryrevert': () => void, - export type StateChangeEnum = 'active'|'configured'|'inactive'; /** - * Corresponds to the 'statechange' LanguageProcessor event. + * Is called synchronously once suggestion application is successful and the context has been updated. + * + * @param outputTarget The `OutputTarget` representation of the context the suggestion was applied to. + * @returns */ - export type StateChangeHandler = (state: StateChangeEnum) => boolean; + 'suggestionapplied': (outputTarget: OutputTarget) => boolean +} - /** - * Covers both 'tryaccept' and 'tryrevert' events. - */ - export type TryUIHandler = (source: string) => boolean; +/* Is more like the model configuration engine */ +export default class LanguageProcessor extends EventEmitter { + private lmEngine: LMLayer; + private currentModel?: ModelSpec; + private configuration?: Configuration; + private currentPromise?: Promise; - export type InvalidateSourceEnum = 'new'|'context'; + private recentTranscriptions: Transcription[] = []; - /** - * Corresponds to the 'invalidatesuggestions' LanguageProcessor event. - */ - export type InvalidateSuggestionsHandler = (source: InvalidateSourceEnum) => boolean; + private _mayPredict: boolean = true; + private _mayCorrect: boolean = true; - export class ReadySuggestions { - suggestions: Suggestion[]; - transcriptionID: number; + private _state: StateChangeEnum = 'inactive'; - constructor(suggestions: Suggestion[], id: number) { - this.suggestions = suggestions; - this.transcriptionID = id; - } - } + private static readonly TRANSCRIPTION_BUFFER: 10 = 10; - type SupportedEventNames = "suggestionsready" | "invalidatesuggestions" | "statechange" | "tryaccept" | "tryrevert"; - type SupportedEventHandler = InvalidateSuggestionsHandler | ReadySuggestionsHandler | StateChangeHandler | TryUIHandler; + public constructor(predictiveTextWorker: Worker, supportsRightDeletions: boolean = false) { + super(); - export class LanguageProcessor extends EventEmitter { - private lmEngine: LMLayer; - private currentModel?: ModelSpec; - private configuration?: Configuration; - private currentPromise?: Promise; + // Establishes KMW's platform 'capabilities', which limit the range of context a LMLayer + // model may expect. + let capabilities: Capabilities = { + maxLeftContextCodePoints: 64, + // Since the apps don't yet support right-deletions. + maxRightContextCodePoints: supportsRightDeletions ? 0 : 64 + } - private recentTranscriptions: Transcription[] = []; + if(!predictiveTextWorker) { + return; + } - private _mayPredict: boolean = true; - private _mayCorrect: boolean = true; + this.lmEngine = new LMLayer(capabilities, predictiveTextWorker); + } - private static readonly TRANSCRIPTION_BUFFER: 10 = 10; + public get activeModel(): ModelSpec { + return this.currentModel; + } - public init(supportsRightDeletions: boolean = false) { - // Establishes KMW's platform 'capabilities', which limit the range of context a LMLayer - // model may expect. - let capabilities: Capabilities = { - maxLeftContextCodePoints: 64, - // Since the apps don't yet support right-deletions. - maxRightContextCodePoints: supportsRightDeletions ? 0 : 64 - } + public get isConfigured(): boolean { + return !!this.configuration; + } - if(!this.canEnable()) { - return; - } + public get state(): StateChangeEnum { + return this._state; + } - this.lmEngine = new LMLayer(capabilities); - } + public unloadModel() { + this.lmEngine.unloadModel(); + delete this.currentModel; + delete this.configuration; - public get activeModel(): ModelSpec { - return this.currentModel; + this._state = 'inactive'; + this.emit('statechange', 'inactive'); + } + + loadModel(model: ModelSpec): Promise { + if(!model) { + throw new Error("Null reference not allowed."); } - public unloadModel() { - this.lmEngine.unloadModel(); - delete this.currentModel; - delete this.configuration; + let specType: 'file'|'raw' = model.path ? 'file' : 'raw'; + let source = specType == 'file' ? model.path : model.code; - this.emit('statechange', 'inactive'); + // We pre-emptively emit so that the banner's DOM elements may update synchronously. + // Prevents an ugly "flash of unstyled content" layout issue during keyboard load + // on our mobile platforms when embedded. + this.currentModel = model; + if(this.mayPredict) { + this._state = 'active'; + this.emit('statechange', 'active'); } - loadModel(model: ModelSpec): Promise { - if(!model) { - throw new Error("Null reference not allowed."); + return this.lmEngine.loadModel(source, specType).then((config: Configuration) => { + this.configuration = config; + this._state = 'configured'; + this.emit('statechange', 'configured'); + }).catch((error) => { + // Does this provide enough logging information? + let message: string; + if(error instanceof Error) { + message = error.message; + } else { + message = String(error); } + console.error("Could not load model '" + model.id + "': " + message); - let specType: 'file'|'raw' = model.path ? 'file' : 'raw'; - let source = specType == 'file' ? model.path : model.code; - let lp = this; - - // We pre-emptively emit so that the banner's DOM elements may update synchronously. - // Prevents an ugly "flash of unstyled content" layout issue during keyboard load - // on our mobile platforms when embedded. - lp.currentModel = model; - if(this.mayPredict) { - lp.emit('statechange', 'active'); - } + // Since the model couldn't load, immediately deactivate. Visually, it'll look + // like the banner crashed shortly after load. + this.currentModel = null; + this._state = 'inactive'; + this.emit('statechange', 'inactive'); + }); + } - return this.lmEngine.loadModel(source, specType).then(function(config: Configuration) { - lp.configuration = config; - lp.emit('statechange', 'configured'); - }).catch(function(error) { - // Does this provide enough logging information? - let message: string; - if(error instanceof Error) { - message = error.message; - } else { - message = String(error); - } - console.error("Could not load model '" + model.id + "': " + message); + public invalidateContext(outputTarget: OutputTarget, layerId: string): Promise { + // Signal to any predictive text UI that the context has changed, invalidating recent predictions. + this.emit('invalidatesuggestions', 'context'); - // Since the model couldn't load, immediately deactivate. Visually, it'll look - // like the banner crashed shortly after load. - lp.currentModel = null; - lp.emit('statechange', 'inactive'); - }); + // If there's no active model, there can be no predictions. + // We'll also be missing important data needed to even properly REQUEST the predictions. + if(!this.currentModel || !this.configuration) { + return Promise.resolve([]); } - public invalidateContext(outputTarget: OutputTarget, layerId: string) { - // Signal to any predictive text UI that the context has changed, invalidating recent predictions. - this.emit('invalidatesuggestions', 'context'); - - // If there's no active model, there can be no predictions. - // We'll also be missing important data needed to even properly REQUEST the predictions. - if(!this.currentModel || !this.configuration) { - return; - } - - // Don't attempt predictions when disabled! - // invalidateContext otherwise bypasses .predict()'s check against this. - if(!this.isActive) { - return; - } else if(outputTarget) { - let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); - this.predict_internal(transcription, true, layerId); - } + // Don't attempt predictions when disabled! + // invalidateContext otherwise bypasses .predict()'s check against this. + if(!this.isActive) { + return Promise.resolve([]); + } else if(outputTarget) { + let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); + return this.predict_internal(transcription, true, layerId); } + } - public wordbreak(target: OutputTarget, layerId: string): Promise { - if(!this.isActive) { - return null; - } - - let context = new ContextWindow(Mock.from(target, false), this.configuration, layerId); - return this.lmEngine.wordbreak(context); + public wordbreak(target: OutputTarget, layerId: string): Promise { + if(!this.isActive) { + return null; } - public predict(transcription: Transcription, layerId: string): Promise { - if(!this.isActive) { - return null; - } - - // If there's no active model, there can be no predictions. - // We'll also be missing important data needed to even properly REQUEST the predictions. - if(!this.currentModel || !this.configuration) { - return null; - } - - // We've already invalidated any suggestions resulting from any previously-existing Promise - - // may as well officially invalidate them via event. - this.emit("invalidatesuggestions", 'new'); + let context = new ContextWindow(Mock.from(target, false), this.configuration, layerId); + return this.lmEngine.wordbreak(context); + } - return this.predict_internal(transcription, false, layerId); + public predict(transcription: Transcription, layerId: string): Promise { + if(!this.isActive) { + return null; } - /** - * - * @param suggestion - * @param outputTarget - * @param getLayerId a function that returns the current layerId, - * required because layerid can be changed by PostKeystroke - * @returns - */ - public applySuggestion(suggestion: Suggestion, outputTarget: OutputTarget, getLayerId: ()=>string): Promise { - if(!outputTarget) { - throw "Accepting suggestions requires a destination OutputTarget instance." - } - - // Find the state of the context at the time the suggestion was generated. - // This may refer to the context before an input keystroke or before application - // of a predictive suggestion. - let original = this.getPredictionState(suggestion.transformId); - if(!original) { - console.warn("Could not apply the Suggestion!"); - return null; - } else { - // Apply the Suggestion! - - // Step 1: determine the final output text - let final = text.Mock.from(original.preInput, false); - final.apply(suggestion.transform); - - // Step 2: build a final, master Transform that will produce the desired results from the CURRENT state. - // In embedded mode, both Android and iOS are best served by calculating this transform and applying its - // values as needed for use with their IME interfaces. - let transform = final.buildTransformFrom(outputTarget); - outputTarget.apply(transform); - - // Tell the banner that a suggestion was applied, so it can call the - // keyboard's PostKeystroke entry point as needed - this.emit('suggestionapplied', outputTarget); - - // Build a 'reversion' Transcription that can be used to undo this apply() if needed, - // replacing the suggestion transform with the original input text. - let preApply = text.Mock.from(original.preInput, false); - preApply.apply(original.transform); - - // Builds the reversion option according to the loaded lexical model's known - // syntactic properties. - let suggestionContext = new ContextWindow(original.preInput, this.configuration, getLayerId()); - - // We must accept the Suggestion from its original context, which was before - // `original.transform` was applied. - let reversionPromise: Promise = this.lmEngine.acceptSuggestion(suggestion, suggestionContext, original.transform); - - // Also, request new prediction set based on the resulting context. - let lp = this; - reversionPromise = reversionPromise.then(function(reversion) { - let mappedReversion: Reversion = { - // By mapping back to the original Transcription that generated the Suggestion, - // the input will be automatically rewound to the preInput state. - transform: original.transform, - // The ID part is critical; the reversion can't be applied without it. - transformId: -original.token, // reversions use the additive inverse. - displayAs: reversion.displayAs, // The real reason we needed to call the LMLayer. - id: reversion.id, - tag: reversion.tag - } - // // If using the version from lm-layer: - // let mappedReversion = reversion; - // mappedReversion.transformId = reversionTranscription.token; - lp.predictFromTarget(outputTarget, getLayerId()); - return mappedReversion; - }); - - return reversionPromise; - } + // If there's no active model, there can be no predictions. + // We'll also be missing important data needed to even properly REQUEST the predictions. + if(!this.currentModel || !this.configuration) { + return null; } - public applyReversion(reversion: Reversion, outputTarget: OutputTarget) { - if(!outputTarget) { - throw "Accepting suggestions requires a destination OutputTarget instance." - } + // We've already invalidated any suggestions resulting from any previously-existing Promise - + // may as well officially invalidate them via event. + this.emit("invalidatesuggestions", 'new'); - // Find the state of the context at the time the suggestion was generated. - // This may refer to the context before an input keystroke or before application - // of a predictive suggestion. - // - // Reversions use the additive inverse of the id token of the Transcription being - // reverted to. - let original = this.getPredictionState(-reversion.transformId); - if(!original) { - console.warn("Could not apply the Suggestion!"); - return; - } + return this.predict_internal(transcription, false, layerId); + } + + /** + * + * @param suggestion + * @param outputTarget + * @param getLayerId a function that returns the current layerId, + * required because layerid can be changed by PostKeystroke + * @returns + */ + public applySuggestion(suggestion: Suggestion, outputTarget: OutputTarget, getLayerId: ()=>string): Promise { + if(!outputTarget) { + throw "Accepting suggestions requires a destination OutputTarget instance." + } - // Apply the Reversion! + // Find the state of the context at the time the suggestion was generated. + // This may refer to the context before an input keystroke or before application + // of a predictive suggestion. + let original = this.getPredictionState(suggestion.transformId); + if(!original) { + console.warn("Could not apply the Suggestion!"); + return null; + } else { + // Apply the Suggestion! // Step 1: determine the final output text - let final = text.Mock.from(original.preInput, false); - final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion) + let final = Mock.from(original.preInput, false); + final.apply(suggestion.transform); // Step 2: build a final, master Transform that will produce the desired results from the CURRENT state. // In embedded mode, both Android and iOS are best served by calculating this transform and applying its @@ -290,155 +227,237 @@ namespace com.keyman.text.prediction { let transform = final.buildTransformFrom(outputTarget); outputTarget.apply(transform); - // The reason we need to preserve the additive-inverse 'transformId' property on Reversions. - let promise = this.lmEngine.revertSuggestion(reversion, new ContextWindow(original.preInput, this.configuration, null)) - - let lp = this; - return promise.then(function(suggestions: Suggestion[]) { - let result = new ReadySuggestions(suggestions, transform.id); - lp.emit("suggestionsready", result); - lp.currentPromise = null; - - return suggestions; + // Tell the banner that a suggestion was applied, so it can call the + // keyboard's PostKeystroke entry point as needed + this.emit('suggestionapplied', outputTarget); + + // Build a 'reversion' Transcription that can be used to undo this apply() if needed, + // replacing the suggestion transform with the original input text. + let preApply = Mock.from(original.preInput, false); + preApply.apply(original.transform); + + // Builds the reversion option according to the loaded lexical model's known + // syntactic properties. + let suggestionContext = new ContextWindow(original.preInput, this.configuration, getLayerId()); + + // We must accept the Suggestion from its original context, which was before + // `original.transform` was applied. + let reversionPromise: Promise = this.lmEngine.acceptSuggestion(suggestion, suggestionContext, original.transform); + + // Also, request new prediction set based on the resulting context. + reversionPromise = reversionPromise.then((reversion) => { + let mappedReversion: Reversion = { + // By mapping back to the original Transcription that generated the Suggestion, + // the input will be automatically rewound to the preInput state. + transform: original.transform, + // The ID part is critical; the reversion can't be applied without it. + transformId: -original.token, // reversions use the additive inverse. + displayAs: reversion.displayAs, // The real reason we needed to call the LMLayer. + id: reversion.id, + tag: reversion.tag + } + // // If using the version from lm-layer: + // let mappedReversion = reversion; + // mappedReversion.transformId = reversionTranscription.token; + this.predictFromTarget(outputTarget, getLayerId()); + return mappedReversion; }); + + return reversionPromise; } + } - public predictFromTarget(outputTarget: OutputTarget, layerId: string): Promise { - if(!outputTarget) { - return null; - } + public applyReversion(reversion: Reversion, outputTarget: OutputTarget) { + if(!outputTarget) { + throw "Accepting suggestions requires a destination OutputTarget instance." + } - let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); - return this.predict(transcription, layerId); + // Find the state of the context at the time the suggestion was generated. + // This may refer to the context before an input keystroke or before application + // of a predictive suggestion. + // + // Reversions use the additive inverse of the id token of the Transcription being + // reverted to. + let original = this.getPredictionState(-reversion.transformId); + if(!original) { + console.warn("Could not apply the Suggestion!"); + return; } - /** - * Called internally to do actual predictions after any relevant "invalidatesuggestions" events - * have been raised. - * @param transcription The triggering transcription (if it exists) - */ - private predict_internal(transcription: Transcription, resetContext: boolean, layerId: string): Promise { - if(!transcription) { - return null; - } + // Apply the Reversion! - let context = new ContextWindow(transcription.preInput, this.configuration, layerId); - this.recordTranscription(transcription); + // Step 1: determine the final output text + let final = Mock.from(original.preInput, false); + final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion) - if(resetContext) { - this.lmEngine.resetContext(context); - } + // Step 2: build a final, master Transform that will produce the desired results from the CURRENT state. + // In embedded mode, both Android and iOS are best served by calculating this transform and applying its + // values as needed for use with their IME interfaces. + let transform = final.buildTransformFrom(outputTarget); + outputTarget.apply(transform); - let alternates = transcription.alternates; - if(!alternates || alternates.length == 0) { - alternates = [{ - sample: transcription.transform, - p: 1.0 - }]; - } + // The reason we need to preserve the additive-inverse 'transformId' property on Reversions. + let promise = this.lmEngine.revertSuggestion(reversion, new ContextWindow(original.preInput, this.configuration, null)) - let transform = transcription.transform; - var promise = this.currentPromise = this.lmEngine.predict(alternates, context); + return promise.then((suggestions: Suggestion[]) => { + let result = new ReadySuggestions(suggestions, transform.id); + this.emit("suggestionsready", result); + this.currentPromise = null; - let lp = this; - return promise.then(function(suggestions: Suggestion[]) { - if(promise == lp.currentPromise) { - let result = new ReadySuggestions(suggestions, transform.id); - lp.emit("suggestionsready", result); - lp.currentPromise = null; - } + return suggestions; + }); + } - return suggestions; - }); + public predictFromTarget(outputTarget: OutputTarget, layerId: string): Promise { + if(!outputTarget) { + return null; } - private recordTranscription(transcription: Transcription) { - this.recentTranscriptions.push(transcription); + let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false); + return this.predict(transcription, layerId); + } - if(this.recentTranscriptions.length > LanguageProcessor.TRANSCRIPTION_BUFFER) { - this.recentTranscriptions.splice(0, 1); - } + /** + * Called internally to do actual predictions after any relevant "invalidatesuggestions" events + * have been raised. + * @param transcription The triggering transcription (if it exists) + */ + private predict_internal(transcription: Transcription, resetContext: boolean, layerId: string): Promise { + if(!transcription) { + return null; } - /** - * Retrieves the context and output state of KMW immediately before the prediction with - * token `id` was generated. Must correspond to a 'recent' one, as only so many are stored - * in `ModelManager`'s history buffer. - * @param id A unique identifier corresponding to a recent `Transcription`. - * @returns The matching `Transcription`, or `null` none is found. - */ - public getPredictionState(id: number): Transcription { - let match = this.recentTranscriptions.filter(function(t: Transcription) { - return t.token == id; - }) - - return match.length == 0 ? null : match[0]; + let context = new ContextWindow(transcription.preInput, this.configuration, layerId); + this.recordTranscription(transcription); + + if(resetContext) { + this.lmEngine.resetContext(context); } - public shutdown() { - this.lmEngine.shutdown(); + let alternates = transcription.alternates; + if(!alternates || alternates.length == 0) { + alternates = [{ + sample: transcription.transform, + p: 1.0 + }]; } - public get isActive(): boolean { - if(!this.canEnable()) { - this._mayPredict = false; - return false; + let transform = transcription.transform; + var promise = this.currentPromise = this.lmEngine.predict(alternates, context); + + return promise.then((suggestions: Suggestion[]) => { + if(promise == this.currentPromise) { + let result = new ReadySuggestions(suggestions, transform.id); + this.emit("suggestionsready", result); + this.currentPromise = null; } - return (this.activeModel || false) && this._mayPredict; - } - public canEnable(): boolean { - // Is overridden for dom-aware KMW in case of old IE versions. - return true; + return suggestions; + }); + } + + private recordTranscription(transcription: Transcription) { + this.recentTranscriptions.push(transcription); + + if(this.recentTranscriptions.length > LanguageProcessor.TRANSCRIPTION_BUFFER) { + this.recentTranscriptions.splice(0, 1); } + } + + /** + * Retrieves the context and output state of KMW immediately before the prediction with + * token `id` was generated. Must correspond to a 'recent' one, as only so many are stored + * in `ModelManager`'s history buffer. + * @param id A unique identifier corresponding to a recent `Transcription`. + * @returns The matching `Transcription`, or `null` none is found. + */ + public getPredictionState(id: number): Transcription { + let match = this.recentTranscriptions.filter((t: Transcription) => { + return t.token == id; + }) + + return match.length == 0 ? null : match[0]; + } - public get mayPredict() { - return this._mayPredict; + public shutdown() { + this.lmEngine.shutdown(); + } + + public get isActive(): boolean { + if(!this.canEnable()) { + this._mayPredict = false; + return false; } + return (this.activeModel || false) && this._mayPredict; + } - public set mayPredict(flag: boolean) { - if(!this.canEnable()) { - return; - } + canEnable(): boolean { + // Is not initialized if there is no worker. + return !!this.lmEngine; + } - let oldVal = this._mayPredict; - this._mayPredict = flag; + public get mayPredict() { + return this._mayPredict; + } - if(oldVal != flag) { - // If there's no model to be activated and we've reached this point, - // the banner should remain inactive, as it already was. - // If it there was one and we've reached this point, we're globally - // deactivating, so we're fine. - if(this.activeModel) { - this.emit('statechange', flag ? 'active' : 'inactive'); + public set mayPredict(flag: boolean) { + if(!this.canEnable()) { + return; + } + + let oldVal = this._mayPredict; + this._mayPredict = flag; + + if(oldVal != flag) { + // If there's no model to be activated and we've reached this point, + // the banner should remain inactive, as it already was. + // If it there was one and we've reached this point, we're globally + // deactivating, so we're fine. + if(this.activeModel) { + // If someone toggles predictions on and off without changing the model, it is possible + // that the model is already configured! + let state: StateChangeEnum = flag ? 'active' : 'inactive'; + + // We always signal the 'active' state here, even if 'configured', b/c of an + // anti-banner-flicker optimization in the Android app. + this._state = state; + this.emit('statechange', state); + + if(this.isConfigured) { + this._state = 'configured'; + this.emit('statechange', 'configured'); } } } + } - public get mayCorrect() { - return this._mayCorrect; - } + public get mayCorrect() { + return this._mayCorrect; + } - public set mayCorrect(flag: boolean) { - this._mayCorrect = flag; - } + public set mayCorrect(flag: boolean) { + this._mayCorrect = flag; + } - public get wordbreaksAfterSuggestions() { - return this.configuration.wordbreaksAfterSuggestions; - } + public get wordbreaksAfterSuggestions() { + return this.configuration.wordbreaksAfterSuggestions; + } - public tryAcceptSuggestion(source: string): boolean { - let returnObj = {shouldSwallow: false}; - this.emit('tryaccept', source, returnObj); + public tryAcceptSuggestion(source: string): boolean { + // If and when we do auto-correct, the suggestion is to pass this object to the event and + // denote any mutations to the contained value. + //let returnObj = {shouldSwallow: false}; + this.emit('tryaccept', source); - return returnObj.shouldSwallow; - } + return false; + } - public tryRevertSuggestion(): boolean { - let returnObj = {shouldSwallow: false}; - this.emit('tryrevert', returnObj); + public tryRevertSuggestion(): boolean { + // If and when we do auto-revert, the suggestion is to pass this object to the event and + // denote any mutations to the contained value. + //let returnObj = {shouldSwallow: false}; + this.emit('tryrevert'); - return returnObj.shouldSwallow; - } + return false; } } \ No newline at end of file diff --git a/common/web/input-processor/src/text/prediction/modelSpec.ts b/common/web/input-processor/src/text/prediction/modelSpec.ts new file mode 100644 index 0000000000000000000000000000000000000000..986d94631160d2853b9c9540dca07dae2e11b3ec --- /dev/null +++ b/common/web/input-processor/src/text/prediction/modelSpec.ts @@ -0,0 +1,27 @@ +/** + * Defines the metadata needed to support and load lexical models compatible with + * Keyman Engine for Web. + */ +export default interface ModelSpec { + /** + * The model's unique identifier. + */ + id: string; + + /** + * The list of supported BCP-47 language codes. Only one language should be supported, + * although multiple variants based on region code (at min) may be specified. + */ + languages: string[]; + + /** + * The path/URL to the file that defines the model. If both `path` and `code` are specified, + * `path` takes precedence. + */ + path: string; + + /** + * The raw JS script defining the model. Only used if `path` is not specified. + */ + code: string; +} \ No newline at end of file diff --git a/common/web/input-processor/src/text/prediction/predictionContext.ts b/common/web/input-processor/src/text/prediction/predictionContext.ts new file mode 100644 index 0000000000000000000000000000000000000000..7578545b98507986157d2094e51ce6bdbe78514e --- /dev/null +++ b/common/web/input-processor/src/text/prediction/predictionContext.ts @@ -0,0 +1,360 @@ +import EventEmitter from "eventemitter3"; +import type LanguageProcessor from "./languageProcessor.js"; +import { type ReadySuggestions, type InvalidateSourceEnum, StateChangeEnum, StateChangeHandler } from './languageProcessor.js'; +import { type KeyboardProcessor, type OutputTarget } from "@keymanapp/keyboard-processor"; + +interface PredictionContextEventMap { + update: (suggestions: Suggestion[]) => void; +} + +/** + * Maintains predictive-text state information corresponding to the current context. + */ +export default class PredictionContext extends EventEmitter { + // Historical note: before 17.0, this code was intertwined with /web/source/osk/banner.ts's + // SuggestionBanner class. This class serves as the main implementation of the banner's core logic. + + // Designed for use with auto-correct behavior + private selected: Suggestion; + + private initNewContext: boolean = true; + + private _currentSuggestions: Suggestion[] = []; + private keepSuggestion: Keep; + private revertSuggestion: Reversion; + + private recentAccept: boolean = false; + private revertAcceptancePromise: Promise; + + private swallowPrediction: boolean = false; + + private doRevert: boolean = false; + private recentRevert: boolean = false; + + private langProcessor: LanguageProcessor; + private kbdProcessor: KeyboardProcessor; + + /** + * Represents the active context used when requesting and applying predictive-text operations. + */ + private _currentTarget: OutputTarget; + + public get currentTarget(): OutputTarget { + return this._currentTarget; + } + + public setCurrentTarget(target: OutputTarget): Promise { + const originalTarget = this._currentTarget; + this._currentTarget = target; + + if(originalTarget != target) { + // Note: should be triggered after the corresponding new-context event rule has been processed, + // as that may affect the value of layerId here. + return this.resetContext(); + } else { + return Promise.resolve([]); + } + } + + private readonly suggestionApplier: (suggestion: Suggestion) => Promise; + private readonly suggestionReverter: (reversion: Reversion) => void; + + /** + * Handler for post-processing once a suggestion has been applied: calls + * into the active keyboard's `begin postKeystroke` entry point. + * + * Called after the suggestion is applied but _before_ new predictions are + * requested based on the resulting context. + */ + private readonly postApplicationHandler: () => void; + + public constructor(langProcessor: LanguageProcessor, kbdProcessor: KeyboardProcessor) { + super(); + + this.langProcessor = langProcessor; + this.kbdProcessor = kbdProcessor; + + const validSuggestionState: () => boolean = () => + this.currentTarget && langProcessor.state == 'configured'; + + this.suggestionApplier = (suggestion) => { + if(validSuggestionState()) { + return langProcessor.applySuggestion(suggestion, this.currentTarget, () => kbdProcessor.layerId); + } + } + + this.suggestionReverter = (reversion) => { + if(validSuggestionState()) { + langProcessor.applyReversion(reversion, this.currentTarget); + } + } + + // As it's called synchronously via event-callback during `this.suggestionApplier`, + // `this.currentTarget` is guaranteed to remain unchanged. + this.postApplicationHandler = () => { + // Tell the keyboard that the current layer has not changed + kbdProcessor.newLayerStore.set(''); + kbdProcessor.oldLayerStore.set(''); + // Call the keyboard's entry point. + kbdProcessor.processPostKeystroke(kbdProcessor.contextDevice, this.currentTarget) + // If we have a RuleBehavior as a result, run it on the target. This should + // only change system store and variable store values. + ?.finalize(kbdProcessor, this.currentTarget, true); + }; + + this.connect(); + } + + private connect() { + this.langProcessor.addListener('invalidatesuggestions', this.invalidateSuggestions); + this.langProcessor.addListener('suggestionsready', this.updateSuggestions); + this.langProcessor.addListener('tryaccept', this.doTryAccept); + this.langProcessor.addListener('tryrevert', this.doTryRevert); + this.langProcessor.addListener('statechange', this.onModelStateChange); + + this.langProcessor.addListener('suggestionapplied', this.postApplicationHandler); + } + + public disconnect() { + this.langProcessor.removeListener('invalidatesuggestions', this.invalidateSuggestions); + this.langProcessor.removeListener('suggestionsready', this.updateSuggestions); + this.langProcessor.removeListener('tryaccept', this.doTryAccept); + this.langProcessor.removeListener('tryrevert', this.doTryRevert); + this.langProcessor.removeListener('statechange', this.onModelStateChange); + + this.langProcessor.removeListener('suggestionapplied', this.postApplicationHandler); + this.clearSuggestions(); + } + + public get currentSuggestions(): Suggestion[] { + let suggestions = []; + // Insert 'current text' if/when valid as the leading option. + // Since we don't yet do auto-corrections, we only show 'keep' whenever it's + // a valid word (according to the model). + + if(this.activateKeep() && this.keepSuggestion && this.keepSuggestion.matchesModel) { + suggestions.push(this.keepSuggestion); + } else if(this.doRevert) { + suggestions.push(this.revertSuggestion); + } + + return suggestions.concat(this._currentSuggestions); + } + + /** + * Function apply + * Description Applies the predictive `Suggestion` represented by this `BannerSuggestion`. + */ + private acceptInternal(suggestion: Suggestion): Promise { + if(!suggestion) { + return null; + } + + // Should be safe to convert into an event handled externally. + // layerID can be obtained by whoever/whatever holds the InputProcessor instance. + if(suggestion.tag == 'revert') { + this.suggestionReverter(suggestion as Reversion); + return null; + } else { + return this.suggestionApplier(suggestion); + } + } + + /** + * Applies predictive-text suggestions and post-acceptance reversions to the current + * prediction context. + * + * Note that both cases will additionally trigger a new asynchronous `predict` operation, + * though no corresponding Promise is returned by this function. As such, the current + * suggestions should be considered outdated after calling this method, pending replacement + * upon the completed async `predict`. + * + * @param suggestion Either a `Suggestion` or `Reversion`. + * @returns if `suggestion` is a `Suggestion`, will return a `Promise`; else, `null`. + */ + public accept(suggestion: Suggestion): Promise | null { + let _this = this; + + // Selecting a suggestion or a reversion should both clear selection + // and clear the reversion-displaying state of the banner. + this.selected = null; + this.doRevert = false; + + this.revertAcceptancePromise = this.acceptInternal(suggestion); + if(!this.revertAcceptancePromise) { + // We get here either if suggestion acceptance fails or if it was a reversion. + if(suggestion && suggestion.tag == 'revert') { + // Reversion state management + this.recentAccept = false; + this.recentRevert = true; + } + return null; + } + + this.revertAcceptancePromise.then(function(suggestion) { + // Always null-check! + if(suggestion) { + _this.revertSuggestion = suggestion; + } + }); + + this.recentAccept = true; + this.recentRevert = false; + + this.swallowPrediction = true; + + return this.revertAcceptancePromise; + } + + private showRevert() { + // Construct a 'revert suggestion' to facilitate a reversion UI component. + this.doRevert = true; + this.sendUpdateEvent(); + } + + /** + * Receives messages from the keyboard that the 'accept' keystroke has been entered. + * Should return 'false' if the current state allows accepting a suggestion and act accordingly. + * Otherwise, return true. + */ + private doTryAccept = (source: string /*, returnObj: {shouldSwallow: boolean}*/): void => { + //let keyman = com.keyman.singleton; + + if(!this.recentAccept && this.selected) { + this.accept(this.selected); + // returnObj.shouldSwallow = true; + } else if(this.recentAccept && source == 'space') { + this.recentAccept = false; + // // If the model doesn't insert wordbreaks, don't swallow the space. If it does, + // // we consider that insertion to be the results of the first post-accept space. + // returnObj.shouldSwallow = !!keyman.core.languageProcessor.wordbreaksAfterSuggestions; // can be handed outside + } else { + // returnObj.shouldSwallow = false; + } + } + + /** + * Receives messages from the keyboard that the 'revert' keystroke has been entered. + * Should return 'false' if the current state allows reverting a recently-applied suggestion and act accordingly. + * Otherwise, return true. + */ + private doTryRevert = (/*returnObj: {shouldSwallow: boolean}*/): boolean => { + // Has the revert keystroke (BKSP) already been sent once since the last accept? + if(this.doRevert) { + // If so, clear the 'revert' option and start doing normal predictions again. + this.doRevert = false; + this.recentAccept = false; + // Otherwise, did we just accept something before the revert signal was received? + } else if(this.recentAccept) { + this.showRevert(); + this.swallowPrediction = true; + } + + // // We don't yet actually do key-based reversions. + // returnObj.shouldSwallow = false; + return; + } + + /** + * Function invalidateSuggestions + * Scope Public + * Description Clears the suggestions in the suggestion banner + */ + private invalidateSuggestions = (source: InvalidateSourceEnum): void => { + // By default, we assume that the context is the same until we notice otherwise. + this.initNewContext = false; + + if(!this.swallowPrediction || source == 'context') { + this.recentAccept = false; + this.doRevert = false; + this.recentRevert = false; + + if(source == 'context') { + this.swallowPrediction = false; + this.initNewContext = true; + } + } + + // Not checking this can result in a perceptible 'flash' of sorts due to the suggestion-update delay. + if(source != 'new') { + this.clearSuggestions(); + // this.options.forEach((option: BannerSuggestion) => { + // option.update(null); + // }); + } + } + + private clearSuggestions() { + this.updateSuggestions({ + suggestions: [], + transcriptionID: 0 + }); + } + + private activateKeep(): boolean { + return !this.recentAccept && !this.recentRevert && !this.initNewContext; + } + + /** + * Function updateSuggestions + * Scope Public + * @param {Suggestion[]} suggestions Array of suggestions from the lexical model. + * Description Update the displayed suggestions in the SuggestionBanner + */ + private updateSuggestions = (prediction: ReadySuggestions): void => { + let suggestions = prediction.suggestions; + + this._currentSuggestions = suggestions; + + // Do we have a keep suggestion? If so, remove it from the list so that we can control its display position + // and prevent it from being hidden after reversion operations. + this.keepSuggestion = null; + for(let s of suggestions) { + if(s.tag == 'keep') { + this.keepSuggestion = s as Keep; + } + } + + if(this.keepSuggestion) { + this._currentSuggestions.splice(this._currentSuggestions.indexOf(this.keepSuggestion), 1); + } + + // If we've gotten an update request like this, it's almost always user-triggered and means the context has shifted. + if(!this.swallowPrediction) { + this.recentAccept = false; + this.doRevert = false; + this.recentRevert = false; + } else { // This prediction was triggered by a recent 'accept.' Now that it's fulfilled, we clear the flag. + this.swallowPrediction = false; + } + + // The rest is the same, whether from input or from "self-updating" after a reversion to provide new suggestions. + this.sendUpdateEvent(); + } + + public sendUpdateEvent() { + this.emit('update', this.currentSuggestions); + } + + public resetContext(): Promise { + const target = this.currentTarget; + + if(target) { + // Note: should be triggered after the corresponding new-context event rule has been processed, + // as that may affect the value of layerId here. + return this.langProcessor.invalidateContext(target, this.kbdProcessor.layerId); + } else { + return Promise.resolve([]); + } + } + + private onModelStateChange: StateChangeHandler = (state) => { + // Either way, the model has changed; either state marks the completion of such a transition. + // The 'active' state displays the banner while a model loads... but its predictions are + // only possible once fully 'configured'. They may appear to 'blink on' after a small delay + // as a result. + if(state == 'configured' || state == 'inactive') { + this.resetContext(); + } + } +} \ No newline at end of file diff --git a/common/web/input-processor/src/tsconfig.bundled.json b/common/web/input-processor/src/tsconfig.bundled.json deleted file mode 100644 index d16fe709fb25f7e6598db3ff3d58962eaee96dd0..0000000000000000000000000000000000000000 --- a/common/web/input-processor/src/tsconfig.bundled.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - // This variant of the tsconfig.json exists to create a 'leaf', 'bundled' - // version of the keyboard-processor build product. The same reference - // cannot be prepended twice in a composite tsc build, posing problems - // for certain down-line builds if the two tsconfigs are not differentiated. - "extends": "./tsconfig.json", - "compilerOptions": { - "outFile": "../build/index.bundled.js", - }, - "references": [ - // Prepended by lm-worker. - { "path": "../../keyman-version", "prepend": true }, - { "path": "../../utils", "prepend": true }, - { "path": "../../keyboard-processor/src", "prepend": true }, - // For headless tests, we need the headless version of the compile. - { "path": "../../../predictive-text/tsconfig.json", "prepend": true } - ], - "files": [ - "text/inputProcessor.ts", - "../../../../node_modules/eventemitter3/index.js", - ] -} diff --git a/common/web/input-processor/src/tsconfig.json b/common/web/input-processor/src/tsconfig.json deleted file mode 100644 index 230233a2d1d95c9e6a2d10e64ed5bd3769556c8a..0000000000000000000000000000000000000000 --- a/common/web/input-processor/src/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "extends": "../../../../tsconfig-base.json", - "compilerOptions": { - "allowJs": true, - "module": "none", - "outDir": "../build/", - "declaration": true, - "inlineSources": true, - "sourceMap": true, - "sourceRoot": "keyman/", - "target": "es5", - "types": ["node"], - "lib": ["es6"], - "outFile": "../build/index.js" - }, - "references": [ - { "path": "../../keyman-version" }, - { "path": "../../utils" }, - { "path": "../../keyboard-processor/src" }, - { "path": "../../../predictive-text/browser.tsconfig.json" }, - ], - "include": ["./**/*.ts"], - "files": [ - "text/inputProcessor.ts", - "../../../../node_modules/eventemitter3/index.js" - ] -} diff --git a/common/web/input-processor/tests/cases/inputProcessor.js b/common/web/input-processor/tests/cases/inputProcessor.js index d69ccd5af92f4601035ea3561d33bee3f41130f2..a2ba363edcec7e47b4910fef83f0aae5adcf100d 100644 --- a/common/web/input-processor/tests/cases/inputProcessor.js +++ b/common/web/input-processor/tests/cases/inputProcessor.js @@ -1,11 +1,17 @@ -var assert = require('chai').assert; -var fs = require("fs"); -var vm = require("vm"); +import { assert } from 'chai'; +import fs from 'fs'; -let InputProcessor = require('../../build/index.bundled.js'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import InputProcessor from '#./text/inputProcessor.js'; +import { KeyboardInterface, MinimalKeymanGlobal, Mock } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; + +import { Worker } from '@keymanapp/lexical-model-layer/node'; +import * as utils from '@keymanapp/web-utils'; // Required initialization setup. -global.com = InputProcessor.com; // exports all keyboard-processor namespacing. global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. @@ -27,35 +33,43 @@ describe('InputProcessor', function() { }); it('has expected default values after initialization', function () { - let core = new InputProcessor(device); - - assert.isOk(core.keyboardProcessor); - assert.isDefined(core.keyboardProcessor.contextDevice); - assert.isOk(core.languageProcessor); - assert.isOk(core.keyboardInterface); - assert.isUndefined(core.activeKeyboard); // No keyboard should be loaded yet. - assert.isUndefined(core.activeModel); // Same for the model. - - // These checks are lifted from the keyboard-processor init checks found in - // common/web/keyboard-processor/tests/cases/basic-init.js. - assert.equal('us', core.keyboardProcessor.baseLayout, 'KeyboardProcessor has unexpected base layout') - assert.isNotNull(global.KeymanWeb, 'KeymanWeb global was not automatically installed'); - assert.equal('default', core.keyboardProcessor.layerId, 'Default layer is not set to "default"'); - assert.isUndefined(core.keyboardProcessor.activeKeyboard, 'Initialized with already-active keyboard'); - - // Lifted from languageProcessor.js - the core should not be changing these with its init. - assert.isUndefined(core.languageProcessor.activeModel); - assert.isFalse(core.languageProcessor.isActive); - assert.isTrue(core.languageProcessor.mayPredict); + // Can construct without the second parameter; if so, the final assertion - .mayPredict + // will be invalidated. (No worker, no ability to predict.) + let worker = Worker.constructInstance(); + + try { + let core = new InputProcessor(device, worker); + + assert.isOk(core.keyboardProcessor); + assert.isDefined(core.keyboardProcessor.contextDevice); + assert.isOk(core.languageProcessor); + assert.isOk(core.keyboardInterface); + assert.isUndefined(core.activeKeyboard); // No keyboard should be loaded yet. + assert.isUndefined(core.activeModel); // Same for the model. + + // These checks are lifted from the keyboard-processor init checks found in + // common/web/keyboard-processor/tests/cases/basic-init.js. + assert.equal('us', core.keyboardProcessor.baseLayout, 'KeyboardProcessor has unexpected base layout') + assert.isNotNull(global.KeymanWeb, 'KeymanWeb global was not automatically installed'); + assert.equal('default', core.keyboardProcessor.layerId, 'Default layer is not set to "default"'); + assert.isUndefined(core.keyboardProcessor.activeKeyboard, 'Initialized with already-active keyboard'); + + // Lifted from languageProcessor.js - the core should not be changing these with its init. + assert.isUndefined(core.languageProcessor.activeModel); + assert.isFalse(core.languageProcessor.isActive); + assert.isTrue(core.languageProcessor.mayPredict); + } finally { + worker.terminate(); + } }); }); describe('efficiency tests', function() { let testDistribution = []; - var keyboard; + let keyboardWithHarness; // Easy peasy long context: use the input processor's full source! - let coreSourceCode = fs.readFileSync('build/index.bundled.js', 'utf-8'); + let coreSourceCode = fs.readFileSync('build/lib/index.mjs', 'utf-8'); // At the time this test block was written... 810485 chars. // Let's force it to the same order of magnitude, even if the codebase grows. @@ -63,7 +77,7 @@ describe('InputProcessor', function() { coreSourceCode = coreSourceCode.substring(0, 1000000); } - this.beforeAll(function() { + this.beforeAll(async function() { testDistribution = []; for(let c = 'A'.charCodeAt(0); c <= 'Z'.charCodeAt(0); c++) { @@ -75,27 +89,24 @@ describe('InputProcessor', function() { }); } - // Load the keyboard. We'll need an InputProcessor instance as an intermediary. - let core = new InputProcessor(device); - - // These two lines will load a keyboard from its file; headless-mode `registerKeyboard` will - // automatically set the keyboard as active. - let kbdScript = new vm.Script(fs.readFileSync('../../test/resources/keyboards/test_chirality.js')); - kbdScript.runInThisContext(); - - keyboard = core.activeKeyboard; + // Load the keyboard. + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_chirality.js')); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; }); describe('without fat-fingering', function() { it('with minimal context (no fat-fingers)', function() { this.timeout(32); // ms let core = new InputProcessor(device); - let context = new com.keyman.text.Mock("", 0); + let context = new Mock("", 0); - core.activeKeyboard = keyboard; - let layout = keyboard.layout(com.keyman.utils.FormFactor.Phone); + core.keyboardProcessor.keyboardInterface = keyboardWithHarness; + let keyboard = keyboardWithHarness.activeKeyboard; + let layout = keyboard.layout(utils.DeviceSpec.FormFactor.Phone); let key = layout.getLayer('default').getKey('K_A'); - let event = key.constructKeyEvent(core.keyboardProcessor, device); + let event = keyboard.constructKeyEvent(key, device, core.keyboardProcessor.stateKeys); let behavior = core.processKeyEvent(event, context); assert.isNotNull(behavior); @@ -103,7 +114,7 @@ describe('InputProcessor', function() { it('with extremely long context (' + coreSourceCode._kmwLength() + ' chars, no fat-fingers)', function() { // Assumes no SMP chars in the source, which is fine. - let context = new com.keyman.text.Mock(coreSourceCode, coreSourceCode._kmwLength()); + let context = new Mock(coreSourceCode, coreSourceCode._kmwLength()); this.timeout(500); // 500 ms, excluding text import. // These often run on VMs, so we'll be a bit generous. @@ -111,10 +122,11 @@ describe('InputProcessor', function() { let core = new InputProcessor(device); // I mean, it IS long context, and time // thresholding is disabled within Node. - core.activeKeyboard = keyboard; - let layout = keyboard.layout(com.keyman.utils.FormFactor.Phone); + core.keyboardProcessor.keyboardInterface = keyboardWithHarness; + let keyboard = keyboardWithHarness.activeKeyboard; + let layout = keyboard.layout(utils.DeviceSpec.FormFactor.Phone); let key = layout.getLayer('default').getKey('K_A'); - let event = key.constructKeyEvent(core.keyboardProcessor, device); + let event = keyboard.constructKeyEvent(key, device, core.keyboardProcessor.stateKeys); let behavior = core.processKeyEvent(event, context); assert.isNotNull(behavior); @@ -125,13 +137,14 @@ describe('InputProcessor', function() { it('with minimal context (with fat-fingers)', function() { this.timeout(32); // ms let core = new InputProcessor(device); - let context = new com.keyman.text.Mock("", 0); + let context = new Mock("", 0); - core.activeKeyboard = keyboard; - let layout = keyboard.layout(com.keyman.utils.FormFactor.Phone); + core.keyboardProcessor.keyboardInterface = keyboardWithHarness; + let keyboard = keyboardWithHarness.activeKeyboard; + let layout = keyboard.layout(utils.DeviceSpec.FormFactor.Phone); let key = layout.getLayer('default').getKey('K_A'); key.keyDistribution = testDistribution; - let event = key.constructKeyEvent(core.keyboardProcessor, device); + let event = keyboard.constructKeyEvent(key, device, core.keyboardProcessor.stateKeys); let behavior = core.processKeyEvent(event, context); assert.isNotNull(behavior); @@ -139,7 +152,7 @@ describe('InputProcessor', function() { it('with extremely long context (' + coreSourceCode._kmwLength() + ' chars, with fat-fingers)', function() { // Assumes no SMP chars in the source, which is fine. - let context = new com.keyman.text.Mock(coreSourceCode, coreSourceCode._kmwLength()); + let context = new Mock(coreSourceCode, coreSourceCode._kmwLength()); this.timeout(500); // 500 ms, excluding text import. // These often run on VMs, so we'll be a bit generous. @@ -150,11 +163,12 @@ describe('InputProcessor', function() { let core = new InputProcessor(device); // It IS long context, and time // thresholding is disabled within Node. - core.activeKeyboard = keyboard; - let layout = keyboard.layout(com.keyman.utils.FormFactor.Phone); + core.keyboardProcessor.keyboardInterface = keyboardWithHarness; + let keyboard = keyboardWithHarness.activeKeyboard; + let layout = keyboard.layout(utils.DeviceSpec.FormFactor.Phone); let key = layout.getLayer('default').getKey('K_A'); key.keyDistribution = testDistribution; - let event = key.constructKeyEvent(core.keyboardProcessor, device); + let event = keyboard.constructKeyEvent(key, device, core.keyboardProcessor.stateKeys); let behavior = core.processKeyEvent(event, context); assert.isNotNull(behavior); diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index c3e02a97621023e13bc95d4d02f6750f8fdc6412..f1934a5f6eea6063cae4c85f53d80343943bcb78 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -1,48 +1,54 @@ -var assert = require('chai').assert; -var fs = require("fs"); -var vm = require("vm"); +import { assert } from 'chai'; + +import LanguageProcessor from '#./text/prediction/languageProcessor.js'; +import { SourcemappedWorker as LMWorker } from "@keymanapp/lexical-model-layer/node"; +import { Mock } from '@keymanapp/keyboard-processor'; /* * Unit tests for the Dummy prediction model. */ -// TODO: this relies on esbuild output for lexical-model-compiler; later should use import -var LexicalModelCompiler = require('../../../../../developer/src/kmc-model/build/cjs-src/lexical-model-compiler.cjs').default; -var path = require('path'); - -let InputProcessor = require('../../build/index.bundled.js'); +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; +import { fileURLToPath } from 'url'; +import path from 'path'; // Required initialization setup. -global.com = InputProcessor.com; // exports all keyboard-processor namespacing. global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. // Initialize supplementary plane string extensions String.kmwEnableSupplementaryPlane(false); -let LanguageProcessor = com.keyman.text.prediction.LanguageProcessor; - // Test the KeyboardProcessor interface. describe('LanguageProcessor', function() { + let worker; + + beforeEach(function() { + worker = LMWorker.constructInstance(); + }); + + afterEach(function() { + worker.terminate(); + }); + describe('[[constructor]]', function () { it('should initialize without errors', function () { - let lp = new LanguageProcessor(); + let lp = new LanguageProcessor(worker); assert.isNotNull(lp); }); it('has expected default values after initialization', function () { - let languageProcessor = new LanguageProcessor(); + let languageProcessor = new LanguageProcessor(worker); // These checks are lifted from the keyboard-processor init checks found in // common/web/keyboard-processor/tests/cases/basic-init.js. - assert.isUndefined(languageProcessor.lmEngine); + assert.isDefined(languageProcessor.lmEngine); assert.isUndefined(languageProcessor.activeModel); assert.isFalse(languageProcessor.isActive); assert.isTrue(languageProcessor.mayPredict); // Some aspects of initialization must wait until after construction and overall // load of the core. See /web/source/kmwbase.ts, in the final IIFE. - languageProcessor.init(); assert.isOk(languageProcessor.lmEngine); }); }); @@ -50,6 +56,11 @@ describe('LanguageProcessor', function() { describe('.predict', function() { let compiler = new LexicalModelCompiler(); const MODEL_ID = 'example.qaa.trivial'; + + // ES-module mode leaves out `__dirname`, so we rebuild it using other components. + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID); describe('using angle brackets for quotes', function() { @@ -69,8 +80,7 @@ describe('LanguageProcessor', function() { }; it("successfully loads the model", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); languageProcessor.loadModel(modelSpec).then(function() { assert.isOk(languageProcessor.activeModel); // is only set after a successful load. @@ -81,10 +91,9 @@ describe('LanguageProcessor', function() { }); it("generates the expected prediction set", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("li", 2); + let contextSource = new Mock("li", 2); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -118,10 +127,9 @@ describe('LanguageProcessor', function() { describe("does not alter casing when input is lowercased", function() { it("when input is fully lowercased", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("li", 2); + let contextSource = new Mock("li", 2); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -138,10 +146,9 @@ describe('LanguageProcessor', function() { }); it("when input has non-initial uppercased letters", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("lI", 2); + let contextSource = new Mock("lI", 2); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -159,10 +166,9 @@ describe('LanguageProcessor', function() { }); it("unless the suggestion has uppercased letters", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("i", 1); + let contextSource = new Mock("i", 1); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -181,10 +187,9 @@ describe('LanguageProcessor', function() { describe("uppercases suggestions when input is fully capitalized ", function() { it("for suggestions with default casing (== 'lower')", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("LI", 2); + let contextSource = new Mock("LI", 2); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -202,10 +207,9 @@ describe('LanguageProcessor', function() { }); it("for precapitalized suggestions", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("I", 1); + let contextSource = new Mock("I", 1); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -225,10 +229,9 @@ describe('LanguageProcessor', function() { describe("initial-cases suggestions when input uses initial casing ", function() { describe("when input is a single capitalized letter", function() { it("for suggestions with default casing (== 'lower')", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("L", 1); + let contextSource = new Mock("L", 1); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { @@ -248,10 +251,9 @@ describe('LanguageProcessor', function() { describe("input length > 1", function() { it("for suggestions with default casing (== 'lower')", function(done) { - let languageProcessor = new LanguageProcessor(); - languageProcessor.init(); + let languageProcessor = new LanguageProcessor(worker); - let contextSource = new com.keyman.text.Mock("Li", 2); + let contextSource = new Mock("Li", 2); let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); languageProcessor.loadModel(modelSpec).then(function() { diff --git a/common/web/input-processor/tests/cases/predictionContext.js b/common/web/input-processor/tests/cases/predictionContext.js new file mode 100644 index 0000000000000000000000000000000000000000..fd03dda639e20e30b8f7dee40aed18c6aef1e11b --- /dev/null +++ b/common/web/input-processor/tests/cases/predictionContext.js @@ -0,0 +1,305 @@ +import { assert } from 'chai'; +import sinon from 'sinon'; + +import { LanguageProcessor, PredictionContext } from '@keymanapp/input-processor'; +import { Worker as LMWorker } from "@keymanapp/lexical-model-layer/node"; +import { DeviceSpec, KeyboardProcessor, Mock } from '@keymanapp/keyboard-processor'; + +function compileDummyModel(suggestionSets) { + return ` +LMLayerWorker.loadModel(new models.DummyModel({ + futureSuggestions: ${JSON.stringify(suggestionSets, null, 2)}, +})); +`; +} + +// Common spec used for each test's setup. It's actually irrelevant for the tests, +// but KeyboardProcessor needs an instance. +const deviceSpec = new DeviceSpec( + DeviceSpec.Browser.Chrome, + DeviceSpec.FormFactor.Desktop, + DeviceSpec.OperatingSystem.Windows +); + +const appleDummySuggestionSets = [[ + // Set 1: + { + transform: { insert: 'e', deleteLeft: 0}, + displayAs: 'apple', + }, { + transform: { insert: 'y', deleteLeft: 0}, + displayAs: 'apply' + }, { + transform: { insert: 'es', deleteLeft: 0}, + displayAs: 'apples' + } +], [ + // Set 2: + { + transform: { insert: '', deleteLeft: 0}, + displayAs: 'apple', + tag: 'keep' + }, { + transform: { insert: 'y', deleteLeft: 0}, + displayAs: 'apply' + }, { + transform: { insert: 's', deleteLeft: 1}, + displayAs: 'apps' + } +], [ + // Set 3: + { + transform: { insert: 'ied', deleteLeft: 2}, + displayAs: 'applied' + } +], [ + { + transform: { insert: ' reverted', deleteLeft: 5}, + displayAs: 'reverted' + } +]]; + +const appleDummyModel = { + id: 'dummy', + languages: ['en'], + code: compileDummyModel(appleDummySuggestionSets) +}; + +describe("PredictionContext", () => { + let worker; + + beforeEach(function() { + worker = LMWorker.constructInstance(); + }); + + afterEach(function() { + worker.terminate(); + }); + + it('receives predictions as they are generated', async function () { + const langProcessor = new LanguageProcessor(worker); + await langProcessor.loadModel(appleDummyModel); // await: must fully 'configure', load script into worker. + + const kbdProcessor = new KeyboardProcessor(deviceSpec); + const predictiveContext = new PredictionContext(langProcessor, kbdProcessor); + + let updateFake = sinon.fake(); + predictiveContext.on('update', updateFake); + + let mock = new Mock("appl", 4); // "appl|", with '|' as the caret position. + const initialMock = Mock.from(mock); + const promise = predictiveContext.setCurrentTarget(mock); + + // Initial predictive state: no suggestions. context.initializeState() has not yet been called. + assert.equal(updateFake.callCount, 1); + assert.isEmpty(updateFake.firstCall.args[0]); // should have no suggestions. (if convenient for testing) + + await promise; + let suggestions; + + // Initialization results: our first set of dummy suggestions. + assert.equal(updateFake.callCount, 2); + suggestions = updateFake.secondCall.args[0]; + assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apples']); + assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); + assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 0)); + + mock.insertTextBeforeCaret('e'); // appl| + e = apple + let transcription = mock.buildTranscriptionFrom(initialMock, null, true); + await langProcessor.predict(transcription, kbdProcessor.layerId); + + // First predict call results: our second set of dummy suggestions, the first of which includes + // a 'keep' of the original text. + assert.equal(updateFake.callCount, 3); + suggestions = updateFake.thirdCall.args[0]; + assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); + assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); + assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + }); + + it('sendUpdateState retrieves the most recent suggestion set', async function() { + const langProcessor = new LanguageProcessor(worker); + await langProcessor.loadModel(appleDummyModel); // await: must fully 'configure', load script into worker. + + const kbdProcessor = new KeyboardProcessor(deviceSpec); + const predictiveContext = new PredictionContext(langProcessor, kbdProcessor); + + let mock = new Mock("appl", 4); // "appl|", with '|' as the caret position. + const initialSuggestions = await predictiveContext.setCurrentTarget(mock); + + let updateFake = sinon.fake(); + predictiveContext.on('update', updateFake); + predictiveContext.sendUpdateEvent(); // Allows external code to request a re-retrieval of current suggestion state. + + // Now, let's initialize the predictive state - this will load the initial suggestions. + let suggestions; + + // Initialization results: our first set of dummy suggestions. + assert.isTrue(updateFake.calledOnce); + suggestions = updateFake.firstCall.args[0]; + assert.deepEqual(suggestions, initialSuggestions); + + // The array instances may be different, but their contents should be the same instances and in the same order. + assert.sameOrderedMembers(suggestions, initialSuggestions); + }); + + it('suggestion application logic & triggered effects', async function () { + const langProcessor = new LanguageProcessor(worker); + await langProcessor.loadModel(appleDummyModel); // await: must fully 'configure', load script into worker. + + const kbdProcessor = new KeyboardProcessor(deviceSpec); + const predictiveContext = new PredictionContext(langProcessor, kbdProcessor); + + let textState = new Mock("appl", 4); // "appl|", with '|' as the caret position. + + await predictiveContext.setCurrentTarget(textState); + + let updateFake = sinon.fake(); + predictiveContext.on('update', updateFake); + + let suggestions; + + let previousTextState = Mock.from(textState); + textState.insertTextBeforeCaret('e'); // appl| + e = apple + let transcription = textState.buildTranscriptionFrom(previousTextState, null, true); + await langProcessor.predict(transcription, kbdProcessor.layerId); + + // Verify setup. + assert.equal(updateFake.callCount, 1); + suggestions = updateFake.firstCall.args[0]; + assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); + assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); + assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + + // Now for the real test. + previousTextState = Mock.from(textState); // snapshot it! + + const suggestionApply = suggestions.find((obj) => obj.displayAs == 'apply'); + assert.isOk(suggestionApply); + + // For awaiting the suggestions generated upon applying our desired suggestion. + // We aren't given a direct Promise for that, but we can construct one this way. + let postApplySuggestions = new Promise((resolve) => { + predictiveContext.once('update', resolve); + }); + + // Apply the desired suggestion. Also passively generates new, post-acceptance + // suggestions, but this function itself don't provide a Promise for that... + // hence the previous block. + let promiseForApplyReversion = predictiveContext.accept(suggestionApply); + + assert.equal(updateFake.callCount, 1); // No new 'update' has been raised yet. + + // Check 1: did our active text context get changed? We DID just ask to apply + // a suggestion... + assert.notEqual(textState.getText(), previousTextState.getText()); + assert.equal(textState.getText(), 'apply '); + + let reversion = await promiseForApplyReversion; + await postApplySuggestions; + + // Check 2: a second 'update' - post-application predictions! + assert.equal(updateFake.callCount, 2); + suggestions = updateFake.secondCall.args[0]; + assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['applied']); + assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); + assert.equal(suggestions[0].transform.deleteLeft, 2); + + // Check 4+: for the returned reversion object. + assert.isOk(reversion); + + // All other reversion details are tested in the 'reversion application logic...' section defined below. + }); + + it('reversion application logic & triggered effects', async function () { + const langProcessor = new LanguageProcessor(worker); + await langProcessor.loadModel(appleDummyModel); // await: must fully 'configure', load script into worker. + + const kbdProcessor = new KeyboardProcessor(deviceSpec); + const predictiveContext = new PredictionContext(langProcessor, kbdProcessor); + + let textState = new Mock("appl", 4); // "appl|", with '|' as the caret position. + + // Test setup - return to the state at the end of the prior-defined unit test ('suggestion application...') + + await predictiveContext.setCurrentTarget(textState); + + // This is the point in time that a reversion operation will rewind the context to. + const revertBaseTextState = Mock.from(textState); + textState.insertTextBeforeCaret('e'); // appl| + e = apple + let transcription = textState.buildTranscriptionFrom(revertBaseTextState, null, true); + + let suggestionCaptureFake = sinon.fake(); + predictiveContext.once('update', suggestionCaptureFake); + await langProcessor.predict(transcription, kbdProcessor.layerId); + + // We need to capture the suggestion we wish to apply. We could hardcode a forced + // value, but that might become brittle in the long-term. + const originalSuggestionSet = suggestionCaptureFake.firstCall.args[0]; + const suggestionApply = originalSuggestionSet.find((obj) => obj.displayAs == 'apply'); + assert.isOk(suggestionApply); + + let previousTextState = Mock.from(textState); + + // For awaiting the suggestions generated upon applying our desired suggestion. + // We aren't given a direct Promise for that, but we can construct one this way. + let postApplySuggestions = new Promise((resolve) => { + predictiveContext.once('update', resolve); + }); + + let reversion = await predictiveContext.accept(suggestionApply); + await postApplySuggestions; + + // Test setup complete. + + // Check the assertion object (from end of 'suggestion application' checks) - verify setup. + assert.isOk(reversion); + + // Reversion IDs are inverses of the applied suggestion. + // This is the important link used to rewind the context when reverting suggestions. + assert.equal(reversion.id, -suggestionApply.id); + assert.equal(reversion.transformId, -suggestionApply.transformId); + + // Revert display strings include quotes. + // + // We could check this more rigorously by importing from @keymanapp/models-wordbreakers for + // the default quotes. + assert.isTrue(reversion.displayAs.includes(previousTextState.getText())); + assert.equal(reversion.displayAs.length, previousTextState.getText().length + 2); // +2: opening + closing quotes. + + // Fire away! Time to apply the reversion. + previousTextState = Mock.from(textState); + + // Since the test uses a separate thread via Worker, make sure to set up any important event handlers + // before we request the reversion. + let updateFake = sinon.fake(); + predictiveContext.on('update', updateFake); + + // Now, in order to synchronize... we rely on a Promise. The callback is indeed + // called synchronously. + let postRevertSuggestions = new Promise((resolve) => { + predictiveContext.once('update', resolve); + }); + + // And now, apply the reversion itself. + let returnValue = predictiveContext.accept(reversion); + + // 'accepting' a reversion performs a rewind; there's no need for async ops here. + assert.isNull(returnValue); // as per the method's spec. + + // Verify that the rewind + application of reversion worked! + let rewoundTextStateWithInput = Mock.from(revertBaseTextState); // appl + rewoundTextStateWithInput.apply(reversion.transform); // + e + assert.equal(rewoundTextStateWithInput.getText(), 'apple'); // For visual clarity. + + // Note: no space appended. + assert.equal(textState.getText(), rewoundTextStateWithInput.getText()); + + // Re-synchronize once we've received word of the new post-reversion predictions. + await postRevertSuggestions; + + assert.equal(updateFake.callCount, 1); + const suggestionsPostReversion = updateFake.firstCall.args[0]; + assert.deepEqual(suggestionsPostReversion.map((obj) => obj.displayAs), ['reverted']); + }); +}); \ No newline at end of file diff --git a/common/web/input-processor/tsconfig.json b/common/web/input-processor/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..6e04340f4d1b6476e4aece4933478dbf5fcc7792 --- /dev/null +++ b/common/web/input-processor/tsconfig.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman/", + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "baseUrl": "./", + "outDir": "build/obj/", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" + }, + "references": [ + { "path": "../utils" }, + { "path": "../keyboard-processor" }, + { "path": "../../predictive-text/tsconfig.all.json" }, + ], + // The extra .d.ts is necessary to avoid issues with lack of a Worker type definition when not including + // the standard DOM in the TS "lib" configuration. + "include": ["./src/**/*.ts", "../../predictive-text/src/worker-interface.d.ts"] +} diff --git a/common/web/keyboard-processor/build-bundler.js b/common/web/keyboard-processor/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..87625dd19ce9b6d3753677e76b5cd0f4a181cd5e --- /dev/null +++ b/common/web/keyboard-processor/build-bundler.js @@ -0,0 +1,73 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +/** @type {esbuild.BuildOptions} */ +const commonConfig = { + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + format: "esm", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version and utils imports to resolve. + // + // We also need to point it at the nested build output folder to resolve in-project + // imports when compiled - esbuild doesn't seem to pick up on the shifted base. + nodePaths: ['..', "build/obj"] +}; + +// Bundled ES module version +esbuild.buildSync({ + alias: { + 'tslib': '@keymanapp/tslib' + }, + entryPoints: ['build/obj/index.js'], + outfile: "build/lib/index.mjs", + format: "esm", + ...commonConfig +}); + +// Bundled CommonJS (classic Node) module version +esbuild.buildSync({ + alias: { + 'tslib': '@keymanapp/tslib' + }, + entryPoints: ['build/obj/index.js'], + outfile: 'build/lib/index.cjs', + bundle: true, + sourcemap: true, + platform: "node", + format: "cjs", + ...commonConfig +}); + + +esbuild.buildSync({ + alias: { + 'tslib': '@keymanapp/tslib' + }, + entryPoints: ['build/obj/keyboards/loaders/dom-keyboard-loader.js'], + outfile: 'build/lib/dom-keyboard-loader.mjs', + format: "esm", + ...commonConfig +}); + +// The node-based keyboard loader needs an extra parameter due to Node-built-in imports: +esbuild.buildSync({ + alias: { + 'tslib': '@keymanapp/tslib' + }, + entryPoints: ['build/obj/keyboards/loaders/node-keyboard-loader.js'], + outfile: 'build/lib/node-keyboard-loader.mjs', + format: "esm", + platform: "node", + ...commonConfig +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/build.sh b/common/web/keyboard-processor/build.sh index ebcab580ab5db576ff6face96206fcd2f95590ac..01939b9e62414ffebb0f0cbcc4886c1fd2985f9c 100755 --- a/common/web/keyboard-processor/build.sh +++ b/common/web/keyboard-processor/build.sh @@ -21,6 +21,7 @@ builder_describe \ "Compiles the web-oriented utility function module." \ "@/common/web/recorder test" \ "@/common/web/keyman-version" \ + "@/common/web/tslib" \ "@/common/web/utils" \ configure \ clean \ @@ -30,7 +31,7 @@ builder_describe \ builder_describe_outputs \ configure /node_modules \ - build /common/web/keyboard-processor/build/index.js + build /common/web/keyboard-processor/build/lib/index.mjs builder_parse "$@" @@ -41,26 +42,35 @@ fi if builder_start_action clean; then npm run clean + rm -rf ./build builder_finish_action success clean fi if builder_start_action build; then - tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.json" + tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" + node ./build-bundler.js + + # Declaration bundling. + tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts + tsc --emitDeclarationOnly --outFile ./build/lib/dom-keyboard-loader.d.ts -p src/keyboards/loaders/tsconfig.dom.json + tsc --emitDeclarationOnly --outFile ./build/lib/node-keyboard-loader.d.ts -p src/keyboards/loaders/tsconfig.node.json + builder_finish_action success build fi if builder_start_action test; then - tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.bundled.json" - builder_heading "Running Keyboard Processor test suite" - FLAGS= + MOCHA_FLAGS= + KARMA_CONFIG=manual.conf.cjs if builder_has_option --ci; then echo "Replacing user-friendly test reports with CI-friendly versions." - FLAGS="$FLAGS --reporter mocha-teamcity-reporter" + MOCHA_FLAGS="$MOCHA_FLAGS --reporter mocha-teamcity-reporter" + KARMA_CONFIG=CI.conf.cjs fi - mocha --recursive $FLAGS ./tests/cases/ + mocha --recursive $MOCHA_FLAGS ./tests/node/ + karma start ./tests/dom/$KARMA_CONFIG builder_finish_action success test fi \ No newline at end of file diff --git a/common/web/keyboard-processor/package.json b/common/web/keyboard-processor/package.json index c93d6f124d8de7285762bdd86c7d587b8fb3295f..62a035ba05a31b59b7047bdb1d71b31e493b9721 100644 --- a/common/web/keyboard-processor/package.json +++ b/common/web/keyboard-processor/package.json @@ -19,6 +19,12 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", + "karma": "^6.4.1", + "karma-chai": "^0.1.0", + "karma-chrome-launcher": "^2.2.0", + "karma-mocha": "^2.0.1", + "karma-mocha-reporter": "^2.2.5", + "karma-teamcity-reporter": "^1.1.0", "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^10.9.1", @@ -26,7 +32,8 @@ }, "scripts": { "tsc": "tsc", - "clean": "tsc -b --clean src/tsconfig.json && tsc -b --clean src/tsconfig.bundled.json", + "clean": "tsc -b --clean tsconfig.json", + "karma": "karma", "test": "gosh ./test.sh", "mocha": "mocha" }, @@ -35,5 +42,33 @@ "@keymanapp/keyman-version": "*", "@keymanapp/web-utils": "*", "@types/node": "^11.9.4" + }, + "type": "module", + "main": "./build/obj/index.js", + "types": "./build/obj/index.d.ts", + "exports": { + ".": "./build/obj/index.js", + "./node-keyboard-loader": { + "types": "./build/obj/keyboards/loaders/node-keyboard-loader.d.ts", + "import": "./build/obj/keyboards/loaders/node-keyboard-loader.js" + }, + "./dom-keyboard-loader": { + "types": "./build/obj/keyboards/loaders/dom-keyboard-loader.d.ts", + "import": "./build/obj/keyboards/loaders/dom-keyboard-loader.js" + }, + "./lib": { + "types": "./build/lib/index.d.ts", + "import": "./build/lib/index.mjs", + "require": "./build/lib/index.cjs" + }, + "./lib/node-keyboard-loader": { + "types": "./build/lib/node-keyboard-loader.d.ts", + "import": "./build/lib/node-keyboard-loader.mjs" + }, + "./lib/dom-keyboard-loader": { + "types": "./build/lib/dom-keyboard-loader.d.ts", + "import": "./build/lib/dom-keyboard-loader.mjs" + }, + "./obj/*.js": "./build/obj/*.js" } } diff --git a/common/web/keyboard-processor/src/index.ts b/common/web/keyboard-processor/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..ad7aa495235b0a03f7085b590f771dd90df57ebb --- /dev/null +++ b/common/web/keyboard-processor/src/index.ts @@ -0,0 +1,50 @@ +export * from "./keyboards/activeLayout.js"; +export * from "./keyboards/defaultLayouts.js"; +export { default as Keyboard } from "./keyboards/keyboard.js"; +export * from "./keyboards/keyboard.js"; +export { KeyboardHarness, KeyboardKeymanGlobal, MinimalCodesInterface, MinimalKeymanGlobal } from "./keyboards/keyboardHarness.js"; +export { + default as KeyboardLoaderBase, + KeyboardLoadErrorBuilder, + KeyboardMissingError, + KeyboardScriptError +} from "./keyboards/keyboardLoaderBase.js"; +export { + CloudKeyboardFont, + internalizeFont, + InternalKeyboardFont, + KeyboardAPIPropertyMultilangSpec, + KeyboardAPIPropertySpec, + KeyboardInternalPropertySpec, + default as KeyboardProperties, + KeyboardFont, + MetadataObj as RawKeyboardMetadata, + LanguageAPIPropertySpec +} from "./keyboards/keyboardProperties.js"; +export { default as SpacebarText } from "./keyboards/spacebarText.js"; +export { default as StateKeyMap } from "./keyboards/stateKeyMap.js"; + +export { default as Codes } from "./text/codes.js"; +export * from "./text/codes.js"; +export * from "./text/deadkeys.js"; +export { default as DefaultRules } from "./text/defaultRules.js"; +export * from "./text/defaultRules.js"; +export { default as KeyboardInterface } from "./text/kbdInterface.js"; +export * from "./text/kbdInterface.js"; +export { default as KeyboardProcessor } from "./text/keyboardProcessor.js"; +export * from "./text/keyboardProcessor.js"; +export { default as KeyEvent } from "./text/keyEvent.js"; +export * from "./text/keyEvent.js"; +export { default as KeyMapping } from "./text/keyMapping.js"; +export { default as OutputTarget } from "./text/outputTarget.js"; +export * from "./text/outputTarget.js"; +export { default as RuleBehavior } from "./text/ruleBehavior.js"; +export * from "./text/systemStores.js"; + +export * from "@keymanapp/web-utils"; + +// At the top level, there should be no default export. + +// Without the line below... OutputTarget would likely be aliased there, as it's +// the last `export { default as _ }` => `export * from` pairing seen above. +export default undefined; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 398682becd97877bbd8dadd1e62a820457275f3c..e6d2ab9f1578b82cc6ec97eeae3fc02db2e8200c 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -1,782 +1,765 @@ -namespace com.keyman.keyboards { - type KeyDistribution = text.KeyDistribution; - - // TS 3.9 changed behavior of getters to make them - // non-enumerable by default. This broke our 'polyfill' - // functions which depended on enumeration to copy the - // relevant props over. - // https://github.com/microsoft/TypeScript/pull/32264#issuecomment-677718191 - function Enumerable( - target: unknown, - propertyKey: string, - descriptor: PropertyDescriptor - ) { - descriptor.enumerable = true; +import Codes from "../text/codes.js"; +import KeyEvent, { KeyEventSpec } from "../text/keyEvent.js"; +import KeyMapping from "../text/keyMapping.js"; +import type { KeyDistribution } from "../text/keyEvent.js"; +import type { LayoutKey, LayoutRow, LayoutLayer, LayoutFormFactor, ButtonClass } from "./defaultLayouts.js"; +import type Keyboard from "./keyboard.js"; + +import { type DeviceSpec } from "@keymanapp/web-utils"; + +// TS 3.9 changed behavior of getters to make them +// non-enumerable by default. This broke our 'polyfill' +// functions which depended on enumeration to copy the +// relevant props over. +// https://github.com/microsoft/TypeScript/pull/32264#issuecomment-677718191 +function Enumerable( + target: unknown, + propertyKey: string, + descriptor: PropertyDescriptor +) { + descriptor.enumerable = true; +}; + +export class ActiveKey implements LayoutKey { + static readonly DEFAULT_PAD=15; // Padding to left of key, in virtual units + static readonly DEFAULT_RIGHT_MARGIN=15; // Padding to right of right-most key, in virtual units + static readonly DEFAULT_KEY_WIDTH=100; // Width of a key, if not specified, in virtual units + + // Defines key defaults + static readonly DEFAULT_KEY = { + text: '', + width: ActiveKey.DEFAULT_KEY_WIDTH, + sp: 0, + pad: ActiveKey.DEFAULT_PAD }; - export class ActiveKey implements LayoutKey { - static readonly DEFAULT_PAD=15; // Padding to left of key, in virtual units - static readonly DEFAULT_RIGHT_MARGIN=15; // Padding to right of right-most key, in virtual units - static readonly DEFAULT_KEY_WIDTH=100; // Width of a key, if not specified, in virtual units - - // Defines key defaults - static readonly DEFAULT_KEY = { - text: '', - width: ActiveKey.DEFAULT_KEY_WIDTH, - sp: 0, - pad: ActiveKey.DEFAULT_PAD - }; - - /** WARNING - DO NOT USE DIRECTLY outside of @keymanapp/keyboard-processor! */ - id?: string; - - // These are fine. - width?: number; - pad?: number; - - layer: string; - displayLayer: string; - nextlayer: string; - sp?: ButtonClass; - - private baseKeyEvent: text.KeyEvent; - isMnemonic: boolean = false; - - proportionalPad: number; - proportionalX: number; - proportionalWidth: number; - - sk?: ActiveKey[]; - - // Keeping things simple here, as this was added LATE in 14.0 beta. - // Could definitely extend in the future to instead return an object - // that denotes the 'nature' of the key. - // - isUnicode - // - isHardwareKey - // - etc. - - // Reference for the terminology in the comments below: - // https://help.keyman.com/developer/current-version/guides/develop/creating-a-touch-keyboard-layout-for-amharic-the-nitty-gritty - - /** - * Matches the key code as set within Keyman Developer for the layout. - * For example, K_R or U_0020. Denotes either physical keys or virtual keys with custom output, - * with no additional metadata like layer or active modifiers. - * - * Is used to determine the keycode for input events, rule-matching, and keystroke processing. - */ - @Enumerable - public get baseKeyID(): string { - if(typeof this.id === 'undefined') { - return undefined; - } - - return this.id; - } - - @Enumerable - public get isPadding(): boolean { - // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced - // keystrokes. - return this['sp'] == 10; // Button class: hidden. - } - - /** - * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. - * - * Allows diambiguation of scenarios where the same key ID is used twice within a layer, but - * with different innate modifiers. (Refer to https://github.com/keymanapp/keyman/issues/4617) - * The 'desktop layer' may be omitted if it matches the key's display layer. - * - * Examples, given a 'default' display layer, matching keys to Keyman keyboard language: - * - * ``` - * "K_Q" - * + [K_Q] - * "K_Q+shift" - * + [K_Q SHIFT] - * ``` - * - * Useful when the active layer of an input-event is already known. - */ - @Enumerable - public get coreID(): string { - if(typeof this.id === 'undefined') { - return undefined; - } - - let baseID = this.id || ''; + /** WARNING - DO NOT USE DIRECTLY outside of @keymanapp/keyboard-processor! */ + id?: string; + + // These are fine. + width?: number; + pad?: number; + + layer: string; + displayLayer: string; + nextlayer: string; + sp?: ButtonClass; + + _baseKeyEvent: KeyEvent; + isMnemonic: boolean = false; + + proportionalPad: number; + proportionalX: number; + proportionalWidth: number; + + sk?: ActiveKey[]; + + // Keeping things simple here, as this was added LATE in 14.0 beta. + // Could definitely extend in the future to instead return an object + // that denotes the 'nature' of the key. + // - isUnicode + // - isHardwareKey + // - etc. + + // Reference for the terminology in the comments below: + // https://help.keyman.com/developer/current-version/guides/develop/creating-a-touch-keyboard-layout-for-amharic-the-nitty-gritty + + /** + * Matches the key code as set within Keyman Developer for the layout. + * For example, K_R or U_0020. Denotes either physical keys or virtual keys with custom output, + * with no additional metadata like layer or active modifiers. + * + * Is used to determine the keycode for input events, rule-matching, and keystroke processing. + */ + @Enumerable + public get baseKeyID(): string { + if(typeof this.id === 'undefined') { + return undefined; + } - if(this.displayLayer != this.layer) { - baseID = baseID + '+' + this.layer; - } + return this.id; + } - return baseID; - } - - /** - * A keyboard-unique identifier to be used for any display elements representing this key - * in user interfaces and/or on-screen keyboards. - * - * Distinguishes between otherwise-identical keys on different layers of an OSK. - * Includes identifying information about the key's display layer. - * - * Examples, given a 'default' display layer, matching keys to Keyman keyboard language: - * - * ``` - * "default-K_Q" - * + [K_Q] - * "default-K_Q+shift" - * + [K_Q SHIFT] - * ``` - * - * Useful when only the active keyboard is known about an input event. - */ - @Enumerable - public get elementID(): string { - if(typeof this.id === 'undefined') { - return undefined; - } + @Enumerable + public get isPadding(): boolean { + // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced + // keystrokes. + return this['sp'] == 10; // Button class: hidden. + } - return this.displayLayer + '-' + this.coreID; + /** + * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. + * + * Allows diambiguation of scenarios where the same key ID is used twice within a layer, but + * with different innate modifiers. (Refer to https://github.com/keymanapp/keyman/issues/4617) + * The 'desktop layer' may be omitted if it matches the key's display layer. + * + * Examples, given a 'default' display layer, matching keys to Keyman keyboard language: + * + * ``` + * "K_Q" + * + [K_Q] + * "K_Q+shift" + * + [K_Q SHIFT] + * ``` + * + * Useful when the active layer of an input-event is already known. + */ + @Enumerable + public get coreID(): string { + if(typeof this.id === 'undefined') { + return undefined; } - /** - * Converts key IDs of the U_* form to their corresponding UTF-16 text. - * If an ID not matching the pattern is received, returns null. - * @param id - * @returns - */ - static unicodeIDToText(id: string, errorCallback?: (codeAsString: string) => void) { - if(!id || id.substring(0,2) != 'U_') { - return null; - } + let baseID = this.id || ''; - let result = ''; - const codePoints = id.substring(2).split('_'); - for(let codePoint of codePoints) { - const codePointValue = parseInt(codePoint, 16); - if (((0x0 <= codePointValue) && (codePointValue <= 0x1F)) || - ((0x80 <= codePointValue) && (codePointValue <= 0x9F)) || - isNaN(codePointValue)) { - if(errorCallback) { - errorCallback(codePoint); - } - continue; - } else { - // String.fromCharCode() is inadequate to handle the entire range of Unicode - // Someday after upgrading to ES2015, can use String.fromCodePoint() - result += String.kmwFromCharCode(codePointValue); - } - } - return result ? result : null; + if(this.displayLayer != this.layer) { + baseID = baseID + '+' + this.layer; } - static sanitize(rawKey: LayoutKey) { - if(typeof rawKey.width == 'string') { - rawKey.width = parseInt(rawKey.width, 10); - } - // Handles NaN cases as well as 'set to 0' cases; both are intentional here. - rawKey.width ||= ActiveKey.DEFAULT_KEY_WIDTH; + return baseID; + } - if(typeof rawKey.pad == 'string') { - rawKey.pad = parseInt(rawKey.pad, 10); - } - rawKey.pad ||= ActiveKey.DEFAULT_PAD; + /** + * A keyboard-unique identifier to be used for any display elements representing this key + * in user interfaces and/or on-screen keyboards. + * + * Distinguishes between otherwise-identical keys on different layers of an OSK. + * Includes identifying information about the key's display layer. + * + * Examples, given a 'default' display layer, matching keys to Keyman keyboard language: + * + * ``` + * "default-K_Q" + * + [K_Q] + * "default-K_Q+shift" + * + [K_Q SHIFT] + * ``` + * + * Useful when only the active keyboard is known about an input event. + */ + @Enumerable + public get elementID(): string { + if(typeof this.id === 'undefined') { + return undefined; + } - if(typeof rawKey.sp == 'string') { - rawKey.sp = Number.parseInt(rawKey.sp, 10) as ButtonClass; - } - rawKey.sp ||= 0; // The default button class. - } - - static polyfill(key: LayoutKey, layout: ActiveLayout, displayLayer: string) { - // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. - let dummy = new ActiveKey(); - let proto = Object.getPrototypeOf(dummy); - - for(let prop in dummy) { - if(!key.hasOwnProperty(prop)) { - let descriptor = Object.getOwnPropertyDescriptor(proto, prop); - if(descriptor) { - // It's a computed property! Copy the descriptor onto the key's object. - Object.defineProperty(key, prop, descriptor); - } else { - key[prop] = dummy[prop]; - } - } - } + return this.displayLayer + '-' + this.coreID; + } - // Ensure subkeys are also properly extended. - if(key.sk) { - for(let subkey of key.sk) { - ActiveKey.polyfill(subkey, layout, displayLayer); - } - } + @Enumerable + public get baseKeyEvent(): KeyEvent { + return new KeyEvent(this._baseKeyEvent); + } - let aKey = key as ActiveKey; - aKey.displayLayer = displayLayer; - aKey.layer = aKey.layer || displayLayer; - - // Compute the key's base KeyEvent properties for use in future event generation - aKey.constructBaseKeyEvent(layout, displayLayer); - } - - private constructBaseKeyEvent(layout: ActiveLayout, displayLayer: string) { - // Get key name and keyboard shift state (needed only for default layouts and physical keyboard handling) - // Note - virtual keys should be treated case-insensitive, so we force uppercasing here. - let layer = this.layer || displayLayer || ''; - let keyName= this.id ? this.id.toUpperCase() : null; - - // Start: mirrors _GetKeyEventProperties - - - // First check the virtual key, and process shift, control, alt or function keys - var Lkc: text.KeyEvent = { - // Override key shift state if specified for key in layout (corrected for popup keys KMEW-93) - Lmodifiers: text.KeyboardProcessor.getModifierState(layer), - Lstates: text.KeyboardProcessor.getStateFromLayer(layer), - Lcode: keyName ? text.Codes.keyCodes[keyName] : 0, - LisVirtualKey: true, - vkCode: 0, - kName: keyName, - kLayer: layer, - kbdLayer: displayLayer, - kNextLayer: this.nextlayer, - device: null, - isSynthetic: true - }; - - if(layout.keyboard) { - let keyboard = layout.keyboard; - - // Include *limited* support for mnemonic keyboards (Sept 2012) - // If a touch layout has been defined for a mnemonic keyout, do not perform mnemonic mapping for rules on touch devices. - if(keyboard.isMnemonic && !(layout.isDefault && layout.formFactor != 'desktop')) { - if(Lkc.Lcode != text.Codes.keyCodes['K_SPACE']) { // exception required, March 2013 - // Jan 2019 - interesting that 'K_SPACE' also affects the caps-state check... - Lkc.vkCode = Lkc.Lcode; - this.isMnemonic = true; - } - } else { - Lkc.vkCode=Lkc.Lcode; - } + /** + * Converts key IDs of the U_* form to their corresponding UTF-16 text. + * If an ID not matching the pattern is received, returns null. + * @param id + * @returns + */ + static unicodeIDToText(id: string, errorCallback?: (codeAsString: string) => void) { + if(!id || id.substring(0,2) != 'U_') { + return null; + } - // Support version 1.0 KeymanWeb keyboards that do not define positional vs mnemonic - if(!keyboard.definesPositionalOrMnemonic) { - // Not the best pattern, but currently safe - we don't look up any properties of any of the - // arguments in this use case, and the object's scope is extremely limited. - Lkc.Lcode = KeyMapping._USKeyCodeToCharCode(this.constructKeyEvent(null, null)); - Lkc.LisVirtualKey=false; + let result = ''; + const codePoints = id.substring(2).split('_'); + for(let codePoint of codePoints) { + const codePointValue = parseInt(codePoint, 16); + if (((0x0 <= codePointValue) && (codePointValue <= 0x1F)) || + ((0x80 <= codePointValue) && (codePointValue <= 0x9F)) || + isNaN(codePointValue)) { + if(errorCallback) { + errorCallback(codePoint); } + continue; + } else { + // String.fromCharCode() is inadequate to handle the entire range of Unicode + // Someday after upgrading to ES2015, can use String.fromCodePoint() + result += String.kmwFromCharCode(codePointValue); } + } + return result ? result : null; + } - this.baseKeyEvent = Lkc; + static sanitize(rawKey: LayoutKey) { + if(typeof rawKey.width == 'string') { + rawKey.width = parseInt(rawKey.width, 10); } + // Handles NaN cases as well as 'set to 0' cases; both are intentional here. + rawKey.width ||= ActiveKey.DEFAULT_KEY_WIDTH; - constructKeyEvent(keyboardProcessor: text.KeyboardProcessor, device: utils.DeviceSpec): text.KeyEvent { - // Make a deep copy of our preconstructed key event, filling it out from there. - let Lkc = utils.deepCopy(this.baseKeyEvent); - Lkc.device = device; + if(typeof rawKey.pad == 'string') { + rawKey.pad = parseInt(rawKey.pad, 10); + } + rawKey.pad ||= ActiveKey.DEFAULT_PAD; - if(this.isMnemonic) { - text.KeyboardProcessor.setMnemonicCode(Lkc, this.layer.indexOf('shift') != -1, keyboardProcessor ? keyboardProcessor.stateKeys['K_CAPS'] : false); - } + if(typeof rawKey.sp == 'string') { + rawKey.sp = Number.parseInt(rawKey.sp, 10) as ButtonClass; + } + rawKey.sp ||= 0; // The default button class. + } - // Performs common pre-analysis for both 'native' and 'embedded' OSK key & subkey input events. - // This part depends on the keyboard processor's active state. - if(keyboardProcessor) { - keyboardProcessor.setSyntheticEventDefaults(Lkc); - - // If it's a state key modifier, trigger its effects as part of the - // keystroke. - const bitmap = { - 'K_CAPS': text.Codes.stateBitmasks.CAPS, - 'K_NUMLOCK': text.Codes.stateBitmasks.NUM_LOCK, - 'K_SCROLL': text.Codes.stateBitmasks.SCROLL_LOCK - }; - const bitmask = bitmap[Lkc.kName]; - - if(bitmask) { - Lkc.Lstates ^= bitmask; - Lkc.LmodifierChange = true; + static polyfill(key: LayoutKey, keyboard: Keyboard, layout: ActiveLayout, displayLayer: string) { + // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. + let dummy = new ActiveKey(); + let proto = Object.getPrototypeOf(dummy); + + for(let prop in dummy) { + if(!key.hasOwnProperty(prop)) { + let descriptor = Object.getOwnPropertyDescriptor(proto, prop); + if(descriptor) { + // It's a computed property! Copy the descriptor onto the key's object. + Object.defineProperty(key, prop, descriptor); + } else { + key[prop] = dummy[prop]; } } - - return Lkc; } - public getSubkey(coreID: string): ActiveKey { - if(this.sk) { - for(let key of this.sk) { - if(key.coreID == coreID) { - return key; - } - } + // Ensure subkeys are also properly extended. + if(key.sk) { + for(let subkey of key.sk) { + ActiveKey.polyfill(subkey, keyboard, layout, displayLayer); } - - return null; } - } - export class ActiveRow implements LayoutRow { - // Identify key labels (e.g. *Shift*) that require the special OSK font - static readonly SPECIAL_LABEL=/\*\w+\*/; + let aKey = key as ActiveKey; + aKey.displayLayer = displayLayer; + aKey.layer = aKey.layer || displayLayer; - id: number; - key: ActiveKey[]; + // Compute the key's base KeyEvent properties for use in future event generation + aKey.constructBaseKeyEvent(keyboard, layout, displayLayer); + } - /** - * Used for calculating fat-fingering offsets. - */ - proportionalY: number; + private constructBaseKeyEvent(keyboard: Keyboard, layout: ActiveLayout, displayLayer: string) { + // Get key name and keyboard shift state (needed only for default layouts and physical keyboard handling) + // Note - virtual keys should be treated case-insensitive, so we force uppercasing here. + let layer = this.layer || displayLayer || ''; + let keyName= this.id ? this.id.toUpperCase() : null; + + // Start: mirrors _GetKeyEventProperties + + // First check the virtual key, and process shift, control, alt or function keys + let props: KeyEventSpec = { + // Override key shift state if specified for key in layout (corrected for popup keys KMEW-93) + Lmodifiers: Codes.getModifierState(layer), + Lstates: Codes.getStateFromLayer(layer), + Lcode: keyName ? Codes.keyCodes[keyName] : 0, + LisVirtualKey: true, + vkCode: 0, + kName: keyName, + kLayer: layer, + kbdLayer: displayLayer, + kNextLayer: this.nextlayer, + device: null, + isSynthetic: true + }; - private constructor() { + let Lkc: KeyEvent = new KeyEvent(props); - } + if(layout.keyboard) { + let keyboard = layout.keyboard; - static sanitize(rawRow: LayoutRow) { - for(const key of rawRow.key) { - // Test for a trailing comma included in spec, added as null object by IE - // It has only ever appeared at the end of a row's spec. - if(key == null) { - rawRow.key.length = rawRow.key.length-1; - } else { - ActiveKey.sanitize(key); + // Include *limited* support for mnemonic keyboards (Sept 2012) + // If a touch layout has been defined for a mnemonic keyout, do not perform mnemonic mapping for rules on touch devices. + if(keyboard.isMnemonic && !(layout.isDefault && layout.formFactor != 'desktop')) { + if(Lkc.Lcode != Codes.keyCodes['K_SPACE']) { // exception required, March 2013 + // Jan 2019 - interesting that 'K_SPACE' also affects the caps-state check... + Lkc.vkCode = Lkc.Lcode; + this.isMnemonic = true; } + } else { + Lkc.vkCode=Lkc.Lcode; } - if(typeof rawRow.id == 'string') { - rawRow.id = Number.parseInt(rawRow.id, 10); + // Support version 1.0 KeymanWeb keyboards that do not define positional vs mnemonic + if(!keyboard.definesPositionalOrMnemonic) { + // Not the best pattern, but currently safe - we don't look up any properties of any of the + // arguments in this use case, and the object's scope is extremely limited. + Lkc.Lcode = KeyMapping._USKeyCodeToCharCode(keyboard.constructKeyEvent(null, null, { + K_CAPS: false, + K_NUMLOCK: false, + K_SCROLL: false + })); + Lkc.LisVirtualKey=false; } } - static polyfill(row: LayoutRow, layout: ActiveLayout, displayLayer: string, totalWidth: number, proportionalY: number) { - // Apply defaults, setting the width and other undefined properties for each key - let keys=row['key']; - for(let j=0; j 0) { - const finalKey = keys[keys.length-1] as ActiveKey; + } - // If a single key, and padding is negative, add padding to right align the key - if(keys.length == 1 && finalKey.pad < 0) { - const keyPercent = finalKey.width/totalWidth; - const padPercent = 1-(totalPercent + keyPercent + rightMargin); + static sanitize(rawRow: LayoutRow) { + for(const key of rawRow.key) { + // Test for a trailing comma included in spec, added as null object by IE + // It has only ever appeared at the end of a row's spec. + if(key == null) { + rawRow.key.length = rawRow.key.length-1; + } else { + ActiveKey.sanitize(key); + } + } - // compute center's default x-coord (used in headless modes) - setProportions(finalKey, padPercent, keyPercent, totalPercent); - } else { - const padPercent = finalKey.pad/totalWidth; - const keyPercent = 1-(totalPercent + padPercent + rightMargin); + if(typeof rawRow.id == 'string') { + rawRow.id = Number.parseInt(rawRow.id, 10); + } + } - // compute center's default x-coord (used in headless modes) - setProportions(finalKey, padPercent, keyPercent, totalPercent); + static polyfill(row: LayoutRow, keyboard: Keyboard, layout: ActiveLayout, displayLayer: string, totalWidth: number, proportionalY: number) { + // Apply defaults, setting the width and other undefined properties for each key + let keys=row['key']; + for(let j=0; j 0) { + const finalKey = keys[keys.length-1] as ActiveKey; - constructor() { + // If a single key, and padding is negative, add padding to right align the key + if(keys.length == 1 && finalKey.pad < 0) { + const keyPercent = finalKey.width/totalWidth; + const padPercent = 1-(totalPercent + keyPercent + rightMargin); + // compute center's default x-coord (used in headless modes) + setProportions(finalKey, padPercent, keyPercent, totalPercent); + } else { + const padPercent = finalKey.pad/totalWidth; + const keyPercent = 1-(totalPercent + padPercent + rightMargin); + + // compute center's default x-coord (used in headless modes) + setProportions(finalKey, padPercent, keyPercent, totalPercent); + } } - static sanitize(rawLayer: LayoutLayer) { - for(const row of rawLayer.row) { - ActiveRow.sanitize(row); + // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. + let dummy = new ActiveRow(); + for(let key in dummy) { + if(!row.hasOwnProperty(key)) { + row[key] = dummy[key]; } } - static polyfill(layer: LayoutLayer, layout: ActiveLayout) { - layer.aligned=false; + let aRow = row as ActiveRow; + aRow.proportionalY = proportionalY; + } - // Create a DIV for each row of the group - let rows=layer['row']; + populateKeyMap(map: {[keyId: string]: ActiveKey}) { + this.key.forEach(function(key: ActiveKey) { + if(key.coreID) { + map[key.coreID] = key; + } + }); + } +} - // Calculate the maximum row width (in layout units) - let totalWidth=0; - for(const row of rows) { - let width=0; - const keys=row['key']; +export class ActiveLayer implements LayoutLayer { + row: ActiveRow[]; + id: string; - for(const key of keys) { - // So long as `sanitize` is called first, these coercions are safe. - width += (key.width as number) + (key.pad as number); - } + // These already exist on the objects, pre-polyfill... + // but they still need to be proactively declared on this type. + capsKey?: ActiveKey; + numKey?: ActiveKey; + scrollKey?: ActiveKey; - if(width > totalWidth) { - totalWidth = width; - } - } + totalWidth: number; - // Add default right margin - if(layout.formFactor == 'desktop') { - totalWidth += 5; // TODO: resolve difference between touch and desktop; why don't we use ActiveKey.DEFAULT_RIGHT_MARGIN? - } else { - totalWidth += ActiveKey.DEFAULT_RIGHT_MARGIN; - } + defaultKeyProportionalWidth: number; + rowProportionalHeight: number; - let rowCount = layer.row.length; - for(let i=0; i probability, use a function parameter in place - // of the formula in the loop below. - for(let key in keyDists) { - totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. + if(width > totalWidth) { + totalWidth = width; } + } + + // Add default right margin + if(layout.formFactor == 'desktop') { + totalWidth += 5; // TODO: resolve difference between touch and desktop; why don't we use ActiveKey.DEFAULT_RIGHT_MARGIN? + } else { + totalWidth += ActiveKey.DEFAULT_RIGHT_MARGIN; + } - for(let key in keyProbs) { - keyProbs[key] /= totalMass; + let rowCount = layer.row.length; + for(let i=0; i probability, use a function parameter in place + // of the formula in the loop below. + for(let key in keyDists) { + totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. + } + + for(let key in keyProbs) { + keyProbs[key] /= totalMass; + } - let keyDists: {[keyId: string]: number} = {}; + return keyProbs; + } - // This double-nested loop computes a pseudo-distance for the touch from each key. Quite useful for - // generating a probability distribution. - this.row.forEach(function(row: ActiveRow): void { - row.key.forEach(function(key: ActiveKey): void { - // If the key lacks an ID, just skip it. Sometimes used for padding. - if(!key.baseKeyID) { + /** + * Computes a squared 'pseudo-distance' for the touch from each key. (Not a proper metric.) + * Intended for use in generating a probability distribution over the keys based on the touch input. + * @param touchCoords A proportional (x, y) coordinate of the touch within the keyboard's geometry. + * Should be within [0, 0] to [1, 1]. + * @param kbdScaleRatio The ratio of the keyboard's horizontal scale to its vertical scale. + * For a 400 x 200 keyboard, should be 2. + */ + private keyTouchDistances(touchCoords: {x: number, y: number}, kbdScaleRatio: number): {[keyId: string]: number} { + let layer = this; + + let keyDists: {[keyId: string]: number} = {}; + + // This double-nested loop computes a pseudo-distance for the touch from each key. Quite useful for + // generating a probability distribution. + this.row.forEach(function(row: ActiveRow): void { + row.key.forEach(function(key: ActiveKey): void { + // If the key lacks an ID, just skip it. Sometimes used for padding. + if(!key.baseKeyID) { + return; + } else { + // Attempt to filter out known non-output keys. + // Results in a more optimized distribution. + if(Codes.isKnownOSKModifierKey(key.baseKeyID)) { + return; + } else if(key.isPadding) { // to the user, blank / padding keys do not exist. return; - } else { - // Attempt to filter out known non-output keys. - // Results in a more optimized distribution. - if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) { - return; - } else if(key.isPadding) { // to the user, blank / padding keys do not exist. - return; - } - } - // These represent the within-key distance of the touch from the key's center. - // Both should be on the interval [0, 0.5]. - let dx = Math.abs(touchCoords.x - key.proportionalX); - let dy = Math.abs(touchCoords.y - row.proportionalY); - - // If the touch isn't within the key, these store the out-of-key distance - // from the closest point on the key being checked. - let distX: number, distY: number; - - if(dx > 0.5 * key.proportionalWidth) { - distX = (dx - 0.5 * key.proportionalWidth); - dx = 0.5; - } else { - distX = 0; - dx /= key.proportionalWidth; } + } + // These represent the within-key distance of the touch from the key's center. + // Both should be on the interval [0, 0.5]. + let dx = Math.abs(touchCoords.x - key.proportionalX); + let dy = Math.abs(touchCoords.y - row.proportionalY); + + // If the touch isn't within the key, these store the out-of-key distance + // from the closest point on the key being checked. + let distX: number, distY: number; + + if(dx > 0.5 * key.proportionalWidth) { + distX = (dx - 0.5 * key.proportionalWidth); + dx = 0.5; + } else { + distX = 0; + dx /= key.proportionalWidth; + } - if(dy > 0.5 * layer.rowProportionalHeight) { - distY = (dy - 0.5 * layer.rowProportionalHeight); - dy = 0.5; - } else { - distY = 0; - dy /= layer.rowProportionalHeight; - } + if(dy > 0.5 * layer.rowProportionalHeight) { + distY = (dy - 0.5 * layer.rowProportionalHeight); + dy = 0.5; + } else { + distY = 0; + dy /= layer.rowProportionalHeight; + } - // Now that the differentials are computed, it's time to do distance scaling. - // - // For out-of-key distance, we scale the X component by the keyboard's aspect ratio - // to get the actual out-of-key distance rather than proportional. - distX *= kbdScaleRatio; - - // While the keys are rarely perfect squares, we map all within-key distance - // to a square shape. (ALT/CMD should seem as close to SPACE as a 'B'.) - // - // For that square, we take the rowHeight as its edge lengths. - distX += dx * layer.rowProportionalHeight; - distY += dy * layer.rowProportionalHeight; - - let distance = distX * distX + distY * distY; - keyDists[key.coreID] = distance; - }); + // Now that the differentials are computed, it's time to do distance scaling. + // + // For out-of-key distance, we scale the X component by the keyboard's aspect ratio + // to get the actual out-of-key distance rather than proportional. + distX *= kbdScaleRatio; + + // While the keys are rarely perfect squares, we map all within-key distance + // to a square shape. (ALT/CMD should seem as close to SPACE as a 'B'.) + // + // For that square, we take the rowHeight as its edge lengths. + distX += dx * layer.rowProportionalHeight; + distY += dy * layer.rowProportionalHeight; + + let distance = distX * distX + distY * distY; + keyDists[key.coreID] = distance; }); + }); - return keyDists; - } + return keyDists; + } - getKey(keyId: string) { - // Keys usually are specified in a "long form" prefixed with their layer's ID. - if(keyId.indexOf(this.id + '-') == 0) { - keyId = keyId.replace(this.id + '-', ''); - } + getKey(keyId: string) { + // Keys usually are specified in a "long form" prefixed with their layer's ID. + if(keyId.indexOf(this.id + '-') == 0) { + keyId = keyId.replace(this.id + '-', ''); + } - let idComponents = keyId.split('::'); - if(idComponents.length > 1) { - let baseKey = this.keyMap[idComponents[0]]; - return baseKey.getSubkey(idComponents[1]); - } else { - return this.keyMap[keyId]; - } + let idComponents = keyId.split('::'); + if(idComponents.length > 1) { + let baseKey = this.keyMap[idComponents[0]]; + return baseKey.getSubkey(idComponents[1]); + } else { + return this.keyMap[keyId]; } } +} - export class ActiveLayout implements LayoutFormFactor{ - layer: ActiveLayer[]; - font: string; - keyLabels: boolean; - isDefault?: boolean; - keyboard: Keyboard; - formFactor: utils.FormFactor; - - /** - * Facilitates mapping layer id strings to their specification objects. - */ - layerMap: {[layerId: string]: ActiveLayer}; - - private constructor() { - - } - - getLayer(layerId: string): ActiveLayer { - return this.layerMap[layerId]; - } - - /** - * Refer to https://github.com/keymanapp/keyman/issues/254, which mentions - * KD-11 from a prior issue-tracking system from the closed-source days that - * resulted in an unintended extra empty row. - * - * It'll be pretty rare to see a keyboard affected by the bug, but we don't - * 100% control all keyboards out there, so it's best we make sure the edge - * case is covered. - * - * @param layers The layer group to be loaded for the form factor. Will be - * mutated by this operation. - */ - static correctLayerEmptyRowBug(layers: LayoutLayer[]) { - for(let n=0; n=0; i--) { - if(!Array.isArray(rows[i]['key']) || rows[i]['key'].length == 0) { - rows.splice(i, 1) - } +export class ActiveLayout implements LayoutFormFactor{ + layer: ActiveLayer[]; + font: string; + keyLabels: boolean; + isDefault?: boolean; + keyboard: Keyboard; + formFactor: DeviceSpec.FormFactor; + + /** + * Facilitates mapping layer id strings to their specification objects. + */ + layerMap: {[layerId: string]: ActiveLayer}; + + private constructor() { + + } + + getLayer(layerId: string): ActiveLayer { + return this.layerMap[layerId]; + } + + /** + * Refer to https://github.com/keymanapp/keyman/issues/254, which mentions + * KD-11 from a prior issue-tracking system from the closed-source days that + * resulted in an unintended extra empty row. + * + * It'll be pretty rare to see a keyboard affected by the bug, but we don't + * 100% control all keyboards out there, so it's best we make sure the edge + * case is covered. + * + * @param layers The layer group to be loaded for the form factor. Will be + * mutated by this operation. + */ + static correctLayerEmptyRowBug(layers: LayoutLayer[]) { + for(let n=0; n=0; i--) { + if(!Array.isArray(rows[i]['key']) || rows[i]['key'].length == 0) { + rows.splice(i, 1) } } } + } - static sanitize(rawLayout: LayoutFormFactor) { - ActiveLayout.correctLayerEmptyRowBug(rawLayout.layer); + static sanitize(rawLayout: LayoutFormFactor) { + ActiveLayout.correctLayerEmptyRowBug(rawLayout.layer); - for(const layer of rawLayout.layer) { - ActiveLayer.sanitize(layer); - } + for(const layer of rawLayout.layer) { + ActiveLayer.sanitize(layer); } + } - /** - * - * @param layout - * @param formFactor - */ - static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: utils.FormFactor): ActiveLayout { - if(layout == null) { - throw new Error("Cannot build an ActiveLayout for a null specification."); - } + /** + * + * @param layout + * @param formFactor + */ + static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout { + if(layout == null) { + throw new Error("Cannot build an ActiveLayout for a null specification."); + } - /* Standardize the layout object's data types. - * - * In older versions of KMW, some numeric properties were long represented as strings instead, - * and that lives on within a _lot_ of keyboards. The data should be sanitized before it - * is processed by this method. - */ - this.sanitize(layout); - - // Create a separate OSK div for each OSK layer, only one of which will ever be visible - var n: number; - let layerMap: {[layerId: string]: ActiveLayer} = {}; - - let layers=layout.layer; - - // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. - let dummy = new ActiveLayout(); - for(let key in dummy) { - if(!layout.hasOwnProperty(key)) { - layout[key] = dummy[key]; - } - } + /* Standardize the layout object's data types. + * + * In older versions of KMW, some numeric properties were long represented as strings instead, + * and that lives on within a _lot_ of keyboards. The data should be sanitized before it + * is processed by this method. + */ + this.sanitize(layout); - let aLayout = layout as ActiveLayout; - aLayout.keyboard = keyboard; - aLayout.formFactor = formFactor; + // Create a separate OSK div for each OSK layer, only one of which will ever be visible + var n: number; + let layerMap: {[layerId: string]: ActiveLayer} = {}; - for(n=0; n?~~~~~ '; + + static readonly DEFAULT_RAW_SPEC = {'F':'Tahoma', 'BK': Layouts.dfltText}; + + // Cross-reference with the ids in osk.setButtonClass. + static buttonClasses: {[name: string]: ButtonClass} = { + 'DEFAULT':0, + 'SHIFT':1, + 'SHIFT-ON':2, + 'SPECIAL':3, + 'SPECIAL-ON':4, + 'DEADKEY':8, + 'BLANK':9, + 'HIDDEN':10 + }; - export type LayoutSpec = { - "desktop"?: LayoutFormFactor, - "phone"?: LayoutFormFactor, - "tablet"?: LayoutFormFactor - } + static modifierSpecials = { + 'leftalt': '*LAlt*', + 'rightalt': '*RAlt*', + 'alt': '*Alt*', + 'leftctrl': '*LCtrl*', + 'rightctrl': '*RCtrl*', + 'ctrl': '*Ctrl*', + 'ctrl-alt': '*AltGr*', + 'leftctrl-leftalt': '*LAltCtrl*', + 'rightctrl-rightalt': '*RAltCtrl*', + 'leftctrl-leftalt-shift': '*LAltCtrlShift*', + 'rightctrl-rightalt-shift': '*RAltCtrlShift*', + 'shift': '*Shift*', + 'shift-alt': '*AltShift*', + 'shift-ctrl': '*CtrlShift*', + 'shift-ctrl-alt': '*AltCtrlShift*', + 'leftalt-shift': '*LAltShift*', + 'rightalt-shift': '*RAltShift*', + 'leftctrl-shift': '*LCtrlShift*', + 'rightctrl-shift': '*RCtrlShift*' + }; - // This class manages default layout construction for consumption by OSKs without a specified layout. - export class Layouts { - static dfltCodes=[ - "K_BKQUOTE","K_1","K_2","K_3","K_4","K_5","K_6","K_7","K_8","K_9","K_0", - "K_HYPHEN","K_EQUAL","K_*","K_*","K_*","K_Q","K_W","K_E","K_R","K_T", - "K_Y","K_U","K_I","K_O","K_P","K_LBRKT","K_RBRKT","K_BKSLASH","K_*", - "K_*","K_*","K_A","K_S","K_D","K_F","K_G","K_H","K_J","K_K","K_L", - "K_COLON","K_QUOTE","K_*","K_*","K_*","K_*","K_*","K_oE2", - "K_Z","K_X","K_C","K_V","K_B","K_N","K_M","K_COMMA","K_PERIOD", - "K_SLASH","K_*","K_*","K_*","K_*","K_*","K_SPACE" - ]; - - static dfltText='`1234567890-=\xA7~~qwertyuiop[]\\~~~asdfghjkl;\'~~~~~?zxcvbnm,./~~~~~ ' - +'~!@#$%^&*()_+\xA7~~QWERTYUIOP{}\\~~~ASDFGHJKL:"~~~~~?ZXCVBNM<>?~~~~~ '; - - static readonly DEFAULT_RAW_SPEC = {'F':'Tahoma', 'BK': Layouts.dfltText}; - - // Cross-reference with the ids in osk.setButtonClass. - static buttonClasses: {[name: string]: ButtonClass} = { - 'DEFAULT':0, - 'SHIFT':1, - 'SHIFT-ON':2, - 'SPECIAL':3, - 'SPECIAL-ON':4, - 'DEADKEY':8, - 'BLANK':9, - 'HIDDEN':10 - }; - - static modifierSpecials = { - 'leftalt': '*LAlt*', - 'rightalt': '*RAlt*', - 'alt': '*Alt*', - 'leftctrl': '*LCtrl*', - 'rightctrl': '*RCtrl*', - 'ctrl': '*Ctrl*', - 'ctrl-alt': '*AltGr*', - 'leftctrl-leftalt': '*LAltCtrl*', - 'rightctrl-rightalt': '*RAltCtrl*', - 'leftctrl-leftalt-shift': '*LAltCtrlShift*', - 'rightctrl-rightalt-shift': '*RAltCtrlShift*', - 'shift': '*Shift*', - 'shift-alt': '*AltShift*', - 'shift-ctrl': '*CtrlShift*', - 'shift-ctrl-alt': '*AltCtrlShift*', - 'leftalt-shift': '*LAltShift*', - 'rightalt-shift': '*RAltShift*', - 'leftctrl-shift': '*LCtrlShift*', - 'rightctrl-shift': '*RCtrlShift*' - }; - - /** - * Build a default layout for keyboards with no explicit layout - * - * @param {Object} PVK raw specifications - * @param {Keyboard} keyboard keyboard object (as loaded) - * @param {string} formFactor (really utils.FormFactor) - * @return {LayoutFormFactor} - */ - static buildDefaultLayout(PVK, keyboard: Keyboard, formFactor: string): LayoutFormFactor { - // Build a layout using the default for the device - var layoutType=formFactor; - - if(typeof Layouts.dfltLayout[layoutType] != 'object') { - layoutType = 'desktop'; - } + /** + * Build a default layout for keyboards with no explicit layout + * + * @param {Object} PVK raw specifications + * @param {Keyboard} keyboard keyboard object (as loaded) + * @param {string} formFactor (really utils.FormFactor) + * @return {LayoutFormFactor} + */ + static buildDefaultLayout(PVK, keyboard: Keyboard, formFactor: string): LayoutFormFactor { + // Build a layout using the default for the device + var layoutType=formFactor; + + if(typeof Layouts.dfltLayout[layoutType] != 'object') { + layoutType = 'desktop'; + } - let kbdBitmask = Codes.modifierBitmasks['NON_CHIRAL']; - // An unfortunate dependency there. Should probably also set a version within web-core for use. - let kbdDevVersion = utils.Version.CURRENT; - if(keyboard) { - kbdBitmask = keyboard.modifierBitmask; - kbdDevVersion = keyboard.compilerVersion; - } + let kbdBitmask = Codes.modifierBitmasks['NON_CHIRAL']; + // An unfortunate dependency there. Should probably also set a version within web-core for use. + let kbdDevVersion = Version.CURRENT; + if(keyboard) { + kbdBitmask = keyboard.modifierBitmask; + kbdDevVersion = keyboard.compilerVersion; + } - if(!PVK) { - PVK = this.DEFAULT_RAW_SPEC; - } + if(!PVK) { + PVK = this.DEFAULT_RAW_SPEC; + } - // Clone the default layout object for this device - var layout: LayoutFormFactor = utils.deepCopy(Layouts.dfltLayout[layoutType]); + // Clone the default layout object for this device + var layout: LayoutFormFactor = deepCopy(Layouts.dfltLayout[layoutType]); - var n,layers=layout['layer'], keyLabels: KLS=PVK['KLS'], key102=PVK['K102']; - var i, j, k, m, row, rows: LayoutRow[], key: LayoutKey, keys: LayoutKey[]; - var chiral: boolean = (kbdBitmask & Codes.modifierBitmasks.IS_CHIRAL) != 0; + var n,layers=layout['layer'], keyLabels: KLS=PVK['KLS'], key102=PVK['K102']; + var i, j, k, m, row, rows: LayoutRow[], key: LayoutKey, keys: LayoutKey[]; + var chiral: boolean = (kbdBitmask & Codes.modifierBitmasks.IS_CHIRAL) != 0; - if(PVK['F']) { - // The KeymanWeb compiler generates a string of the format `[italic ][bold ] 1em ""` - // We will ignore the bold, italic and font size spec - let legacyFontSpec = /^(?:(?:italic|bold) )* *[0-9.eE-]+(?:[a-z]+) "(.+)"$/.exec(PVK['F']); - if(legacyFontSpec) { - layout.font = legacyFontSpec[1]; - } + if(PVK['F']) { + // The KeymanWeb compiler generates a string of the format `[italic ][bold ] 1em ""` + // We will ignore the bold, italic and font size spec + let legacyFontSpec = /^(?:(?:italic|bold) )* *[0-9.eE-]+(?:[a-z]+) "(.+)"$/.exec(PVK['F']); + if(legacyFontSpec) { + layout.font = legacyFontSpec[1]; } + } - var kmw10Plus = !(typeof keyLabels == 'undefined' || !keyLabels); - if(!kmw10Plus) { - // Save the processed key label information to the keyboard's general data. - // Makes things more efficient elsewhere and for reloading after keyboard swaps. - keyLabels = PVK['KLS'] = Layouts.processLegacyDefinitions(PVK['BK']); - } + var kmw10Plus = !(typeof keyLabels == 'undefined' || !keyLabels); + if(!kmw10Plus) { + // Save the processed key label information to the keyboard's general data. + // Makes things more efficient elsewhere and for reloading after keyboard swaps. + keyLabels = PVK['KLS'] = Layouts.processLegacyDefinitions(PVK['BK']); + } - // Identify key labels (e.g. *Shift*) that require the special OSK font - var specialLabel=/\*\w+\*/; + // Identify key labels (e.g. *Shift*) that require the special OSK font + var specialLabel=/\*\w+\*/; - // *** Step 1: instantiate the layer objects. *** + // *** Step 1: instantiate the layer objects. *** - // Get the list of valid layers, enforcing that the 'default' layer must be the first one processed. - var validIdList = Object.getOwnPropertyNames(keyLabels), invalidIdList = []; - validIdList.splice(validIdList.indexOf('default'), 1); - validIdList = [ 'default' ].concat(validIdList); + // Get the list of valid layers, enforcing that the 'default' layer must be the first one processed. + var validIdList = Object.getOwnPropertyNames(keyLabels), invalidIdList = []; + validIdList.splice(validIdList.indexOf('default'), 1); + validIdList = [ 'default' ].concat(validIdList); - // Automatic AltGr emulation if the 'leftctrl-leftalt' layer is otherwise undefined. - if(keyboard && keyboard.emulatesAltGr) { - // We insert only the layers that need to be emulated. - if((validIdList.indexOf('leftctrl-leftalt') == -1) && validIdList.indexOf('rightalt') != -1) { - validIdList.push('leftctrl-leftalt'); - keyLabels['leftctrl-leftalt'] = keyLabels['rightalt']; - } + // Automatic AltGr emulation if the 'leftctrl-leftalt' layer is otherwise undefined. + if(keyboard && keyboard.emulatesAltGr) { + // We insert only the layers that need to be emulated. + if((validIdList.indexOf('leftctrl-leftalt') == -1) && validIdList.indexOf('rightalt') != -1) { + validIdList.push('leftctrl-leftalt'); + keyLabels['leftctrl-leftalt'] = keyLabels['rightalt']; + } - if((validIdList.indexOf('leftctrl-leftalt-shift') == -1) && validIdList.indexOf('rightalt-shift') != -1) { - validIdList.push('leftctrl-leftalt-shift'); - keyLabels['leftctrl-leftalt-shift'] = keyLabels['rightalt-shift']; - } + if((validIdList.indexOf('leftctrl-leftalt-shift') == -1) && validIdList.indexOf('rightalt-shift') != -1) { + validIdList.push('leftctrl-leftalt-shift'); + keyLabels['leftctrl-leftalt-shift'] = keyLabels['rightalt-shift']; } + } - // If there is no predefined layout, even touch layouts will follow the desktop's - // setting for the displayUnderlying flag. As the desktop layout uses a different - // format for its layout spec, that's found at the field referenced below. - layout["displayUnderlying"] = keyboard ? !!keyboard.scriptObject['KDU'] : false; + // If there is no predefined layout, even touch layouts will follow the desktop's + // setting for the displayUnderlying flag. As the desktop layout uses a different + // format for its layout spec, that's found at the field referenced below. + layout["displayUnderlying"] = keyboard ? !!keyboard.scriptObject['KDU'] : false; - // For desktop devices, we must create all layers, even if invalid. - if(formFactor == 'desktop') { - invalidIdList = Layouts.generateLayerIds(chiral); + // For desktop devices, we must create all layers, even if invalid. + if(formFactor == 'desktop') { + invalidIdList = Layouts.generateLayerIds(chiral); - // Filter out all ids considered valid. (We also don't want duplicates in the following list...) - for(n=0; n 0) { - layers[n]=utils.deepCopy(layers[0]); - } - layers[n]['id']=idList[n]; - layers[n]['nextlayer']=idList[n]; // This would only be different for a dynamic keyboard - - // Extraced into a helper method to improve readability. - Layouts.formatDefaultLayer(layers[n], chiral, formFactor, !!key102); + for(n=0; n 0) { + layers[n]=deepCopy(layers[0]); } + layers[n]['id']=idList[n]; + layers[n]['nextlayer']=idList[n]; // This would only be different for a dynamic keyboard - // *** Step 2: Layer objects now exist; time to fill them with the appropriate key labels and key styles *** - for(n=0; n= 0 && kx < layerSpec.length) key['text']=layerSpec[kx]; - } + // Extraced into a helper method to improve readability. + Layouts.formatDefaultLayer(layers[n], chiral, formFactor, !!key102); + } - // Legacy (pre 12.0) behavior: fall back to US English keycap text as default for the base two layers - // if a key cap is not otherwise defined. (Any intentional 'ghost' keys must be explicitly defined.) - if(isDefault && kbdDevVersion.precedes(utils.Version.NO_DEFAULT_KEYCAPS)) { - if(key['id'] != 'K_SPACE' && kx+65 * isShift < Layouts.dfltText.length && key['text'] !== null) { - key['text'] = key['text'] || Layouts.dfltText[kx+65*isShift]; - } - } + // *** Step 2: Layer objects now exist; time to fill them with the appropriate key labels and key styles *** + for(n=0; n= 0 && kx < layerSpec.length) key['text']=layerSpec[kx]; } - // Leave any unmarked key caps as null strings - if(key['text'] !== null) { - key['text'] = key['text'] || ''; + // Legacy (pre 12.0) behavior: fall back to US English keycap text as default for the base two layers + // if a key cap is not otherwise defined. (Any intentional 'ghost' keys must be explicitly defined.) + if(isDefault && kbdDevVersion.precedes(Version.NO_DEFAULT_KEYCAPS)) { + if(key['id'] != 'K_SPACE' && kx+65 * isShift < Layouts.dfltText.length && key['text'] !== null) { + key['text'] = key['text'] || Layouts.dfltText[kx+65*isShift]; + } } + } - // Detect important tracking keys. - switch(key['id']) { - case "K_SHIFT": - shiftKey=key; - break; - case "K_TAB": - nextKey=key; - break; - case "K_CAPS": - capsKey=key; - break; - case "K_NUMLOCK": - numKey=key; - break; - case "K_SCROLL": - scrollKey=key; - break; - } + // Leave any unmarked key caps as null strings + if(key['text'] !== null) { + key['text'] = key['text'] || ''; + } - // Remove pop-up shift keys referencing invalid layers (Build 349) - if(key['sk'] != null) { - for(k=0; k 0 && shiftKey != null) { - shiftKey['sp']=Layouts.buttonClasses['SHIFT-ON']; - shiftKey['sk']=null; - shiftKey['text'] = Layouts.modifierSpecials[layers[n].id] ? Layouts.modifierSpecials[layers[n].id] : "*Shift*"; + if(key['sk'].length == 0) { + key['sk']=null; + } } } } - return layout; - } - - /** - * Function getLayerId - * Scope Private - * @param {number} m shift modifier code - * @return {string} layer string from shift modifier code (desktop keyboards) - * Description Get name of layer from code, where the modifer order is determined by ascending bit-flag value. - */ - static getLayerId(m: number): string { - let modifierCodes = Codes.modifierCodes; - - var s=''; - if(m == 0) { - return 'default'; - } else { - if(m & modifierCodes['LCTRL']) { - s = (s.length > 0 ? s + '-' : '') + 'leftctrl'; - } - if(m & modifierCodes['RCTRL']) { - s = (s.length > 0 ? s + '-' : '') + 'rightctrl'; + // We're done with the layer keys initialization pass. Time to do post-analysis layer-level init where necessary. + layer.shiftKey=shiftKey; + layer.capsKey=capsKey; + layer.numKey=numKey; + layer.scrollKey=scrollKey; + + // Set modifier key appearance and behaviour for non-desktop devices using the default layout + if(formFactor != 'desktop') { + if(n > 0 && shiftKey != null) { + shiftKey['sp']=Layouts.buttonClasses['SHIFT-ON']; + shiftKey['sk']=null; + shiftKey['text'] = Layouts.modifierSpecials[layers[n].id] ? Layouts.modifierSpecials[layers[n].id] : "*Shift*"; } - if(m & modifierCodes['LALT']) { - s = (s.length > 0 ? s + '-' : '') + 'leftalt'; - } - if(m & modifierCodes['RALT']) { - s = (s.length > 0 ? s + '-' : '') + 'rightalt'; - } - if(m & modifierCodes['SHIFT']) { - s = (s.length > 0 ? s + '-' : '') + 'shift'; - } - if(m & modifierCodes['CTRL']) { - s = (s.length > 0 ? s + '-' : '') + 'ctrl'; - } - if(m & modifierCodes['ALT']) { - s = (s.length > 0 ? s + '-' : '') + 'alt'; - } - return s; } } - /** - * Generates a list of potential layer ids for the specified chirality mode. - * - * @param {boolean} chiral // Does the keyboard use chiral modifiers or not? - */ - static generateLayerIds(chiral: boolean): string[] { - var layerCnt, offset; - - if(chiral) { - layerCnt=32; - offset=0x01; - } else { - layerCnt=8; - offset=0x10; + return layout; + } + + /** + * Function getLayerId + * Scope Private + * @param {number} m shift modifier code + * @return {string} layer string from shift modifier code (desktop keyboards) + * Description Get name of layer from code, where the modifer order is determined by ascending bit-flag value. + */ + static getLayerId(m: number): string { + let modifierCodes = Codes.modifierCodes; + + var s=''; + if(m == 0) { + return 'default'; + } else { + if(m & modifierCodes['LCTRL']) { + s = (s.length > 0 ? s + '-' : '') + 'leftctrl'; } + if(m & modifierCodes['RCTRL']) { + s = (s.length > 0 ? s + '-' : '') + 'rightctrl'; + } + if(m & modifierCodes['LALT']) { + s = (s.length > 0 ? s + '-' : '') + 'leftalt'; + } + if(m & modifierCodes['RALT']) { + s = (s.length > 0 ? s + '-' : '') + 'rightalt'; + } + if(m & modifierCodes['SHIFT']) { + s = (s.length > 0 ? s + '-' : '') + 'shift'; + } + if(m & modifierCodes['CTRL']) { + s = (s.length > 0 ? s + '-' : '') + 'ctrl'; + } + if(m & modifierCodes['ALT']) { + s = (s.length > 0 ? s + '-' : '') + 'alt'; + } + return s; + } + } - var layerIds = []; + /** + * Generates a list of potential layer ids for the specified chirality mode. + * + * @param {boolean} chiral // Does the keyboard use chiral modifiers or not? + */ + static generateLayerIds(chiral: boolean): string[] { + var layerCnt, offset; + + if(chiral) { + layerCnt=32; + offset=0x01; + } else { + layerCnt=8; + offset=0x10; + } - for(var i=0; i < layerCnt; i++) { - layerIds.push(Layouts.getLayerId(i * offset)); - } + var layerIds = []; - return layerIds; + for(var i=0; i < layerCnt; i++) { + layerIds.push(Layouts.getLayerId(i * offset)); } - /** - * Sets a formatting property for the modifier keys when constructing a default layout for a keyboard. - * - * @param {Object} layer // One layer specification - * @param {boolean} chiral // Whether or not the keyboard uses chiral modifier information. - * @param {string} formFactor // The form factor of the device the layout is being constructed for. - * @param {boolean} key102 // Whether or not the extended key 102 should be hidden. - */ - static formatDefaultLayer(layer: LayoutLayer, chiral: boolean, formFactor: string, key102: boolean) { - var layerId = layer['id']; - let buttonClasses = Layouts.buttonClasses; - - // Correct appearance of state-dependent modifier keys according to group - for(var i=0; i -/// -/// +import Codes from "../text/codes.js"; +import { Layouts, type LayoutFormFactor } from "./defaultLayouts.js"; +import { ActiveKey, ActiveLayout } from "./activeLayout.js"; +import KeyEvent from "../text/keyEvent.js"; +import type OutputTarget from "../text/outputTarget.js"; + +import type { ComplexKeyboardStore } from "../text/kbdInterface.js"; + +import { Version, DeviceSpec } from "@keymanapp/web-utils"; +import StateKeyMap from "./stateKeyMap.js"; + +/** + * Stores preprocessed properties of a keyboard for quick retrieval later. + */ +class CacheTag { + stores: {[storeName: string]: ComplexKeyboardStore}; + + constructor() { + this.stores = {}; + } +} + +export enum LayoutState { + NOT_LOADED = undefined, + POLYFILLED = 1, + CALIBRATED = 2 +} + +export interface VariableStoreDictionary { + [name: string]: string; +}; + + +/** + * Acts as a wrapper class for Keyman keyboards compiled to JS, providing type information + * and keyboard-centered functionality in an object-oriented way without modifying the + * wrapped keyboard itself. + */ +export default class Keyboard { + public static DEFAULT_SCRIPT_OBJECT = { + 'gs': function(outputTarget, keystroke) { return false; }, // no matching rules; rely on defaultRuleOutput entirely + 'KI': '', // The currently-existing default keyboard ID; we already have checks that focus against this. + 'KN': '', + 'KV': Layouts.DEFAULT_RAW_SPEC, + 'KM': 0 // May not be the best default, but this matches current behavior when there is no activeKeyboard. + } -namespace com.keyman.keyboards { /** - * Stores preprocessed properties of a keyboard for quick retrieval later. + * This is the object provided to KeyboardInterface.registerKeyboard - that is, the keyboard + * being wrapped. + * + * TODO: Make this private instead. But there are a LOT of references that must be rooted out first. */ - class CacheTag { - stores: {[storeName: string]: text.ComplexKeyboardStore}; + public readonly scriptObject: any; + private layoutStates: {[layout: string]: LayoutState}; - constructor() { - this.stores = {}; + constructor(keyboardScript: any) { + if(keyboardScript) { + this.scriptObject = keyboardScript; + } else { + this.scriptObject = Keyboard.DEFAULT_SCRIPT_OBJECT; } + this.layoutStates = {}; } - export enum LayoutState { - NOT_LOADED = undefined, - POLYFILLED = 1, - CALIBRATED = 2 + /** + * Calls the keyboard's `gs` function, which represents the keyboard source's begin Unicode group. + */ + process(outputTarget: OutputTarget, keystroke: KeyEvent): boolean { + return this.scriptObject['gs'](outputTarget, keystroke); } - export interface VariableStoreDictionary { - [name: string]: string; - }; + /** + * Calls the keyboard's `gn` function, which represents the keyboard source's begin newContext group. + */ + processNewContextEvent(outputTarget: OutputTarget, keystroke: KeyEvent): boolean { + return this.scriptObject['gn'] ? this.scriptObject['gn'](outputTarget, keystroke) : false; + } + + /** + * Calls the keyboard's `gpk` function, which represents the keyboard source's begin postKeystroke group. + */ + processPostKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): boolean { + return this.scriptObject['gpk'] ? this.scriptObject['gpk'](outputTarget, keystroke) : false; + } + get isHollow(): boolean { + return this.scriptObject == Keyboard.DEFAULT_SCRIPT_OBJECT; + } + + get id(): string { + return this.scriptObject['KI']; + } + + get name(): string { + return this.scriptObject['KN']; + } /** - * Acts as a wrapper class for Keyman keyboards compiled to JS, providing type information - * and keyboard-centered functionality in an object-oriented way without modifying the - * wrapped keyboard itself. + * Cache variable store values + * + * Primarily used for predictive text to prevent variable store + * values from being changed in 'fat finger' processing. + * + * KVS is available in keyboards compiled with Keyman Developer 15 + * and later versions. See #2924. + * + * @returns an object with each property referencing a variable store */ - export class Keyboard { - public static DEFAULT_SCRIPT_OBJECT = { - 'gs': function(outputTarget, keystroke) { return false; }, // no matching rules; rely on defaultRuleOutput entirely - 'KI': '', // The currently-existing default keyboard ID; we already have checks that focus against this. - 'KN': '', - 'KV': Layouts.DEFAULT_RAW_SPEC, - 'KM': 0 // May not be the best default, but this matches current behavior when there is no activeKeyboard. + get variableStores(): VariableStoreDictionary { + const storeNames = this.scriptObject['KVS']; + let values = {}; + if(Array.isArray(storeNames)) { + for(let store of storeNames) { + values[store] = this.scriptObject[store]; + } } + return values; + } - /** - * This is the object provided to KeyboardInterface.registerKeyboard - that is, the keyboard - * being wrapped. - * - * TODO: Make this private instead. But there are a LOT of references that must be rooted out first. - */ - public readonly scriptObject: any; - private layoutStates: {[layout: string]: LayoutState}; - - constructor(keyboardScript: any) { - if(keyboardScript) { - this.scriptObject = keyboardScript; - } else { - this.scriptObject = Keyboard.DEFAULT_SCRIPT_OBJECT; + /** + * Restore variable store values from cache + * + * KVS is available in keyboards compiled with Keyman Developer 15 + * and later versions. See #2924. + * + * @param values name-value pairs for each store value + */ + set variableStores(values: VariableStoreDictionary) { + const storeNames = this.scriptObject['KVS']; + if(Array.isArray(storeNames)) { + for(let store of storeNames) { + // If the value is not present in the cache, don't overwrite it; + // while this is not used in initial implementation, we could use + // it in future to update a single variable store value rather than + // the whole cache. + if(typeof values[store] == 'string') { + this.scriptObject[store] = values[store]; + } } - this.layoutStates = {}; } + } - /** - * Calls the keyboard's `gs` function, which represents the keyboard source's begin Unicode group. - */ - process(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean { - return this.scriptObject['gs'](outputTarget, keystroke); - } + // TODO: Better typing. + private get _legacyLayoutSpec(): any { + return this.scriptObject['KV']; // used with buildDefaultLayout; layout must be constructed at runtime. + } - /** - * Calls the keyboard's `gn` function, which represents the keyboard source's begin newContext group. - */ - processNewContextEvent(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean { - return this.scriptObject['gn'] ? this.scriptObject['gn'](outputTarget, keystroke) : false; - } + // May return null if no layouts exist or have been initialized. + private get _layouts(): {[formFactor: string]: LayoutFormFactor} { + return this.scriptObject['KVKL']; // This one is compiled by Developer's visual keyboard layout editor. + } - /** - * Calls the keyboard's `gpk` function, which represents the keyboard source's begin postKeystroke group. - */ - processPostKeystroke(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean { - return this.scriptObject['gpk'] ? this.scriptObject['gpk'](outputTarget, keystroke) : false; - } + private set _layouts(value) { + this.scriptObject['KVKL'] = value; + } - get isHollow(): boolean { - return this.scriptObject == Keyboard.DEFAULT_SCRIPT_OBJECT; - } + get compilerVersion(): Version { + return new Version(this.scriptObject['KVER']); + } - get id(): string { - return this.scriptObject['KI']; - } + get isMnemonic(): boolean { + return !!this.scriptObject['KM']; + } - get name(): string { - return this.scriptObject['KN']; - } + get definesPositionalOrMnemonic(): boolean { + return typeof this.scriptObject['KM'] != 'undefined'; + } - /** - * Cache variable store values - * - * Primarily used for predictive text to prevent variable store - * values from being changed in 'fat finger' processing. - * - * KVS is available in keyboards compiled with Keyman Developer 15 - * and later versions. See #2924. - * - * @returns an object with each property referencing a variable store - */ - get variableStores(): VariableStoreDictionary { - const storeNames = this.scriptObject['KVS']; - let values = {}; - if(Array.isArray(storeNames)) { - for(let store of storeNames) { - values[store] = this.scriptObject[store]; - } - } - return values; - } + /** + * HTML help text, as specified by either the &kmw_helptext or &kmw_helpfile system stores. + * + * Reference: https://help.keyman.com/developer/language/reference/kmw_helptext, + * https://help.keyman.com/developer/language/reference/kmw_helpfile + */ + get helpText(): string { + return this.scriptObject['KH']; + } - /** - * Restore variable store values from cache - * - * KVS is available in keyboards compiled with Keyman Developer 15 - * and later versions. See #2924. - * - * @param values name-value pairs for each store value - */ - set variableStores(values: VariableStoreDictionary) { - const storeNames = this.scriptObject['KVS']; - if(Array.isArray(storeNames)) { - for(let store of storeNames) { - // If the value is not present in the cache, don't overwrite it; - // while this is not used in initial implementation, we could use - // it in future to update a single variable store value rather than - // the whole cache. - if(typeof values[store] == 'string') { - this.scriptObject[store] = values[store]; - } - } - } - } + /** + * Embedded JS script designed for use with a keyboard's HTML help text. Always defined + * within the file referenced by &kmw_embedjs in a keyboard's source, though that file + * may also contain _other_ script definitions as well. (`KHF` must be explicitly defined + * within that file.) + */ + get hasScript(): boolean { + return !!this.scriptObject['KHF']; + } - // TODO: Better typing. - private get _legacyLayoutSpec(): any { - return this.scriptObject['KV']; // used with buildDefaultLayout; layout must be constructed at runtime. - } + /** + * Embeds a custom script for use by the OSK, which may be interactive (like with sil_euro_latin). + * Note: this must be called AFTER any contents of `helpText` have been inserted into the DOM. + * (See sil_euro_latin's source -> sil_euro_latin_js.txt) + * + * Reference: https://help.keyman.com/developer/language/reference/kmw_embedjs + */ + embedScript(e: any) { + // e: Expects the OSKManager's _Box element. We don't add type info here b/c it would + // reference the DOM. + this.scriptObject['KHF'](e); + } - // May return null if no layouts exist or have been initialized. - private get _layouts(): {[formFactor: string]: LayoutFormFactor} { - return this.scriptObject['KVKL']; // This one is compiled by Developer's visual keyboard layout editor. - } + get oskStyling(): string { + return this.scriptObject['KCSS']; + } - private set _layouts(value) { - this.scriptObject['KVKL'] = value; + /** + * true if this keyboard uses a (legacy) pick list (Chinese, Japanese, Korean, etc.) + * + * TODO: Make a property on keyboards (say, `isPickList` / `KPL`) to signal this when we + * get around to better, generalized picker-list support. + */ + get isCJK(): boolean { // I3363 (Build 301) + var lg: string; + if(typeof(this.scriptObject['KLC']) != 'undefined') { + lg = this.scriptObject['KLC']; + } else if(typeof(this.scriptObject['LanguageCode']) != 'undefined') { + lg = this.scriptObject['LanguageCode']; } - get compilerVersion(): utils.Version { - return new utils.Version(this.scriptObject['KVER']); - } + // While some of these aren't proper BCP-47 language codes, the CJK keyboards predate our use of BCP-47. + // So, we preserve the old ISO 639-3 codes, as that's what the keyboards are matching against. + return ((lg == 'cmn') || (lg == 'jpn') || (lg == 'kor')); + } + + get isRTL(): boolean { + return !!this.scriptObject['KRTL']; + } + + /** + * Obtains the currently-active modifier bitmask for the active keyboard. + */ + get modifierBitmask(): number { + // NON_CHIRAL is the default bitmask if KMBM is not defined. + // We always need a bitmask to compare against, as seen in `isChiral`. + return this.scriptObject['KMBM'] || Codes.modifierBitmasks['NON_CHIRAL']; + } - get isMnemonic(): boolean { - return !!this.scriptObject['KM']; + get isChiral(): boolean { + return !!(this.modifierBitmask & Codes.modifierBitmasks['IS_CHIRAL']); + } + + get desktopFont(): string { + if(this.scriptObject['KV']) { + return this.scriptObject['KV']['F']; + } else { + return null; } + } - get definesPositionalOrMnemonic(): boolean { - return typeof this.scriptObject['KM'] != 'undefined'; + private get cacheTag(): CacheTag { + let tag = this.scriptObject['_kmw']; + + if(!tag) { + tag = new CacheTag(); + this.scriptObject['_kmw'] = tag; } - /** - * HTML help text, as specified by either the &kmw_helptext or &kmw_helpfile system stores. - * - * Reference: https://help.keyman.com/developer/language/reference/kmw_helptext, - * https://help.keyman.com/developer/language/reference/kmw_helpfile - */ - get helpText(): string { - return this.scriptObject['KH']; + return tag; + } + + get explodedStores(): {[storeName: string]: ComplexKeyboardStore} { + return this.cacheTag.stores; + } + + /** + * Signifies whether or not a layout or OSK should include AltGr / Right-alt emulation for this keyboard. + * @param {Object=} keyLabels + * @return {boolean} + */ + get emulatesAltGr(): boolean { + let modifierCodes = Codes.modifierCodes; + + // If we're not chiral, we're not emulating. + if(!this.isChiral) { + return false; } - /** - * Embedded JS script designed for use with a keyboard's HTML help text. Always defined - * within the file referenced by &kmw_embedjs in a keyboard's source, though that file - * may also contain _other_ script definitions as well. (`KHF` must be explicitly defined - * within that file.) - */ - get hasScript(): boolean { - return !!this.scriptObject['KHF']; + if(this._legacyLayoutSpec == null) { + return false; } - /** - * Embeds a custom script for use by the OSK, which may be interactive (like with sil_euro_latin). - * Note: this must be called AFTER any contents of `helpText` have been inserted into the DOM. - * (See sil_euro_latin's source -> sil_euro_latin_js.txt) - * - * Reference: https://help.keyman.com/developer/language/reference/kmw_embedjs - */ - embedScript(e: any) { - // e: Expects the OSKManager's _Box element. We don't add type info here b/c it would - // reference the DOM. - this.scriptObject['KHF'](e); + // Only exists in KMW 10.0+, but before that Web had no chirality support, so... return false. + let layers = this._legacyLayoutSpec['KLS']; + if(!layers) { + return false; } - get oskStyling(): string { - return this.scriptObject['KCSS']; + var emulationMask = modifierCodes['LCTRL'] | modifierCodes['LALT']; + var unshiftedEmulationLayer = layers[Layouts.getLayerId(emulationMask)]; + var shiftedEmulationLayer = layers[Layouts.getLayerId(modifierCodes['SHIFT'] | emulationMask)]; + + // buildDefaultLayout ensures that these are aliased to the original modifier set being emulated. + // As a result, we can directly test for reference equality. + // + // This allows us to still return `true` after creating the layers for emulation; during keyboard + // construction, the two layers should be null for AltGr emulation to succeed. + if(unshiftedEmulationLayer != null && + unshiftedEmulationLayer != layers[Layouts.getLayerId(modifierCodes['RALT'])]) { + return false; } - /** - * true if this keyboard uses a (legacy) pick list (Chinese, Japanese, Korean, etc.) - * - * TODO: Make a property on keyboards (say, `isPickList` / `KPL`) to signal this when we - * get around to better, generalized picker-list support. - */ - get isCJK(): boolean { // I3363 (Build 301) - var lg: string; - if(typeof(this.scriptObject['KLC']) != 'undefined') { - lg = this.scriptObject['KLC']; - } else if(typeof(this.scriptObject['LanguageCode']) != 'undefined') { - lg = this.scriptObject['LanguageCode']; - } + if(shiftedEmulationLayer != null && + shiftedEmulationLayer != layers[Layouts.getLayerId(modifierCodes['RALT'] | modifierCodes['SHIFT'])]) { + return false; + } - // While some of these aren't proper BCP-47 language codes, the CJK keyboards predate our use of BCP-47. - // So, we preserve the old ISO 639-3 codes, as that's what the keyboards are matching against. - return ((lg == 'cmn') || (lg == 'jpn') || (lg == 'kor')); + // It's technically possible for the OSK to not specify anything while allowing chiral input. A last-ditch catch: + var bitmask = this.modifierBitmask; + if((bitmask & emulationMask) != emulationMask) { + // At least one of the emulation modifiers is never used by the keyboard! We can confirm everything's safe. + return true; } - get isRTL(): boolean { - return !!this.scriptObject['KRTL']; + if(unshiftedEmulationLayer == null && shiftedEmulationLayer == null) { + // We've run out of things to go on; we can't detect if chiral AltGr emulation is intended or not. + // TODO: handle this again! + // if(!osk.altGrWarning) { + // console.warn("Could not detect if AltGr emulation is safe, but defaulting to active emulation!") + // // Avoid spamming the console with warnings on every call of the method. + // osk.altGrWarning = true; + // } + return true; } + return true; + } - /** - * Obtains the currently-active modifier bitmask for the active keyboard. - */ - get modifierBitmask(): number { - // NON_CHIRAL is the default bitmask if KMBM is not defined. - // We always need a bitmask to compare against, as seen in `isChiral`. - return this.scriptObject['KMBM'] || text.Codes.modifierBitmasks['NON_CHIRAL']; + get usesSupplementaryPlaneChars(): boolean { + let kbd = this.scriptObject; + // I3319 - SMP extension, I3363 (Build 301) + return kbd && ((kbd['KS'] && kbd['KS'] == 1) || kbd['KN'] == 'Hieroglyphic'); + } + + usesDesktopLayoutOnDevice(device: DeviceSpec) { + if(this.scriptObject['KVKL']) { + // A custom mobile layout is defined... but are we using it? + return device.formFactor == DeviceSpec.FormFactor.Desktop; + } else { + return true; } + } - get isChiral(): boolean { - return !!(this.modifierBitmask & text.Codes.modifierBitmasks['IS_CHIRAL']); + /** + * @param {number} _PCommand event code (16,17,18) or 0 + * @param {Object} _PTarget target element + * @param {number} _PData 1 or 0 + * Notifies keyboard of keystroke or other event + */ + notify(_PCommand: number, _PTarget: OutputTarget, _PData: number) { // I2187 + // Good example use case - the Japanese CJK-picker keyboard + if(typeof(this.scriptObject['KNS']) == 'function') { + this.scriptObject['KNS'](_PCommand, _PTarget, _PData); } + } - get desktopFont(): string { - if(this.scriptObject['KV']) { - return this.scriptObject['KV']['F']; - } else { - return null; + private findOrConstructLayout(formFactor: DeviceSpec.FormFactor): LayoutFormFactor { + if(this._layouts) { + // Search for viable layouts. `null` is allowed for desktop form factors when help text is available, + // so we check explicitly against `undefined`. + if(this._layouts[formFactor] !== undefined) { + return this._layouts[formFactor]; + } else if(formFactor == DeviceSpec.FormFactor.Phone && this._layouts[DeviceSpec.FormFactor.Tablet]) { + return this._layouts[DeviceSpec.FormFactor.Phone] = this._layouts[DeviceSpec.FormFactor.Tablet]; + } else if(formFactor == DeviceSpec.FormFactor.Tablet && this._layouts[DeviceSpec.FormFactor.Phone]) { + return this._layouts[DeviceSpec.FormFactor.Tablet] = this._layouts[DeviceSpec.FormFactor.Phone]; } } - private get cacheTag(): CacheTag { - let tag = this.scriptObject['_kmw']; - - if(!tag) { - tag = new CacheTag(); - this.scriptObject['_kmw'] = tag; + // No pre-built layout available; time to start constructing it via defaults. + // First, if we have non-default keys specified by the ['BK'] array, we've got + // enough to work with to build a default layout. + let rawSpecifications: any = null; // TODO: better typing, same type as this._legacyLayoutSpec. + if(this._legacyLayoutSpec != null && this._legacyLayoutSpec['KLS']) { // KLS is only specified whenever there are non-default keys. + rawSpecifications = this._legacyLayoutSpec; + } else if(this._legacyLayoutSpec != null && this._legacyLayoutSpec['BK'] != null) { + var keyCaps=this._legacyLayoutSpec['BK']; + for(var i=0; i 0) { + rawSpecifications = this._legacyLayoutSpec; + break; + } } + } - return tag; + // If we don't have key definitions to use for a layout but also lack help text or are a touch-based layout, + // we make a default layout anyway. We have to show display something usable. + if(!rawSpecifications && (this.helpText == '' || formFactor != DeviceSpec.FormFactor.Desktop)) { + rawSpecifications = {'F':'Tahoma', 'BK': Layouts.dfltText}; } - get explodedStores(): {[storeName: string]: text.ComplexKeyboardStore} { - return this.cacheTag.stores; + // Regardless of success, we'll want to initialize the field that backs the property; + // may as well cache the default layout we just built, or a 'null' if it shouldn't exist.. + if(!this._layouts) { + this._layouts = {}; } - /** - * Signifies whether or not a layout or OSK should include AltGr / Right-alt emulation for this keyboard. - * @param {Object=} keyLabels - * @return {boolean} - */ - get emulatesAltGr(): boolean { - let modifierCodes = text.Codes.modifierCodes; - - // If we're not chiral, we're not emulating. - if(!this.isChiral) { - return false; - } + // Final check - do we construct a layout, or is this a case where helpText / insertHelpHTML should take over? + if(rawSpecifications) { + // Now to generate a layout from our raw specifications. + let layout = this._layouts[formFactor] = Layouts.buildDefaultLayout(rawSpecifications, this, formFactor); + layout.isDefault = true; + return layout; + } else { + // The fact that it doesn't exist will indicate that help text/HTML should be inserted instead. + this._layouts[formFactor] = null; // provides a cached value for the check at the top of this method. + return null; + } + } - if(this._legacyLayoutSpec == null) { - return false; + /** + * Returns an ActiveLayout object representing the keyboard's layout for this form factor. May return null if a custom desktop "help" OSK is defined, as with sil_euro_latin. + * + * In such cases, please use either `helpText` or `insertHelpHTML` instead. + * @param formFactor {string} The desired form factor for the layout. + */ + public layout(formFactor: DeviceSpec.FormFactor): ActiveLayout { + let rawLayout = this.findOrConstructLayout(formFactor); + + if(rawLayout) { + // Prevents accidentally reprocessing layouts; it's a simple enough check. + if(this.layoutStates[formFactor] == LayoutState.NOT_LOADED) { + rawLayout = ActiveLayout.polyfill(rawLayout, this, formFactor); + this.layoutStates[formFactor] = LayoutState.POLYFILLED; } - // Only exists in KMW 10.0+, but before that Web had no chirality support, so... return false. - let layers = this._legacyLayoutSpec['KLS']; - if(!layers) { - return false; - } + return rawLayout as ActiveLayout; + } else { + return null; + } + } - var emulationMask = modifierCodes['LCTRL'] | modifierCodes['LALT']; - var unshiftedEmulationLayer = layers[Layouts.getLayerId(emulationMask)]; - var shiftedEmulationLayer = layers[Layouts.getLayerId(modifierCodes['SHIFT'] | emulationMask)]; - - // buildDefaultLayout ensures that these are aliased to the original modifier set being emulated. - // As a result, we can directly test for reference equality. - // - // This allows us to still return `true` after creating the layers for emulation; during keyboard - // construction, the two layers should be null for AltGr emulation to succeed. - if(unshiftedEmulationLayer != null && - unshiftedEmulationLayer != layers[Layouts.getLayerId(modifierCodes['RALT'])]) { - return false; - } + public refreshLayouts() { + let formFactors = [ DeviceSpec.FormFactor.Desktop, DeviceSpec.FormFactor.Phone, DeviceSpec.FormFactor.Tablet ]; - if(shiftedEmulationLayer != null && - shiftedEmulationLayer != layers[Layouts.getLayerId(modifierCodes['RALT'] | modifierCodes['SHIFT'])]) { - return false; - } + let _this = this; - // It's technically possible for the OSK to not specify anything while allowing chiral input. A last-ditch catch: - var bitmask = this.modifierBitmask; - if((bitmask & emulationMask) != emulationMask) { - // At least one of the emulation modifiers is never used by the keyboard! We can confirm everything's safe. - return true; - } + formFactors.forEach(function(form) { + // Currently doesn't work if we reset it to POLYFILLED, likely due to how 'calibration' + // currently works. + _this.layoutStates[form] = LayoutState.NOT_LOADED; + }); + } - if(unshiftedEmulationLayer == null && shiftedEmulationLayer == null) { - // We've run out of things to go on; we can't detect if chiral AltGr emulation is intended or not. - // TODO: handle this again! - // if(!osk.altGrWarning) { - // console.warn("Could not detect if AltGr emulation is safe, but defaulting to active emulation!") - // // Avoid spamming the console with warnings on every call of the method. - // osk.altGrWarning = true; - // } - return true; - } - return true; + public markLayoutCalibrated(formFactor: DeviceSpec.FormFactor) { + if(this.layoutStates[formFactor] != LayoutState.NOT_LOADED) { + this.layoutStates[formFactor] = LayoutState.CALIBRATED; } + } - get usesSupplementaryPlaneChars(): boolean { - let kbd = this.scriptObject; - // I3319 - SMP extension, I3363 (Build 301) - return kbd && ((kbd['KS'] && kbd['KS'] == 1) || kbd['KN'] == 'Hieroglyphic'); - } + public getLayoutState(formFactor: DeviceSpec.FormFactor) { + return this.layoutStates[formFactor]; + } - usesDesktopLayoutOnDevice(device: utils.DeviceSpec) { - if(this.scriptObject['KVKL']) { - // A custom mobile layout is defined... but are we using it? - return device.formFactor == utils.FormFactor.Desktop; - } else { - return true; - } - } - /** - * @param {number} _PCommand event code (16,17,18) or 0 - * @param {Object} _PTarget target element - * @param {number} _PData 1 or 0 - * Notifies keyboard of keystroke or other event - */ - notify(_PCommand: number, _PTarget: text.OutputTarget, _PData: number) { // I2187 - // Good example use case - the Japanese CJK-picker keyboard - if(typeof(this.scriptObject['KNS']) == 'function') { - this.scriptObject['KNS'](_PCommand, _PTarget, _PData); - } + constructNullKeyEvent(device: DeviceSpec, stateKeys?: StateKeyMap): KeyEvent { + stateKeys = stateKeys || { + K_CAPS: false, + K_NUMLOCK: false, + K_SCROLL: false } - private findOrConstructLayout(formFactor: utils.FormFactor): LayoutFormFactor { - if(this._layouts) { - // Search for viable layouts. `null` is allowed for desktop form factors when help text is available, - // so we check explicitly against `undefined`. - if(this._layouts[formFactor] !== undefined) { - return this._layouts[formFactor]; - } else if(formFactor == utils.FormFactor.Phone && this._layouts[utils.FormFactor.Tablet]) { - return this._layouts[utils.FormFactor.Phone] = this._layouts[utils.FormFactor.Tablet]; - } else if(formFactor == utils.FormFactor.Tablet && this._layouts[utils.FormFactor.Phone]) { - return this._layouts[utils.FormFactor.Tablet] = this._layouts[utils.FormFactor.Phone]; - } - } + const keyEvent = KeyEvent.constructNullKeyEvent(device); + this.setSyntheticEventDefaults(keyEvent, stateKeys); + return keyEvent; + } - // No pre-built layout available; time to start constructing it via defaults. - // First, if we have non-default keys specified by the ['BK'] array, we've got - // enough to work with to build a default layout. - let rawSpecifications: any = null; // TODO: better typing, same type as this._legacyLayoutSpec. - if(this._legacyLayoutSpec != null && this._legacyLayoutSpec['KLS']) { // KLS is only specified whenever there are non-default keys. - rawSpecifications = this._legacyLayoutSpec; - } else if(this._legacyLayoutSpec != null && this._legacyLayoutSpec['BK'] != null) { - var keyCaps=this._legacyLayoutSpec['BK']; - for(var i=0; i 0) { - rawSpecifications = this._legacyLayoutSpec; - break; - } - } - } + constructKeyEvent(key: ActiveKey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent { + // Make a deep copy of our preconstructed key event, filling it out from there. + const Lkc = key.baseKeyEvent; + Lkc.device = device; - // If we don't have key definitions to use for a layout but also lack help text or are a touch-based layout, - // we make a default layout anyway. We have to show display something usable. - if(!rawSpecifications && (this.helpText == '' || formFactor != utils.FormFactor.Desktop)) { - rawSpecifications = {'F':'Tahoma', 'BK': Layouts.dfltText}; - } + if(this.isMnemonic) { + Lkc.setMnemonicCode(key.layer.indexOf('shift') != -1, stateKeys['K_CAPS']); + } - // Regardless of success, we'll want to initialize the field that backs the property; - // may as well cache the default layout we just built, or a 'null' if it shouldn't exist.. - if(!this._layouts) { - this._layouts = {}; - } + // Performs common pre-analysis for both 'native' and 'embedded' OSK key & subkey input events. + // This part depends on the keyboard processor's active state. + this.setSyntheticEventDefaults(Lkc, stateKeys); - // Final check - do we construct a layout, or is this a case where helpText / insertHelpHTML should take over? - if(rawSpecifications) { - // Now to generate a layout from our raw specifications. - let layout = this._layouts[formFactor] = Layouts.buildDefaultLayout(rawSpecifications, this, formFactor); - layout.isDefault = true; - return layout; - } else { - // The fact that it doesn't exist will indicate that help text/HTML should be inserted instead. - this._layouts[formFactor] = null; // provides a cached value for the check at the top of this method. - return null; - } + // If it's a state key modifier, trigger its effects as part of the + // keystroke. + const bitmap = { + 'K_CAPS': Codes.stateBitmasks.CAPS, + 'K_NUMLOCK': Codes.stateBitmasks.NUM_LOCK, + 'K_SCROLL': Codes.stateBitmasks.SCROLL_LOCK + }; + const bitmask = bitmap[Lkc.kName]; + + if(bitmask) { + Lkc.Lstates ^= bitmask; + Lkc.LmodifierChange = true; } - /** - * Returns an ActiveLayout object representing the keyboard's layout for this form factor. May return null if a custom desktop "help" OSK is defined, as with sil_euro_latin. - * - * In such cases, please use either `helpText` or `insertHelpHTML` instead. - * @param formFactor {string} The desired form factor for the layout. - */ - public layout(formFactor: utils.FormFactor): ActiveLayout { - let rawLayout = this.findOrConstructLayout(formFactor); - - if(rawLayout) { - // Prevents accidentally reprocessing layouts; it's a simple enough check. - if(this.layoutStates[formFactor] == LayoutState.NOT_LOADED) { - rawLayout = ActiveLayout.polyfill(rawLayout, this, formFactor); - this.layoutStates[formFactor] = LayoutState.POLYFILLED; - } + return Lkc; + } - return rawLayout as ActiveLayout; - } else { - return null; + setSyntheticEventDefaults(Lkc: KeyEvent, stateKeys: StateKeyMap) { + // Set the flags for the state keys - for desktop devices. For touch + // devices, the only state key in use currently is Caps Lock, which is set + // when the 'caps' layer is active in ActiveKey::constructBaseKeyEvent. + if(!Lkc.device.touchable) { + /* + * For desktop-style keyboards, start from a blank slate. They have a 'default' + * (implicit 'NO_CAPS') layer but not a 'caps' layer. With caps set, it just + * highlights the key on the 'default' layer instead. + * + * We should never set both `CAPS` and `NO_CAPS` at the same time, and + * same for the other modifiers. + */ + Lkc.Lstates = 0; + Lkc.Lstates |= stateKeys['K_CAPS'] ? Codes.modifierCodes['CAPS'] : Codes.modifierCodes['NO_CAPS']; + Lkc.Lstates |= stateKeys['K_NUMLOCK'] ? Codes.modifierCodes['NUM_LOCK'] : Codes.modifierCodes['NO_NUM_LOCK']; + Lkc.Lstates |= stateKeys['K_SCROLL'] ? Codes.modifierCodes['SCROLL_LOCK'] : Codes.modifierCodes['NO_SCROLL_LOCK']; + } + + // Set LisVirtualKey to false to ensure that nomatch rule does fire for U_xxxx keys + if(Lkc.kName && Lkc.kName.substr(0,2) == 'U_') { + Lkc.LisVirtualKey=false; + } + + // Get code for non-physical keys (T_KOKAI, U_05AB etc) + if(typeof Lkc.Lcode == 'undefined') { + Lkc.Lcode = this.getVKDictionaryCode(Lkc.kName);// Updated for Build 347 + if(!Lkc.Lcode) { + // Special case for U_xxxx keys. This vk code will never be used + // in a keyboard, so we use this to ensure that keystroke processing + // occurs for the key. + Lkc.Lcode = 1; } } - public refreshLayouts() { - let formFactors = [ utils.FormFactor.Desktop, utils.FormFactor.Phone, utils.FormFactor.Tablet ]; - - let _this = this; - - formFactors.forEach(function(form) { - // Currently doesn't work if we reset it to POLYFILLED, likely due to how 'calibration' - // currently works. - _this.layoutStates[form] = LayoutState.NOT_LOADED; - }); + // Handles modifier states when the OSK is emulating rightalt through the leftctrl-leftalt layer. + if((Lkc.Lmodifiers & Codes.modifierBitmasks['ALT_GR_SIM']) == Codes.modifierBitmasks['ALT_GR_SIM'] && this.emulatesAltGr) { + Lkc.Lmodifiers &= ~Codes.modifierBitmasks['ALT_GR_SIM']; + Lkc.Lmodifiers |= Codes.modifierCodes['RALT']; } + } - public markLayoutCalibrated(formFactor: utils.FormFactor) { - if(this.layoutStates[formFactor] != LayoutState.NOT_LOADED) { - this.layoutStates[formFactor] = LayoutState.CALIBRATED; + /** + * @summary Look up a custom virtual key code in the virtual key code dictionary KVKD. + * On first run, will build the dictionary. + * + * `VKDictionary` is constructed from the keyboard's `KVKD` member. This list is constructed + * at compile-time and is a list of 'additional' virtual key codes, starting at 256 (i.e. + * outside the range of standard virtual key codes). These additional codes are both + * `[T_xxx]` and `[U_xxxx]` custom key codes from the Keyman keyboard language. However, + * `[U_xxxx]` keys only generate an entry in `KVKD` if there is a corresponding rule that + * is associated with them in the keyboard rules. If the `[U_xxxx]` key code is only + * referenced as the id of a key in the touch layout, then it does not get an entry in + * the `KVKD` property. + * + * @private + * @param {string} keyName custom virtual key code to lookup in the dictionary + * @return {number} key code > 255 on success, or 0 if not found + */ + getVKDictionaryCode(keyName: string) { + if(!this.scriptObject['VKDictionary']) { + const a=[]; + if(typeof this.scriptObject['KVKD'] == 'string') { + // Build the VK dictionary + // TODO: Move the dictionary build into the compiler -- so compiler generates code such as following. + // Makes the VKDictionary member unnecessary. + // this.KVKD={"K_ABC":256,"K_DEF":257,...}; + const s=this.scriptObject['KVKD'].split(' '); + for(var i=0; i { + this.harness.install(); + const promise = this.loadKeyboardInternal(uri, new UriBasedErrorBuilder(uri)); + + return promise; + } + + public loadKeyboardFromStub(stub: KeyboardStub) { + this.harness.install(); + let promise = this.loadKeyboardInternal(stub.filename, new StubBasedErrorBuilder(stub), stub.id); + + return promise; + } + + protected abstract loadKeyboardInternal( + uri: string, + errorBuilder: KeyboardLoadErrorBuilder, + id?: string + ): Promise; +} \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/keyboardProperties.ts b/common/web/keyboard-processor/src/keyboards/keyboardProperties.ts new file mode 100644 index 0000000000000000000000000000000000000000..e83d1a137f192aa5460dce6531a9103a5d22e4e1 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/keyboardProperties.ts @@ -0,0 +1,264 @@ +import SpacebarText from './spacebarText.js'; + +export interface InternalKeyboardFont { + family: string; + filename?: never; + files: string | string[]; // internal + source?: never; + path: string; +} + +interface CloudKeyboardFont1 { + family: string; + filename: string | string[]; + files?: never; + source?: never; +} + +interface CloudKeyboardFont2 { + family: string; + filename?: never; + files?: never; + source: string | string[]; +} + +export type CloudKeyboardFont = CloudKeyboardFont1 | CloudKeyboardFont2; + +/** + * Converts one of three public-facing font-specification formats into a consistent structure + * used generally among the Keyman JS/TS modules. + * @param fontObj + * @param fontPath + * @returns + */ +export function internalizeFont(fontObj: CloudKeyboardFont, fontPath: string): InternalKeyboardFont { + if(!fontObj) { + return undefined; + } else { + return { + family: fontObj.family, + path: fontPath, + files: fontObj.filename || fontObj.source + } + } +} + +export type KeyboardFont = CloudKeyboardFont | InternalKeyboardFont; + +// Filename properties are deliberately omitted here; we can add that at higher-levels where it matters +// via 'mix-in'. +// +// For example, the OSK module doesn't care about the filename of a loaded keyboard. It doesn't do +// keyboard loading on its own whatsoever. + +// Corresponds to Keyman Engine for Web's internal "keyboard stub" format. +// Also referred to by KMW 2.0-era loaders: https://help.keyman.com/developer/8.0/docs/reference_kmw20_example +export interface KeyboardInternalPropertySpec { + KI: string, + KFont: InternalKeyboardFont, + KOskFont: InternalKeyboardFont, + displayName?: string, + KN?: string, + KL?: string, + KLC?: string +}; + +export type LanguageAPIPropertySpec = { + id: string, + name: string, + font: CloudKeyboardFont, + oskFont: CloudKeyboardFont, + region?: number|string +} + +/** + * Corresponds to the documented API for the Web engine's `addKeyboards` function + * when a single language object is specified - not an array. + * + * See https://help.keyman.com/DEVELOPER/ENGINE/WEB/15.0/reference/core/addKeyboards, + * "Using an `object`". + */ +export type KeyboardAPIPropertySpec = { + id: string, + name: string, + + /** + * @deprecated Replaced with `languages`. + */ + language?: LanguageAPIPropertySpec; + languages: LanguageAPIPropertySpec; +} + +/** + * Corresponds to the documented API for the Web engine's `addKeyboards` function + * when a language array is specified for the object. + * + * See https://help.keyman.com/DEVELOPER/ENGINE/WEB/15.0/reference/core/addKeyboards, + * "Using an `object`". + */ +export type KeyboardAPIPropertyMultilangSpec = { + id: string, + name: string, + + /** + * @deprecated Replaced with `languages`. + */ + language?: LanguageAPIPropertySpec[]; + languages: LanguageAPIPropertySpec[]; +} + +export type MetadataObj = KeyboardInternalPropertySpec | KeyboardAPIPropertySpec | KeyboardAPIPropertyMultilangSpec; + +export default class KeyboardProperties implements KeyboardInternalPropertySpec { + KI: string; + KN: string; + KL: string; + KLC: string; + KFont: InternalKeyboardFont; + KOskFont: InternalKeyboardFont; + _displayName?: string; + + private static spacebarTextModeSrc: SpacebarText | (() => SpacebarText) = SpacebarText.KEYBOARD; + + public static get spacebarTextMode(): SpacebarText { + if(typeof this.spacebarTextModeSrc == 'string') { + return this.spacebarTextModeSrc; + } else { + return this.spacebarTextModeSrc(); + } + } + + public static set spacebarTextMode(source: typeof KeyboardProperties.spacebarTextModeSrc) { + this.spacebarTextModeSrc = source; + } + + public constructor(metadataObj: MetadataObj, fontPath?: string); + public constructor(keyboardId: string, languageCode: string); + public constructor(arg1: MetadataObj | string, arg2?: string) { + if(!(typeof arg1 == 'string')) { + if(arg1['KI'] || arg1['KL'] || arg1['KLC'] || arg1['KFont'] || arg1['KOskFont']) { + const other = arg1 as KeyboardInternalPropertySpec; + this.KI = other.KI; + this.KN = other.KN; + this.KL = other.KL; + this.KLC = other.KLC; + // Do NOT apply fontPath here; the mobile apps will have font issues if you do! + this.KFont = other.KFont; + this.KOskFont = other.KOskFont; + this._displayName = (other instanceof KeyboardProperties) ? other._displayName : other.displayName; + } else { + let apiStub = arg1 as KeyboardAPIPropertySpec; // TODO: could be an array, as currently specified. :( + + apiStub.languages ||= apiStub.language; + + this.KI = apiStub.id, + this.KN = apiStub.name, + this.KL = apiStub.languages.name, + this.KLC = apiStub.languages.id, + this.KFont = internalizeFont(apiStub.languages.font, arg2), + this.KOskFont = internalizeFont(apiStub.languages.oskFont, arg2) + } + } else { + this.KI = arg1; + this.KLC = arg2; + } + } + + public static fromMultilanguageAPIStub(apiStub: KeyboardAPIPropertyMultilangSpec): KeyboardProperties[] { + let stubs: KeyboardProperties[] = []; + + apiStub.languages ||= apiStub.language; + + for(let langSpec of apiStub.languages) { + let stub: KeyboardAPIPropertySpec = { + id: apiStub.id, + name: apiStub.name, + languages: langSpec + }; + + stubs.push(new KeyboardProperties(stub)); + } + + return stubs; + } + + public get id(): string { + return this.KI; + } + + public get name(): string { + return this.KN; + } + + public get langId(): string { + return this.KLC; + } + + public get langName(): string { + return this.KL; + } + + public get displayName(): string { + if(this._displayName) { + return this._displayName; + } + + // else, construct it. + const kbdName = this.KN; + const lgName = this.KL; + + switch (KeyboardProperties.spacebarTextMode) { + case SpacebarText.KEYBOARD: + return kbdName; + case SpacebarText.LANGUAGE: + return lgName; + case SpacebarText.LANGUAGE_KEYBOARD: + return (kbdName == lgName) ? lgName : lgName + ' - ' + kbdName; + case SpacebarText.BLANK: + return ''; + default: + return kbdName; + } + } + + public set displayName(name: string) { + this._displayName = name; + } + + public get textFont() { + return this.KFont; + } + + public get oskFont() { + return this.KOskFont; + } + + /** + * Generates an error for objects with specification levels insufficient for use in the on-screen-keyboard + * module, complete with a message about one or more details in need of correction. + * @returns A preconstructed `Error` instance that may be thrown by the caller. + */ + public validateForOSK(): Error { + if(!this.KLC) { + if(this.KI || this.KN) { + return new Error(`No language code was specified for use with the ${this.KI || this.KN} keyboard`); + } else { + return new Error("No language code was specified for use with the corresponding keyboard") + } + } + + if(this.displayName === undefined || (KeyboardProperties.spacebarTextMode != SpacebarText.BLANK && !this.displayName)) { + return new Error("A display name is missing for this keyboard and cannot be generated under current settings.") + } + + return null; + } + + public validateForCustomKeyboard(): Error { + if(!this.KI || !this.KN || !this.KL || !this.KLC) { + return new Error("To use a custom keyboard, you must specify keyboard id, keyboard name, language and language code."); + } else { + return null; + } + } +} \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/loaders/dom-keyboard-loader.ts b/common/web/keyboard-processor/src/keyboards/loaders/dom-keyboard-loader.ts new file mode 100644 index 0000000000000000000000000000000000000000..989d5b647774819f31926dccfbb23f72740c2c2c --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/dom-keyboard-loader.ts @@ -0,0 +1 @@ +export * from './domKeyboardLoader.js'; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/loaders/domKeyboardLoader.ts b/common/web/keyboard-processor/src/keyboards/loaders/domKeyboardLoader.ts new file mode 100644 index 0000000000000000000000000000000000000000..84ee1f0fb19bc412684ae3f5bc5e912f11fc1187 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/domKeyboardLoader.ts @@ -0,0 +1,87 @@ +// Enables DOM types, but just for this one module. + +/// + +import { Keyboard, KeyboardHarness, KeyboardLoaderBase, KeyboardLoadErrorBuilder, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; + +import { ManagedPromise } from '@keymanapp/web-utils'; + +export class DOMKeyboardLoader extends KeyboardLoaderBase { + public readonly element: HTMLIFrameElement; + private readonly performCacheBusting: boolean; + + constructor() + constructor(harness: KeyboardHarness); + constructor(harness: KeyboardHarness, cacheBust?: boolean) + constructor(harness?: KeyboardHarness, cacheBust?: boolean) { + if(harness && harness._jsGlobal != window) { + // Copy the String typing over; preserve string extensions! + harness._jsGlobal['String'] = window['String']; + } + + if(!harness) { + super(new KeyboardHarness(window, MinimalKeymanGlobal)); + } else { + super(harness); + } + + this.performCacheBusting = cacheBust || false; + } + + protected loadKeyboardInternal( + uri: string, + errorBuilder: KeyboardLoadErrorBuilder, + id?: string + ): Promise { + const promise = new ManagedPromise(); + + if(this.performCacheBusting) { + uri = this.cacheBust(uri); + } + + try { + const document = this.harness._jsGlobal.document; + const script = document.createElement('script'); + if(id) { + script.id = id; + } + document.head.appendChild(script); + script.onerror = (err) => { + promise.reject(errorBuilder.missingError(err)); + } + script.onload = () => { + if(this.harness.loadedKeyboard) { + const keyboard = this.harness.loadedKeyboard; + this.harness.loadedKeyboard = null; + promise.resolve(keyboard); + } else { + promise.reject(errorBuilder.scriptError()); + } + } + + // On the oldest mobile devices we support, Promise.finally may not actually exist. + // Fortunately... it's not that hard of an issue to work around. + // Note: es6-shim doesn't polyfill Promise.finally! + promise.then(() => { + // It is safe to remove the script once it has been run (https://stackoverflow.com/a/37393041) + script.remove(); + }).catch(() => { + script.remove(); + }); + + // Now that EVERYTHING ELSE is ready, establish the link to the keyboard's script. + script.src = uri; + } catch (err) { + return Promise.reject(err); + } + + return promise.corePromise; + } + + private cacheBust(uri: string) { + // Our WebView version directly sets the keyboard path, and it may replace the file + // after KMW has loaded. We need cache-busting to prevent the new version from + // being ignored. + return uri + "?v=" + (new Date()).getTime(); /*cache buster*/ + } +} \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/loaders/node-keyboard-loader.ts b/common/web/keyboard-processor/src/keyboards/loaders/node-keyboard-loader.ts new file mode 100644 index 0000000000000000000000000000000000000000..9630e113b0144d4d60ad512369bc34c25c8ae835 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/node-keyboard-loader.ts @@ -0,0 +1 @@ +export * from './nodeKeyboardLoader.js'; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/loaders/nodeKeyboardLoader.ts b/common/web/keyboard-processor/src/keyboards/loaders/nodeKeyboardLoader.ts new file mode 100644 index 0000000000000000000000000000000000000000..143e1c16c1121999b7b63fa1835cd901a853454a --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/nodeKeyboardLoader.ts @@ -0,0 +1,47 @@ +import { Keyboard, KeyboardHarness, KeyboardLoaderBase, KeyboardLoadErrorBuilder, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; + +import vm from 'vm'; +import fs from 'fs'; +import { globalObject } from '@keymanapp/web-utils'; + +export class NodeKeyboardLoader extends KeyboardLoaderBase { + constructor() + constructor(harness: KeyboardHarness); + constructor(harness?: KeyboardHarness) { + if(!harness) { + super(new KeyboardHarness(vm.createContext(), MinimalKeymanGlobal)); + } else { + // If we're going to sandbox, make sure the sandbox is sufficient! + if(globalObject() != harness._jsGlobal) { + vm.createContext(harness._jsGlobal); + // Ensure any and all string extensions are available within the sandbox context! + harness._jsGlobal.String = globalObject().String; + } + + super(harness); + } + } + + protected loadKeyboardInternal(uri: string, errorBuilder: KeyboardLoadErrorBuilder): Promise { + // `fs` does not like 'file:///'; it IS "File System" oriented, after all, and wants a path, not a URI. + if(uri.indexOf('file:///') == 0) { + uri = uri.substring('file:///'.length); + } + + let script; + try { + script = new vm.Script(fs.readFileSync(uri).toString()); + } catch (err) { + return Promise.reject(errorBuilder.missingError(err)); + } + try { + script.runInContext(this.harness._jsGlobal); + } catch (err) { + return Promise.reject(errorBuilder.scriptError(err)); + } + + const keyboard = this.harness.loadedKeyboard; + this.harness.loadedKeyboard = null; + return Promise.resolve(keyboard); + } +} \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.dom.json b/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.dom.json new file mode 100644 index 0000000000000000000000000000000000000000..e5ab52729e78c9ce3a5b54fd4e3e920e1e37f5e6 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.dom.json @@ -0,0 +1,26 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman/", + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "experimentalDecorators": true, + "baseUrl": "../../../", + "outDir": "../../../build/obj/keyboards/loaders/", + "tsBuildInfoFile": "../../../build/obj/keyboards/loaders/tsconfig.dom.tsbuildinfo", + "rootDir": "." + }, + "references": [ + { "path": "../../../tsconfig.common.json" } + ], + "include": ["dom-keyboard-loader.ts", "domKeyboardLoader.ts"], +} diff --git a/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.node.json b/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.node.json new file mode 100644 index 0000000000000000000000000000000000000000..7175be8333e2e5b59c61180b75917c8e24283cb4 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/loaders/tsconfig.node.json @@ -0,0 +1,25 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman/", + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "experimentalDecorators": true, + "baseUrl": "../../../", + "outDir": "../../../build/obj/keyboards/loaders/", + "tsBuildInfoFile": "../../../build/obj/keyboards/loaders/tsconfig.node.tsbuildinfo", + "rootDir": "." + }, + "references": [ + { "path": "../../../tsconfig.common.json" } + ], + "include": ["node-keyboard-loader.ts", "nodeKeyboardLoader.ts"], +} diff --git a/common/web/keyboard-processor/src/keyboards/spacebarText.ts b/common/web/keyboard-processor/src/keyboards/spacebarText.ts new file mode 100644 index 0000000000000000000000000000000000000000..d98e7b6462d4481302f678806418ecd9d9bfee1e --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/spacebarText.ts @@ -0,0 +1,9 @@ +// Compiles completely out if `const enum`, making it unavailable in JS-based unit tests. +enum SpacebarText { + KEYBOARD = 'keyboard', + LANGUAGE = 'language', + LANGUAGE_KEYBOARD = 'languageKeyboard', + BLANK = 'blank' +}; + +export default SpacebarText; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/stateKeyMap.ts b/common/web/keyboard-processor/src/keyboards/stateKeyMap.ts new file mode 100644 index 0000000000000000000000000000000000000000..c8e9ee8b7634d66e84fc113ca0ea5e7a66a52192 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/stateKeyMap.ts @@ -0,0 +1,9 @@ +/** + * Provided by @keymanapp/keyboard-processor's `KeyboardProcessor` class and utilized by the OSK + * to provide state feedback on any corresponding keys visible in the OSK. + */ +export default interface StateKeyMap { + K_CAPS: boolean, + K_NUMLOCK: boolean, + K_SCROLL: boolean +} \ No newline at end of file diff --git a/common/web/keyboard-processor/src/keyboards/test.ts b/common/web/keyboard-processor/src/keyboards/test.ts new file mode 100644 index 0000000000000000000000000000000000000000..65d3beae0c9be5ebe7781a4aade2207a111bab97 --- /dev/null +++ b/common/web/keyboard-processor/src/keyboards/test.ts @@ -0,0 +1,5 @@ +// This specific module is allowed to reference the DOM in order to facilitate loading the keyboards +// into the DOM. + +/// + diff --git a/common/web/keyboard-processor/src/text/codes.ts b/common/web/keyboard-processor/src/text/codes.ts index 48fde0d728ccac9eff3f8aac6c6c3192ee8aecd7..5bbd26f51276b9e5e38f9ebf9c94307721568d19 100644 --- a/common/web/keyboard-processor/src/text/codes.ts +++ b/common/web/keyboard-processor/src/text/codes.ts @@ -1,103 +1,169 @@ -namespace com.keyman.text { - export var Codes = { - // Define Keyman Developer modifier bit-flags (exposed for use by other modules) - // Compare against /common/include/kmx_file.h. CTRL+F "#define LCTRLFLAG" to find the secton. - modifierCodes: { - "LCTRL":0x0001, // LCTRLFLAG - "RCTRL":0x0002, // RCTRLFLAG - "LALT":0x0004, // LALTFLAG - "RALT":0x0008, // RALTFLAG - "SHIFT":0x0010, // K_SHIFTFLAG - "CTRL":0x0020, // K_CTRLFLAG - "ALT":0x0040, // K_ALTFLAG - // TENTATIVE: Represents command keys, which some OSes use for shortcuts we don't - // want to block. No rule will ever target a modifier set with this bit set to 1. - "META":0x0080, // K_METAFLAG - "CAPS":0x0100, // CAPITALFLAG - "NO_CAPS":0x0200, // NOTCAPITALFLAG - "NUM_LOCK":0x0400, // NUMLOCKFLAG - "NO_NUM_LOCK":0x0800, // NOTNUMLOCKFLAG - "SCROLL_LOCK":0x1000, // SCROLLFLAG - "NO_SCROLL_LOCK":0x2000, // NOTSCROLLFLAG - "VIRTUAL_KEY":0x4000, // ISVIRTUALKEY - "VIRTUAL_CHAR_KEY":0x8000 // VIRTUALCHARKEY // Unused by KMW, but reserved for use by other Keyman engines. - }, - - modifierBitmasks: { - "ALL":0x007F, - "ALT_GR_SIM": (0x0001 | 0x0004), - "CHIRAL":0x001F, // The base bitmask for chiral keyboards. Includes SHIFT, which is non-chiral. - "IS_CHIRAL":0x000F, // Used to test if a bitmask uses a chiral modifier. - "NON_CHIRAL":0x0070 // The default bitmask, for non-chiral keyboards - }, - - stateBitmasks: { - "ALL":0x3F00, - "CAPS":0x0300, - "NUM_LOCK":0x0C00, - "SCROLL_LOCK":0x3000 - }, - - // Define standard keycode numbers (exposed for use by other modules) - keyCodes: { - "K_BKSP":8,"K_TAB":9,"K_ENTER":13, - "K_SHIFT":16,"K_CONTROL":17,"K_ALT":18,"K_PAUSE":19,"K_CAPS":20, - "K_ESC":27,"K_SPACE":32,"K_PGUP":33, - "K_PGDN":34,"K_END":35,"K_HOME":36,"K_LEFT":37,"K_UP":38, - "K_RIGHT":39,"K_DOWN":40,"K_SEL":41,"K_PRINT":42,"K_EXEC":43, - "K_INS":45,"K_DEL":46,"K_HELP":47,"K_0":48, - "K_1":49,"K_2":50,"K_3":51,"K_4":52,"K_5":53,"K_6":54,"K_7":55, - "K_8":56,"K_9":57,"K_A":65,"K_B":66,"K_C":67,"K_D":68,"K_E":69, - "K_F":70,"K_G":71,"K_H":72,"K_I":73,"K_J":74,"K_K":75,"K_L":76, - "K_M":77,"K_N":78,"K_O":79,"K_P":80,"K_Q":81,"K_R":82,"K_S":83, - "K_T":84,"K_U":85,"K_V":86,"K_W":87,"K_X":88,"K_Y":89,"K_Z":90, - "K_NP0":96,"K_NP1":97,"K_NP2":98, - "K_NP3":99,"K_NP4":100,"K_NP5":101,"K_NP6":102, - "K_NP7":103,"K_NP8":104,"K_NP9":105,"K_NPSTAR":106, - "K_NPPLUS":107,"K_SEPARATOR":108,"K_NPMINUS":109,"K_NPDOT":110, - "K_NPSLASH":111,"K_F1":112,"K_F2":113,"K_F3":114,"K_F4":115, - "K_F5":116,"K_F6":117,"K_F7":118,"K_F8":119,"K_F9":120, - "K_F10":121,"K_F11":122,"K_F12":123,"K_NUMLOCK":144,"K_SCROLL":145, - "K_LSHIFT":160,"K_RSHIFT":161,"K_LCONTROL":162,"K_RCONTROL":163, - "K_LALT":164,"K_RALT":165, - "K_COLON":186,"K_EQUAL":187,"K_COMMA":188,"K_HYPHEN":189, - "K_PERIOD":190,"K_SLASH":191,"K_BKQUOTE":192, - "K_LBRKT":219,"K_BKSLASH":220,"K_RBRKT":221, - "K_QUOTE":222,"K_oE2":226,"K_OE2":226, - "K_LOPT":50001,"K_ROPT":50002, - "K_NUMERALS":50003,"K_SYMBOLS":50004,"K_CURRENCIES":50005, - "K_UPPER":50006,"K_LOWER":50007,"K_ALPHA":50008, - "K_SHIFTED":50009,"K_ALTGR":50010, - "K_TABBACK":50011,"K_TABFWD":50012 - }, - - codesUS: [ - ['0123456789',';=,-./`', '[\\]\''], - [')!@#$%^&*(',':+<_>?~', '{|}"'] - ], - - isKnownOSKModifierKey(keyID: string): boolean { - switch(keyID) { - case 'K_SHIFT': - case 'K_LOPT': - case 'K_ROPT': - case 'K_NUMLOCK': // Often used for numeric layers. - case 'K_CAPS': +// TODO: Move to separate folder: 'codes' +// We should start splitting off code needed by keyboards even without a KeyboardProcessor active. +// There's an upcoming `/common/web/types` package that 'codes' and 'keyboards' may fit well within. +// In fact, there's a file there (on its branch) that should be merged with this one! + +const Codes = { + // Define Keyman Developer modifier bit-flags (exposed for use by other modules) + // Compare against /common/include/kmx_file.h. CTRL+F "#define LCTRLFLAG" to find the secton. + modifierCodes: { + "LCTRL":0x0001, // LCTRLFLAG + "RCTRL":0x0002, // RCTRLFLAG + "LALT":0x0004, // LALTFLAG + "RALT":0x0008, // RALTFLAG + "SHIFT":0x0010, // K_SHIFTFLAG + "CTRL":0x0020, // K_CTRLFLAG + "ALT":0x0040, // K_ALTFLAG + // TENTATIVE: Represents command keys, which some OSes use for shortcuts we don't + // want to block. No rule will ever target a modifier set with this bit set to 1. + "META":0x0080, // K_METAFLAG + "CAPS":0x0100, // CAPITALFLAG + "NO_CAPS":0x0200, // NOTCAPITALFLAG + "NUM_LOCK":0x0400, // NUMLOCKFLAG + "NO_NUM_LOCK":0x0800, // NOTNUMLOCKFLAG + "SCROLL_LOCK":0x1000, // SCROLLFLAG + "NO_SCROLL_LOCK":0x2000, // NOTSCROLLFLAG + "VIRTUAL_KEY":0x4000, // ISVIRTUALKEY + "VIRTUAL_CHAR_KEY":0x8000 // VIRTUALCHARKEY // Unused by KMW, but reserved for use by other Keyman engines. + }, + + modifierBitmasks: { + "ALL":0x007F, + "ALT_GR_SIM": (0x0001 | 0x0004), + "CHIRAL":0x001F, // The base bitmask for chiral keyboards. Includes SHIFT, which is non-chiral. + "IS_CHIRAL":0x000F, // Used to test if a bitmask uses a chiral modifier. + "NON_CHIRAL":0x0070 // The default bitmask, for non-chiral keyboards + }, + + stateBitmasks: { + "ALL":0x3F00, + "CAPS":0x0300, + "NUM_LOCK":0x0C00, + "SCROLL_LOCK":0x3000 + }, + + // Define standard keycode numbers (exposed for use by other modules) + keyCodes: { + "K_BKSP":8,"K_TAB":9,"K_ENTER":13, + "K_SHIFT":16,"K_CONTROL":17,"K_ALT":18,"K_PAUSE":19,"K_CAPS":20, + "K_ESC":27,"K_SPACE":32,"K_PGUP":33, + "K_PGDN":34,"K_END":35,"K_HOME":36,"K_LEFT":37,"K_UP":38, + "K_RIGHT":39,"K_DOWN":40,"K_SEL":41,"K_PRINT":42,"K_EXEC":43, + "K_INS":45,"K_DEL":46,"K_HELP":47,"K_0":48, + "K_1":49,"K_2":50,"K_3":51,"K_4":52,"K_5":53,"K_6":54,"K_7":55, + "K_8":56,"K_9":57,"K_A":65,"K_B":66,"K_C":67,"K_D":68,"K_E":69, + "K_F":70,"K_G":71,"K_H":72,"K_I":73,"K_J":74,"K_K":75,"K_L":76, + "K_M":77,"K_N":78,"K_O":79,"K_P":80,"K_Q":81,"K_R":82,"K_S":83, + "K_T":84,"K_U":85,"K_V":86,"K_W":87,"K_X":88,"K_Y":89,"K_Z":90, + "K_NP0":96,"K_NP1":97,"K_NP2":98, + "K_NP3":99,"K_NP4":100,"K_NP5":101,"K_NP6":102, + "K_NP7":103,"K_NP8":104,"K_NP9":105,"K_NPSTAR":106, + "K_NPPLUS":107,"K_SEPARATOR":108,"K_NPMINUS":109,"K_NPDOT":110, + "K_NPSLASH":111,"K_F1":112,"K_F2":113,"K_F3":114,"K_F4":115, + "K_F5":116,"K_F6":117,"K_F7":118,"K_F8":119,"K_F9":120, + "K_F10":121,"K_F11":122,"K_F12":123,"K_NUMLOCK":144,"K_SCROLL":145, + "K_LSHIFT":160,"K_RSHIFT":161,"K_LCONTROL":162,"K_RCONTROL":163, + "K_LALT":164,"K_RALT":165, + "K_COLON":186,"K_EQUAL":187,"K_COMMA":188,"K_HYPHEN":189, + "K_PERIOD":190,"K_SLASH":191,"K_BKQUOTE":192, + "K_LBRKT":219,"K_BKSLASH":220,"K_RBRKT":221, + "K_QUOTE":222,"K_oE2":226,"K_OE2":226, + "K_LOPT":50001,"K_ROPT":50002, + "K_NUMERALS":50003,"K_SYMBOLS":50004,"K_CURRENCIES":50005, + "K_UPPER":50006,"K_LOWER":50007,"K_ALPHA":50008, + "K_SHIFTED":50009,"K_ALTGR":50010, + "K_TABBACK":50011,"K_TABFWD":50012 + }, + + codesUS: [ + ['0123456789',';=,-./`', '[\\]\''], + [')!@#$%^&*(',':+<_>?~', '{|}"'] + ], + + isKnownOSKModifierKey(keyID: string): boolean { + switch(keyID) { + case 'K_SHIFT': + case 'K_LOPT': + case 'K_ROPT': + case 'K_NUMLOCK': // Often used for numeric layers. + case 'K_CAPS': + return true; + default: + if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. + return true; // is a 'K_' key defined for layer shifting or 'control' use. + } + // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for + // layer shifting keys. To be safe, we currently let K_TABBACK and + // K_TABFWD through, though we might be able to drop them too. + const code = Codes[keyID]; + if(code > 50000 && code < 50011) { return true; - default: - if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. - return true; // is a 'K_' key defined for layer shifting or 'control' use. - } - // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for - // layer shifting keys. To be safe, we currently let K_TABBACK and - // K_TABFWD through, though we might be able to drop them too. - let code = com.keyman.text.Codes[keyID]; - if(code > 50000 && code < 50011) { - return true; - } - } - - return false; + } + } + + return false; + }, + + + /** + * Get modifier key state from layer id + * + * @param {string} layerId layer id (e.g. ctrlshift) + * @return {number} modifier key state (desktop keyboards) + */ + getModifierState(layerId: string): number { + var modifier=0; + if(layerId.indexOf('shift') >= 0) { + modifier |= Codes.modifierCodes['SHIFT']; + } + + // The chiral checks must not be directly exclusive due each other to visual OSK feedback. + var ctrlMatched=false; + if(layerId.indexOf('leftctrl') >= 0) { + modifier |= Codes.modifierCodes['LCTRL']; + ctrlMatched=true; } + if(layerId.indexOf('rightctrl') >= 0) { + modifier |= Codes.modifierCodes['RCTRL']; + ctrlMatched=true; + } + if(layerId.indexOf('ctrl') >= 0 && !ctrlMatched) { + modifier |= Codes.modifierCodes['CTRL']; + } + + var altMatched=false; + if(layerId.indexOf('leftalt') >= 0) { + modifier |= Codes.modifierCodes['LALT']; + altMatched=true; + } + if(layerId.indexOf('rightalt') >= 0) { + modifier |= Codes.modifierCodes['RALT']; + altMatched=true; + } + if(layerId.indexOf('alt') >= 0 && !altMatched) { + modifier |= Codes.modifierCodes['ALT']; + } + + return modifier; + }, + + /** + * Get state key state from layer id + * + * @param {string} layerId layer id (e.g. caps) + * @return {number} modifier key state (desktop keyboards) + */ + getStateFromLayer(layerId: string): number { + var modifier=0; + + if(layerId.indexOf('caps') >= 0) { + modifier |= Codes.modifierCodes['CAPS']; + } else { + modifier |= Codes.modifierCodes['NO_CAPS']; + } + + return modifier; } -} \ No newline at end of file +} + +export default Codes; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/deadkeys.ts b/common/web/keyboard-processor/src/text/deadkeys.ts index d0f775ae0405c40d0f69c305ead0eb68c7f53bbd..e5b32a788fb7773d84df8a5c99f426b26e86fb7c 100644 --- a/common/web/keyboard-processor/src/text/deadkeys.ts +++ b/common/web/keyboard-processor/src/text/deadkeys.ts @@ -1,160 +1,157 @@ -namespace com.keyman.text { - // Defines the base Deadkey-tracking object. - - export class Deadkey { - p: number; // Position of deadkey - d: number; // Numerical id of the deadkey - o: number; // Ordinal value of the deadkey (resolves same-place conflicts) - matched: number; - - static ordinalSeed: number = 0; - - constructor(pos: number, id: number) { - this.p = pos; - this.d = id; - this.o = Deadkey.ordinalSeed++; - } +// Defines the base Deadkey-tracking object. +export class Deadkey { + p: number; // Position of deadkey + d: number; // Numerical id of the deadkey + o: number; // Ordinal value of the deadkey (resolves same-place conflicts) + matched: number; + + static ordinalSeed: number = 0; + + constructor(pos: number, id: number) { + this.p = pos; + this.d = id; + this.o = Deadkey.ordinalSeed++; + } - match(p: number, d: number): boolean { - var result:boolean = (this.p == p && this.d == d); + match(p: number, d: number): boolean { + var result:boolean = (this.p == p && this.d == d); - return result; - } - - set(): void { - this.matched = 1; - } + return result; + } - reset(): void { - this.matched = 0; - } + set(): void { + this.matched = 1; + } - before(other: Deadkey): boolean { - return this.o < other.o; - } + reset(): void { + this.matched = 0; + } - clone(): Deadkey { - let dk = new Deadkey(this.p, this.d); - dk.o = this.o; + before(other: Deadkey): boolean { + return this.o < other.o; + } - return dk; - } + clone(): Deadkey { + let dk = new Deadkey(this.p, this.d); + dk.o = this.o; - /** - * Sorts the deadkeys in reverse order. - */ - static sortFunc = function(a: Deadkey, b: Deadkey) { - // We want descending order, so we want 'later' deadkeys first. - if(a.p != b.p) { - return b.p - a.p; - } else { - return b.o - a.o; - } - }; + return dk; } - // Object-orients deadkey management. - export class DeadkeyTracker { - dks: Deadkey[] = []; + /** + * Sorts the deadkeys in reverse order. + */ + static sortFunc = function(a: Deadkey, b: Deadkey) { + // We want descending order, so we want 'later' deadkeys first. + if(a.p != b.p) { + return b.p - a.p; + } else { + return b.o - a.o; + } + }; +} + +// Object-orients deadkey management. +export class DeadkeyTracker { + dks: Deadkey[] = []; + + toSortedArray(): Deadkey[] { + this.dks = this.dks.sort(Deadkey.sortFunc); + return [].concat(this.dks); + } - toSortedArray(): Deadkey[] { - this.dks = this.dks.sort(Deadkey.sortFunc); - return [].concat(this.dks); - } + clone(): DeadkeyTracker { + let dkt = new DeadkeyTracker(); + let dks = this.toSortedArray(); - clone(): DeadkeyTracker { - let dkt = new DeadkeyTracker(); - let dks = this.toSortedArray(); + // Make sure to clone the deadkeys themselves - the Deadkey object is mutable. + dkt.dks = []; + dks.forEach(function(value: Deadkey) { + dkt.dks.push(value.clone()); + }); - // Make sure to clone the deadkeys themselves - the Deadkey object is mutable. - dkt.dks = []; - dks.forEach(function(value: Deadkey) { - dkt.dks.push(value.clone()); - }); + return dkt; + } - return dkt; - } - - /** - * Function isMatch - * Scope Public - * @param {number} caretPos current cursor position - * @param {number} n expected offset of deadkey from cursor - * @param {number} d deadkey - * @return {boolean} True if deadkey found selected context matches val - * Description Match deadkey at current cursor position - */ - isMatch(caretPos: number, n: number, d: number): boolean { - if(this.dks.length == 0) { - return false; // I3318 + /** + * Function isMatch + * Scope Public + * @param {number} caretPos current cursor position + * @param {number} n expected offset of deadkey from cursor + * @param {number} d deadkey + * @return {boolean} True if deadkey found selected context matches val + * Description Match deadkey at current cursor position + */ + isMatch(caretPos: number, n: number, d: number): boolean { + if(this.dks.length == 0) { + return false; // I3318 + } + + var sp=caretPos; + n = sp - n; + for(var i = 0; i < this.dks.length; i++) { + // Don't re-match an already-matched deadkey. It's possible to have two identical + // entries, and they should be kept separately. + if(this.dks[i].match(n, d) && !this.dks[i].matched) { + this.dks[i].set(); + // Assumption: since we match the first possible entry in the array, we + // match the entry with the lower ordinal - the 'first' deadkey in the position. + return true; // I3318 } + } - var sp=caretPos; - n = sp - n; - for(var i = 0; i < this.dks.length; i++) { - // Don't re-match an already-matched deadkey. It's possible to have two identical - // entries, and they should be kept separately. - if(this.dks[i].match(n, d) && !this.dks[i].matched) { - this.dks[i].set(); - // Assumption: since we match the first possible entry in the array, we - // match the entry with the lower ordinal - the 'first' deadkey in the position. - return true; // I3318 - } - } + this.resetMatched(); // I3318 - this.resetMatched(); // I3318 + return false; + } - return false; - } + add(dk: Deadkey) { + this.dks = this.dks.concat(dk); + } - add(dk: Deadkey) { - this.dks = this.dks.concat(dk); - } + remove(dk: Deadkey) { + var index = this.dks.indexOf(dk); + this.dks.splice(index, 1); + } - remove(dk: Deadkey) { - var index = this.dks.indexOf(dk); - this.dks.splice(index, 1); - } + clear() { + this.dks = []; + } - clear() { - this.dks = []; + resetMatched() { + for(let dk of this.dks) { + dk.reset(); } + } - resetMatched() { - for(let dk of this.dks) { - dk.reset(); - } - } - - deleteMatched(): void { - for(var Li = 0; Li < this.dks.length; Li++) { - if(this.dks[Li].matched) { - this.dks.splice(Li--, 1); // Don't forget to decrement! - } + deleteMatched(): void { + for(var Li = 0; Li < this.dks.length; Li++) { + if(this.dks[Li].matched) { + this.dks.splice(Li--, 1); // Don't forget to decrement! } } + } - /** - * Function adjustPositions (formerly _DeadkeyAdjustPos) - * Scope Private - * @param {number} Lstart start position in context - * @param {number} Ldelta characters to adjust by - * Description Adjust saved positions of deadkeys in context - */ - adjustPositions(Lstart: number, Ldelta: number): void { - if(Ldelta == 0) { - return; - } - - for(let dk of this.dks) { - if(dk.p > Lstart) { - dk.p += Ldelta; - } + /** + * Function adjustPositions (formerly _DeadkeyAdjustPos) + * Scope Private + * @param {number} Lstart start position in context + * @param {number} Ldelta characters to adjust by + * Description Adjust saved positions of deadkeys in context + */ + adjustPositions(Lstart: number, Ldelta: number): void { + if(Ldelta == 0) { + return; + } + + for(let dk of this.dks) { + if(dk.p > Lstart) { + dk.p += Ldelta; } } + } - count(): number { - return this.dks.length; - } + count(): number { + return this.dks.length; } } \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/defaultOutput.ts b/common/web/keyboard-processor/src/text/defaultOutput.ts deleted file mode 100644 index 9d96f4bd1cc8aee6f1df20150e9a0d55fabf3ecc..0000000000000000000000000000000000000000 --- a/common/web/keyboard-processor/src/text/defaultOutput.ts +++ /dev/null @@ -1,191 +0,0 @@ -// Establishes key-code definitions. -/// -// Defines our generalized "KeyEvent" class. -/// - -namespace com.keyman.text { - export enum EmulationKeystrokes { - Enter = '\n', - Backspace = '\b' - } - - /** - * Defines a collection of static library functions that define KeymanWeb's default (implied) keyboard rule behaviors. - */ - export class DefaultOutput { - private constructor() { - } - - static codeForEvent(Lkc: KeyEvent) { - return Codes.keyCodes[Lkc.kName] || Lkc.Lcode;; - } - - /** - * Serves as a default keycode lookup table. This may be referenced safely by mnemonic handling without fear of side-effects. - * Also used by Processor.defaultRuleBehavior to generate output after filtering for special cases. - */ - public static forAny(Lkc: KeyEvent, isMnemonic: boolean, ruleBehavior?: RuleBehavior) { - var char = ''; - - // A pretty simple table of lookups, corresponding VERY closely to the original defaultKeyOutput. - if((char = DefaultOutput.forSpecialEmulation(Lkc, ruleBehavior)) != null) { - return char; - } else if(!isMnemonic && ((char = DefaultOutput.forNumpadKeys(Lkc, ruleBehavior)) != null)) { - return char; - } else if((char = DefaultOutput.forUnicodeKeynames(Lkc, ruleBehavior)) != null) { - return char; - } else if((char = DefaultOutput.forBaseKeys(Lkc, ruleBehavior)) != null) { - return char; - } else { - // // For headless and embeddded, we may well allow '\t'. It's DOM mode that has other uses. - // // Not originally defined for text output within defaultKeyOutput. - // // We can't enable it yet, as it'll cause hardware keystrokes in the DOM to output '\t' rather - // // than rely on the browser-default handling. - let code = DefaultOutput.codeForEvent(Lkc); - switch(code) { - // case Codes.keyCodes['K_TAB']: - // case Codes.keyCodes['K_TABBACK']: - // case Codes.keyCodes['K_TABFWD']: - // return '\t'; - default: - return null; - } - } - } - - /** - * isCommand - returns a boolean indicating if a non-text event should be triggered by the keystroke. - */ - public static isCommand(Lkc: KeyEvent): boolean { - let code = DefaultOutput.codeForEvent(Lkc); - - switch(code) { - // Should we ever implement them: - // case Codes.keyCodes['K_LEFT']: // would not output text, but would alter the caret's position in the context. - // case Codes.keyCodes['K_RIGHT']: - // return true; - default: - return false; - } - } - - /** - * Used when a RuleBehavior represents a non-text "command" within the Engine. This will generally - * trigger events that require context reset - often by moving the caret or by moving what OutputTarget - * the caret is in. However, we let those events perform the actual context reset. - * - * Note: is extended by DOM-aware KeymanWeb code. - */ - public static applyCommand(Lkc: KeyEvent, outputTarget: OutputTarget): void { - // Notes for potential default-handling extensions: - // - // switch(code) { - // // Problem: clusters, and doing them right. - // // The commented-out code below should be a decent starting point, but clusters make it complex. - // // Mostly based on pre-12.0 code, but the general idea should be relatively clear. - // - // case Codes.keyCodes['K_LEFT']: - // if(touchAlias) { - // var caretPos = keymanweb.getTextCaret(Lelem); - // keymanweb.setTextCaret(Lelem, caretPos - 1 >= 0 ? caretPos - 1 : 0); - // } - // break; - // case Codes.keyCodes['K_RIGHT']: - // if(touchAlias) { - // var caretPos = keymanweb.getTextCaret(Lelem); - // keymanweb.setTextCaret(Lelem, caretPos + 1); - // } - // if(code == VisualKeyboard.keyCodes['K_RIGHT']) { - // break; - // } - // } - // - // Note that these would be useful even outside of a DOM context. - } - - /** - * Codes matched here generally have default implementations when in a browser but require emulation - * for 'synthetic' `OutputTarget`s like `Mock`s, which have no default text handling. - */ - public static forSpecialEmulation(Lkc: KeyEvent, ruleBehavior?: RuleBehavior): EmulationKeystrokes { - let code = DefaultOutput.codeForEvent(Lkc); - - switch(code) { - case Codes.keyCodes['K_BKSP']: - return EmulationKeystrokes.Backspace; - case Codes.keyCodes['K_ENTER']: - return EmulationKeystrokes.Enter; - // case Codes.keyCodes['K_DEL']: - // return '\u007f'; // 127, ASCII / Unicode control code for DEL. - default: - return null; - } - } - - // Should not be used for mnenomic keyboards. forAny()'s use of this method checks first. - public static forNumpadKeys(Lkc: KeyEvent, ruleBehavior?: RuleBehavior) { - // Translate numpad keystrokes into their non-numpad equivalents - if(Lkc.Lcode >= Codes.keyCodes["K_NP0"] && Lkc.Lcode <= Codes.keyCodes["K_NPSLASH"]) { - // Number pad, numlock on - if(Lkc.Lcode < 106) { - var Lch = Lkc.Lcode-48; - } else { - Lch = Lkc.Lcode-64; - } - let ch = String._kmwFromCharCode(Lch); //I3319 - return ch; - } else { - return null; - } - } - - // Test for fall back to U_xxxxxx key id - // For this first test, we ignore the keyCode and use the keyName - public static forUnicodeKeynames(Lkc: KeyEvent, ruleBehavior?: RuleBehavior) { - const keyName = Lkc.kName; - - return keyboards.ActiveKey.unicodeIDToText(keyName, (codeWithError) => { - ruleBehavior.errorLog = ("Suppressing Unicode control code in " + keyName + ": " + codeWithError); - }); - } - - // Test for otherwise unimplemented keys on the the base default & shift layers. - // Those keys must be blocked by keyboard rules if intentionally unimplemented; otherwise, this function will trigger. - public static forBaseKeys(Lkc: KeyEvent, ruleBehavior?: RuleBehavior) { - let n = Lkc.Lcode; - let keyShiftState = Lkc.Lmodifiers; - - // check if exact match to SHIFT's code. Only the 'default' and 'shift' layers should have default key outputs. - // TODO: Extend to allow AltGr as well - better mnemonic support. - if(keyShiftState == Codes.modifierCodes['SHIFT']) { - keyShiftState = 1; - } else if(keyShiftState != 0) { - if(ruleBehavior) { - ruleBehavior.warningLog = "KMW only defines default key output for the 'default' and 'shift' layers!"; - } - return null; - } - - // Now that keyShiftState is either 0 or 1, we can use the following structure to determine the default output. - try { - if(n == Codes.keyCodes['K_SPACE']) { - return ' '; - } else if(n >= Codes.keyCodes['K_0'] && n <= Codes.keyCodes['K_9']) { // The number keys. - return Codes.codesUS[keyShiftState][0][n-Codes.keyCodes['K_0']]; - } else if(n >= Codes.keyCodes['K_A'] && n <= Codes.keyCodes['K_Z']) { // The base letter keys - return String.fromCharCode(n+(keyShiftState?0:32)); // 32 is the offset from uppercase to lowercase. - } else if(n >= Codes.keyCodes['K_COLON'] && n <= Codes.keyCodes['K_BKQUOTE']) { - return Codes.codesUS[keyShiftState][1][n-Codes.keyCodes['K_COLON']]; - } else if(n >= Codes.keyCodes['K_LBRKT'] && n <= Codes.keyCodes['K_QUOTE']) { - return Codes.codesUS[keyShiftState][2][n-Codes.keyCodes['K_LBRKT']]; - } - } catch (e) { - if(ruleBehavior) { - ruleBehavior.errorLog = "Error detected with default mapping for key: code = " + n + ", shift state = " + (keyShiftState == 1 ? 'shift' : 'default'); - } - } - - return null; - } - } -} diff --git a/common/web/keyboard-processor/src/text/defaultRules.ts b/common/web/keyboard-processor/src/text/defaultRules.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ad708bd67a1220e0da024f33e7279df3f9b3a58 --- /dev/null +++ b/common/web/keyboard-processor/src/text/defaultRules.ts @@ -0,0 +1,225 @@ +// TODO: Move to separate folder: 'codes' +// We should start splitting off code needed by keyboards even without a KeyboardProcessor active. +// There's an upcoming `/common/web/types` package that 'codes' and 'keyboards' may fit well within. + +import Codes from "./codes.js"; +import type KeyEvent from "./keyEvent.js"; +import type OutputTarget from "./outputTarget.js"; + +// The only members referenced are to produce warning and error logs. A little abstraction +// via an optional 'logger' interface can maintain it while facilitating a the split alluded +// to above. +// +// Alternatively, we could just... not take in the parameter at all, which'd also facilitate +// the future modularization effort. +import RuleBehavior from "./ruleBehavior.js"; + +export enum EmulationKeystrokes { + Enter = '\n', + Backspace = '\b' +} + +/** + * Defines a collection of static library functions that define KeymanWeb's default (implied) keyboard rule behaviors. + */ +export default class DefaultRules { + public constructor() { + } + + codeForEvent(Lkc: KeyEvent) { + return Codes.keyCodes[Lkc.kName] || Lkc.Lcode;; + } + + /** + * Serves as a default keycode lookup table. This may be referenced safely by mnemonic handling without fear of side-effects. + * Also used by Processor.defaultRuleBehavior to generate output after filtering for special cases. + */ + public forAny(Lkc: KeyEvent, isMnemonic: boolean, ruleBehavior?: RuleBehavior) { + var char = ''; + + // A pretty simple table of lookups, corresponding VERY closely to the original defaultKeyOutput. + if((char = this.forSpecialEmulation(Lkc)) != null) { + return char; + } else if(!isMnemonic && ((char = this.forNumpadKeys(Lkc)) != null)) { + return char; + } else if((char = this.forUnicodeKeynames(Lkc, ruleBehavior)) != null) { + return char; + } else if((char = this.forBaseKeys(Lkc, ruleBehavior)) != null) { + return char; + } else { + // // For headless and embeddded, we may well allow '\t'. It's DOM mode that has other uses. + // // Not originally defined for text output within defaultKeyOutput. + // // We can't enable it yet, as it'll cause hardware keystrokes in the DOM to output '\t' rather + // // than rely on the browser-default handling. + let code = this.codeForEvent(Lkc); + switch(code) { + // case Codes.keyCodes['K_TAB']: + // case Codes.keyCodes['K_TABBACK']: + // case Codes.keyCodes['K_TABFWD']: + // return '\t'; + default: + return null; + } + } + } + + /** + * isCommand - returns a boolean indicating if a non-text event should be triggered by the keystroke. + */ + public isCommand(Lkc: KeyEvent): boolean { + let code = this.codeForEvent(Lkc); + + switch(code) { + // Should we ever implement them: + // case Codes.keyCodes['K_LEFT']: // would not output text, but would alter the caret's position in the context. + // case Codes.keyCodes['K_RIGHT']: + // return true; + default: + return false; + } + } + + /** + * Used when a RuleBehavior represents a non-text "command" within the Engine. This will generally + * trigger events that require context reset - often by moving the caret or by moving what OutputTarget + * the caret is in. However, we let those events perform the actual context reset. + * + * Note: is extended by DOM-aware KeymanWeb code. + */ + public applyCommand(Lkc: KeyEvent, outputTarget: OutputTarget): void { + // Notes for potential default-handling extensions: + // + // switch(code) { + // // Problem: clusters, and doing them right. + // // The commented-out code below should be a decent starting point, but clusters make it complex. + // // Mostly based on pre-12.0 code, but the general idea should be relatively clear. + // + // case Codes.keyCodes['K_LEFT']: + // if(touchAlias) { + // var caretPos = keymanweb.getTextCaret(Lelem); + // keymanweb.setTextCaret(Lelem, caretPos - 1 >= 0 ? caretPos - 1 : 0); + // } + // break; + // case Codes.keyCodes['K_RIGHT']: + // if(touchAlias) { + // var caretPos = keymanweb.getTextCaret(Lelem); + // keymanweb.setTextCaret(Lelem, caretPos + 1); + // } + // if(code == VisualKeyboard.keyCodes['K_RIGHT']) { + // break; + // } + // } + // + // Note that these would be useful even outside of a DOM context. + } + + /** + * Codes matched here generally have default implementations when in a browser but require emulation + * for 'synthetic' `OutputTarget`s like `Mock`s, which have no default text handling. + */ + public forSpecialEmulation(Lkc: KeyEvent): EmulationKeystrokes { + let code = this.codeForEvent(Lkc); + + switch(code) { + case Codes.keyCodes['K_BKSP']: + return EmulationKeystrokes.Backspace; + case Codes.keyCodes['K_ENTER']: + return EmulationKeystrokes.Enter; + // case Codes.keyCodes['K_DEL']: + // return '\u007f'; // 127, ASCII / Unicode control code for DEL. + default: + return null; + } + } + + // Should not be used for mnenomic keyboards. forAny()'s use of this method checks first. + public forNumpadKeys(Lkc: KeyEvent) { + // Translate numpad keystrokes into their non-numpad equivalents + if(Lkc.Lcode >= Codes.keyCodes["K_NP0"] && Lkc.Lcode <= Codes.keyCodes["K_NPSLASH"]) { + // Number pad, numlock on + if(Lkc.Lcode < 106) { + var Lch = Lkc.Lcode-48; + } else { + Lch = Lkc.Lcode-64; + } + let ch = String._kmwFromCharCode(Lch); //I3319 + return ch; + } else { + return null; + } + } + + // Test for fall back to U_xxxxxx key id + // For this first test, we ignore the keyCode and use the keyName + public forUnicodeKeynames(Lkc: KeyEvent, ruleBehavior?: RuleBehavior) { + const keyName = Lkc.kName; + + // Test for fall back to U_xxxxxx key id + // For this first test, we ignore the keyCode and use the keyName + if(!keyName || keyName.substr(0,2) != 'U_') { + return null; + } + + let result = ''; + const codePoints = keyName.substr(2).split('_'); + for(let codePoint of codePoints) { + const codePointValue = parseInt(codePoint, 16); + if (((0x0 <= codePointValue) && (codePointValue <= 0x1F)) || ((0x80 <= codePointValue) && (codePointValue <= 0x9F)) || isNaN(codePointValue)) { + // Code points [U_0000 - U_001F] and [U_0080 - U_009F] refer to Unicode C0 and C1 control codes. + // Check the codePoint number and do not allow output of these codes via U_xxxxxx shortcuts. + // Also handles invalid identifiers (e.g. `U_ghij`) for which parseInt returns NaN + if(ruleBehavior) { + ruleBehavior.errorLog = ("Suppressing Unicode control code in " + keyName); + } + // We'll attempt to add valid chars + continue; + } else { + // String.fromCharCode() is inadequate to handle the entire range of Unicode + // Someday after upgrading to ES2015, can use String.fromCodePoint() + result += String.kmwFromCharCode(codePointValue); + } + } + return result ? result : null; + } + + // Test for otherwise unimplemented keys on the the base default & shift layers. + // Those keys must be blocked by keyboard rules if intentionally unimplemented; otherwise, this function will trigger. + public forBaseKeys(Lkc: KeyEvent, ruleBehavior?: RuleBehavior) { + let n = Lkc.Lcode; + let keyShiftState = Lkc.Lmodifiers; + + // check if exact match to SHIFT's code. Only the 'default' and 'shift' layers should have default key outputs. + // TODO: Extend to allow AltGr as well - better mnemonic support. + if(keyShiftState == Codes.modifierCodes['SHIFT']) { + keyShiftState = 1; + } else if(keyShiftState != 0) { + if(ruleBehavior) { + ruleBehavior.warningLog = "KMW only defines default key output for the 'default' and 'shift' layers!"; + } + return null; + } + + // Now that keyShiftState is either 0 or 1, we can use the following structure to determine the default output. + try { + if(n == Codes.keyCodes['K_SPACE']) { + return ' '; + } else if(n >= Codes.keyCodes['K_0'] && n <= Codes.keyCodes['K_9']) { // The number keys. + return Codes.codesUS[keyShiftState][0][n-Codes.keyCodes['K_0']]; + } else if(n >= Codes.keyCodes['K_A'] && n <= Codes.keyCodes['K_Z']) { // The base letter keys + return String.fromCharCode(n+(keyShiftState?0:32)); // 32 is the offset from uppercase to lowercase. + } else if(n >= Codes.keyCodes['K_COLON'] && n <= Codes.keyCodes['K_BKQUOTE']) { + return Codes.codesUS[keyShiftState][1][n-Codes.keyCodes['K_COLON']]; + } else if(n >= Codes.keyCodes['K_LBRKT'] && n <= Codes.keyCodes['K_QUOTE']) { + return Codes.codesUS[keyShiftState][2][n-Codes.keyCodes['K_LBRKT']]; + } else if(n == Codes.keyCodes['K_oE2']) { + return keyShiftState ? '|' : '\\'; + } + } catch (e) { + if(ruleBehavior) { + ruleBehavior.errorLog = "Error detected with default mapping for key: code = " + n + ", shift state = " + (keyShiftState == 1 ? 'shift' : 'default'); + } + } + + return null; + } +} diff --git a/common/web/keyboard-processor/src/text/kbdInterface.ts b/common/web/keyboard-processor/src/text/kbdInterface.ts index 672ecd4dedb266caf424476f266b0bdfde639af9..311a3a75e82c92abbe2658a7d4a86308a4123a0c 100644 --- a/common/web/keyboard-processor/src/text/kbdInterface.ts +++ b/common/web/keyboard-processor/src/text/kbdInterface.ts @@ -1,1164 +1,1178 @@ -/// -/// - -// Defines classes for handling system stores -/// - /*** KeymanWeb 11.0 Copyright 2019 SIL International ***/ -namespace com.keyman.text { - //#region Helper type definitions +//#region Imports - export class KeyInformation { - vk: boolean; - code: number; - modifiers: number; - } +import { type DeviceSpec } from "@keymanapp/web-utils"; - /* - * Type alias definitions to reflect the parameters of the fullContextMatch() callback (KMW 10+). - * No constructors or methods since keyboards will not utilize the same backing prototype, and - * property names are shorthanded to promote minification. - */ - type PlainKeyboardStore = string; +import Codes from "./codes.js"; +import type KeyEvent from "./keyEvent.js"; +import type { Deadkey } from "./deadkeys.js"; +import KeyMapping from "./keyMapping.js"; +import { SystemStore, MutableSystemStore, PlatformSystemStore } from "./systemStores.js"; +import type { VariableStoreSerializer } from "./keyboardProcessor.js"; +import type OutputTarget from "./outputTarget.js"; +import { Mock } from "./outputTarget.js"; +import RuleBehavior from "./ruleBehavior.js"; +import Keyboard, { VariableStoreDictionary } from "../keyboards/keyboard.js"; +import { KeyboardHarness, KeyboardKeymanGlobal } from "../keyboards/keyboardHarness.js"; - export type KeyboardStoreElement = (string|StoreNonCharEntry); - export type ComplexKeyboardStore = KeyboardStoreElement[]; +//#endregion - type KeyboardStore = PlainKeyboardStore | ComplexKeyboardStore; +//#region Helper type definitions - export type VariableStore = {[name: string]: string}; +export class KeyInformation { + vk: boolean; + code: number; + modifiers: number; +} - type RuleChar = string; +/* +* Type alias definitions to reflect the parameters of the fullContextMatch() callback (KMW 10+). +* No constructors or methods since keyboards will not utilize the same backing prototype, and +* property names are shorthanded to promote minification. +*/ +type PlainKeyboardStore = string; - class RuleDeadkey { - /** Discriminant field - 'd' for Deadkey. - */ - ['t']: 'd'; +export type KeyboardStoreElement = (string|StoreNonCharEntry); +export type ComplexKeyboardStore = KeyboardStoreElement[]; - /** - * Value: the deadkey's ID. - */ - ['d']: number; // For 'd'eadkey; also reflects the Deadkey class's 'd' property. - } +type KeyboardStore = PlainKeyboardStore | ComplexKeyboardStore; - class ContextAny { - /** Discriminant field - 'a' for `any()`. - */ - ['t']: 'a'; +export type VariableStore = {[name: string]: string}; - /** - * Value: the store to search. - */ - ['a']: KeyboardStore; // For 'a'ny statement. +type RuleChar = string; - /** - * If set to true, negates the 'any'. - */ - ['n']: boolean|0|1; - } - - class RuleIndex { - /** Discriminant field - 'i' for `index()`. - */ - ['t']: 'i'; +class RuleDeadkey { + /** Discriminant field - 'd' for Deadkey. + */ + ['t']: 'd'; - /** - * Value: the Store from which to output - */ - ['i']: KeyboardStore; + /** + * Value: the deadkey's ID. + */ + ['d']: number; // For 'd'eadkey; also reflects the Deadkey class's 'd' property. +} - /** - * Offset: the offset in context for the corresponding `any()`. - */ - ['o']: number; - } +class ContextAny { + /** Discriminant field - 'a' for `any()`. + */ + ['t']: 'a'; - class ContextEx { - /** Discriminant field - 'c' for `context()`. - */ - ['t']: 'c'; + /** + * Value: the store to search. + */ + ['a']: KeyboardStore; // For 'a'ny statement. - /** - * Value: The offset into the current rule's context to be matched. - */ - ['c']: number; // For 'c'ontext statement. - } + /** + * If set to true, negates the 'any'. + */ + ['n']: boolean|0|1; +} - class ContextNul { - /** Discriminant field - 'n' for `nul` - */ - ['t']: 'n'; - } +class RuleIndex { + /** Discriminant field - 'i' for `index()`. + */ + ['t']: 'i'; - class StoreBeep { - /** Discriminant field - 'b' for `beep` - */ - ['t']: 'b'; - } + /** + * Value: the Store from which to output + */ + ['i']: KeyboardStore; - type ContextNonCharEntry = RuleDeadkey | ContextAny | RuleIndex | ContextEx | ContextNul; - type ContextEntry = RuleChar | ContextNonCharEntry; + /** + * Offset: the offset in context for the corresponding `any()`. + */ + ['o']: number; +} - type StoreNonCharEntry = RuleDeadkey | StoreBeep; +class ContextEx { + /** Discriminant field - 'c' for `context()`. + */ + ['t']: 'c'; /** - * Cache of context storing and retrieving return values from KC - * Must be reset prior to each keystroke and after any text changes - * MCD 3/1/14 - **/ - class CachedContext { - _cache: string[][]; - - reset(): void { - this._cache = []; - } - - get(n: number, ln: number): string { - // return null; // uncomment this line to disable context caching - if(typeof this._cache[n] == 'undefined') { - return null; - } else if(typeof this._cache[n][ln] == 'undefined') { - return null; - } - return this._cache[n][ln]; - } + * Value: The offset into the current rule's context to be matched. + */ + ['c']: number; // For 'c'ontext statement. +} - set(n: number, ln: number, val: string): void { - if(typeof this._cache[n] == 'undefined') { - this._cache[n] = []; - } - this._cache[n][ln] = val; - } - }; +class ContextNul { + /** Discriminant field - 'n' for `nul` + */ + ['t']: 'n'; +} - type CachedExEntry = {valContext: (string|number)[], deadContext: text.Deadkey[]}; - /** - * An extended version of cached context storing designed to work with - * `fullContextMatch` and its helper functions. +class StoreBeep { + /** Discriminant field - 'b' for `beep` */ - class CachedContextEx { - _cache: CachedExEntry[][]; + ['t']: 'b'; +} + +type ContextNonCharEntry = RuleDeadkey | ContextAny | RuleIndex | ContextEx | ContextNul; +type ContextEntry = RuleChar | ContextNonCharEntry; + +type StoreNonCharEntry = RuleDeadkey | StoreBeep; - reset(): void { - this._cache = []; +/** + * Cache of context storing and retrieving return values from KC + * Must be reset prior to each keystroke and after any text changes + * MCD 3/1/14 + **/ +class CachedContext { + _cache: string[][]; + + reset(): void { + this._cache = []; + } + + get(n: number, ln: number): string { + // return null; // uncomment this line to disable context caching + if(typeof this._cache[n] == 'undefined') { + return null; + } else if(typeof this._cache[n][ln] == 'undefined') { + return null; } + return this._cache[n][ln]; + } - get(n: number, ln: number): CachedExEntry { - // return null; // uncomment this line to disable context caching - if(typeof this._cache[n] == 'undefined') { - return null; - } else if(typeof this._cache[n][ln] == 'undefined') { - return null; - } - return this._cache[n][ln]; + set(n: number, ln: number, val: string): void { + if(typeof this._cache[n] == 'undefined') { + this._cache[n] = []; } + this._cache[n][ln] = val; + } +}; + +type CachedExEntry = {valContext: (string|number)[], deadContext: Deadkey[]}; +/** + * An extended version of cached context storing designed to work with + * `fullContextMatch` and its helper functions. + */ +class CachedContextEx { + _cache: CachedExEntry[][]; + + reset(): void { + this._cache = []; + } - set(n: number, ln: number, val: CachedExEntry): void { - if(typeof this._cache[n] == 'undefined') { - this._cache[n] = []; - } - this._cache[n][ln] = val; + get(n: number, ln: number): CachedExEntry { + // return null; // uncomment this line to disable context caching + if(typeof this._cache[n] == 'undefined') { + return null; + } else if(typeof this._cache[n][ln] == 'undefined') { + return null; } + return this._cache[n][ln]; + } - clone(): CachedContextEx { - let r = new CachedContextEx(); - r._cache = this._cache; - return r; + set(n: number, ln: number, val: CachedExEntry): void { + if(typeof this._cache[n] == 'undefined') { + this._cache[n] = []; } - }; + this._cache[n][ln] = val; + } - //#endregion + clone(): CachedContextEx { + let r = new CachedContextEx(); + r._cache = this._cache; + return r; + } +}; - export class KeyboardInterface { - static readonly GLOBAL_NAME = 'KeymanWeb'; +export enum SystemStoreIDs { + TSS_LAYER = 33, + TSS_PLATFORM = 31, + TSS_NEWLAYER = 42, + TSS_OLDLAYER = 43 +} - cachedContext: CachedContext = new CachedContext(); - cachedContextEx: CachedContextEx = new CachedContextEx(); - ruleContextEx: CachedContextEx; +//#endregion - activeTargetOutput: OutputTarget; - ruleBehavior: RuleBehavior; +export default class KeyboardInterface extends KeyboardHarness { + static readonly GLOBAL_NAME = 'KeymanWeb'; - static readonly TSS_LAYER: number = 33; - static readonly TSS_PLATFORM: number = 31; - static readonly TSS_NEWLAYER: number = 42; - static readonly TSS_OLDLAYER: number = 43; + cachedContext: CachedContext = new CachedContext(); + cachedContextEx: CachedContextEx = new CachedContextEx(); + ruleContextEx: CachedContextEx; - systemStores: {[storeID: number]: SystemStore}; + activeTargetOutput: OutputTarget; + ruleBehavior: RuleBehavior; - _AnyIndices: number[] = []; // AnyIndex - array of any/index match indices + systemStores: {[storeID: number]: SystemStore}; - // Must be accessible to some of the keyboard API methods. - activeKeyboard: keyboards.Keyboard; - activeDevice: utils.DeviceSpec; + _AnyIndices: number[] = []; // AnyIndex - array of any/index match indices - variableStoreSerializer?: VariableStoreSerializer; + // Must be accessible to some of the keyboard API methods. + activeKeyboard: Keyboard; + activeDevice: DeviceSpec; - constructor(variableStoreSerializer: VariableStoreSerializer = null) { - this.systemStores = {}; + variableStoreSerializer?: VariableStoreSerializer; - this.systemStores[KeyboardInterface.TSS_PLATFORM] = new PlatformSystemStore(this); - this.systemStores[KeyboardInterface.TSS_LAYER] = new MutableSystemStore(KeyboardInterface.TSS_LAYER, 'default'); - this.systemStores[KeyboardInterface.TSS_NEWLAYER] = new MutableSystemStore(KeyboardInterface.TSS_NEWLAYER, ''); - this.systemStores[KeyboardInterface.TSS_OLDLAYER] = new MutableSystemStore(KeyboardInterface.TSS_OLDLAYER, ''); + // A 'reference point' that debug keyboards may use to access KMW's code constants. + public get Codes(): typeof Codes { + return Codes; + } - this.variableStoreSerializer = variableStoreSerializer; - } + constructor(_jsGlobal: any, keymanGlobal: KeyboardKeymanGlobal, variableStoreSerializer: VariableStoreSerializer = null) { + super(_jsGlobal, keymanGlobal); - /** - * Function KSF - * Scope Public - * - * Saves the document's current focus settings on behalf of the keyboard. Often paired with insertText. - */ - saveFocus(): void { } - - /** - * A text-insertion method used by custom OSKs for helpHTML interaction, like with sil_euro_latin. - * - * This function currently bypasses web-core's standard text handling control path and all predictive text processing. - * It also has DOM-dependencies that help ensure KMW's active OutputTarget retains focus during use. - */ - insertText?: (Ptext: string, PdeadKey: number) => boolean; - - /** - * Function registerKeyboard KR - * Scope Public - * @param {Object} Pk Keyboard object - * Description Registers a keyboard with KeymanWeb once its script has fully loaded. - * - * In web-core, this also activates the keyboard; in other modules, this method - * may be replaced with other implementations. - */ - registerKeyboard(Pk): void { - // NOTE: This implementation is web-core specific and is intentionally replaced, whole-sale, - // by DOM-aware code. - let keyboard = new keyboards.Keyboard(Pk); - this.activeKeyboard = keyboard; - } + this.systemStores = {}; - /** - * Used by DOM-aware KeymanWeb to add keyboard stubs, used by the `KeyboardManager` type - * to optimize resource use. - */ - registerStub?: (Pstub) => number; - - /** - * Get *cached or uncached* keyboard context for a specified range, relative to caret - * - * @param {number} n Number of characters to move back from caret - * @param {number} ln Number of characters to return - * @param {Object} Pelem Element to work with (must be currently focused element) - * @return {string} Context string - * - * Example [abcdef|ghi] as INPUT, with the caret position marked by |: - * KC(2,1,Pelem) == "e" - * KC(3,3,Pelem) == "def" - * KC(10,10,Pelem) == "abcdef" i.e. return as much as possible of the requested string - */ - - context(n: number, ln: number, outputTarget: OutputTarget): string { - var v = this.cachedContext.get(n, ln); - if(v !== null) { - return v; - } + this.systemStores[SystemStoreIDs.TSS_PLATFORM] = new PlatformSystemStore(this); + this.systemStores[SystemStoreIDs.TSS_LAYER] = new MutableSystemStore(SystemStoreIDs.TSS_LAYER, 'default'); + this.systemStores[SystemStoreIDs.TSS_NEWLAYER] = new MutableSystemStore(SystemStoreIDs.TSS_NEWLAYER, ''); + this.systemStores[SystemStoreIDs.TSS_OLDLAYER] = new MutableSystemStore(SystemStoreIDs.TSS_OLDLAYER, ''); - var r = this.KC_(n, ln, outputTarget); - this.cachedContext.set(n, ln, r); - return r; - } + this.variableStoreSerializer = variableStoreSerializer; + } - /** - * Get (uncached) keyboard context for a specified range, relative to caret - * - * @param {number} n Number of characters to move back from caret - * @param {number} ln Number of characters to return - * @param {Object} Pelem Element to work with (must be currently focused element) - * @return {string} Context string - * - * Example [abcdef|ghi] as INPUT, with the caret position marked by |: - * KC(2,1,Pelem) == "e" - * KC(3,3,Pelem) == "def" - * KC(10,10,Pelem) == "XXXXabcdef" i.e. return as much as possible of the requested string, where X = \uFFFE - */ - private KC_(n: number, ln: number, outputTarget: OutputTarget): string { - var tempContext = ''; - - // If we have a selection, we have an empty context - tempContext = outputTarget.isSelectionEmpty() ? outputTarget.getTextBeforeCaret() : ""; - - if(tempContext._kmwLength() < n) { - tempContext = Array(n-tempContext._kmwLength()+1).join("\uFFFE") + tempContext; - } + /** + * Function KSF + * Scope Public + * + * Saves the document's current focus settings on behalf of the keyboard. Often paired with insertText. + */ + saveFocus(): void { } + + /** + * A text-insertion method used by custom OSKs for helpHTML interaction, like with sil_euro_latin. + * + * This function currently bypasses web-core's standard text handling control path and all predictive text processing. + * It also has DOM-dependencies that help ensure KMW's active OutputTarget retains focus during use. + */ + insertText?: (Ptext: string, PdeadKey: number) => void; + + /** + * Function registerKeyboard KR + * Scope Public + * @param {Object} Pk Keyboard object + * Description Registers a keyboard with KeymanWeb once its script has fully loaded. + * + * In web-core, this also activates the keyboard; in other modules, this method + * may be replaced with other implementations. + */ + registerKeyboard(Pk): void { + // NOTE: This implementation is web-core specific and is intentionally replaced, whole-sale, + // by DOM-aware code. + let keyboard = new Keyboard(Pk); + this.loadedKeyboard = keyboard; + } - return tempContext._kmwSubstr(-n)._kmwSubstr(0,ln); + /** + * Get *cached or uncached* keyboard context for a specified range, relative to caret + * + * @param {number} n Number of characters to move back from caret + * @param {number} ln Number of characters to return + * @param {Object} Pelem Element to work with (must be currently focused element) + * @return {string} Context string + * + * Example [abcdef|ghi] as INPUT, with the caret position marked by |: + * KC(2,1,Pelem) == "e" + * KC(3,3,Pelem) == "def" + * KC(10,10,Pelem) == "abcdef" i.e. return as much as possible of the requested string + */ + + context(n: number, ln: number, outputTarget: OutputTarget): string { + var v = this.cachedContext.get(n, ln); + if(v !== null) { + return v; } - /** - * Function nul KN - * Scope Public - * @param {number} n Length of context to check - * @param {Object} Ptarg Element to work with (must be currently focused element) - * @return {boolean} True if length of context is less than or equal to n - * Description Test length of context, return true if the length of the context is less than or equal to n - * - * Example [abc|def] as INPUT, with the caret position marked by |: - * KN(3,Pelem) == TRUE - * KN(2,Pelem) == FALSE - * KN(4,Pelem) == TRUE - */ - nul(n: number, outputTarget: OutputTarget): boolean { - var cx=this.context(n+1, 1, outputTarget); - - // With #31, the result will be a replacement character if context is empty. - return cx === "\uFFFE"; + var r = this.KC_(n, ln, outputTarget); + this.cachedContext.set(n, ln, r); + return r; + } + + /** + * Get (uncached) keyboard context for a specified range, relative to caret + * + * @param {number} n Number of characters to move back from caret + * @param {number} ln Number of characters to return + * @param {Object} Pelem Element to work with (must be currently focused element) + * @return {string} Context string + * + * Example [abcdef|ghi] as INPUT, with the caret position marked by |: + * KC(2,1,Pelem) == "e" + * KC(3,3,Pelem) == "def" + * KC(10,10,Pelem) == "XXXXabcdef" i.e. return as much as possible of the requested string, where X = \uFFFE + */ + private KC_(n: number, ln: number, outputTarget: OutputTarget): string { + var tempContext = ''; + + // If we have a selection, we have an empty context + tempContext = outputTarget.isSelectionEmpty() ? outputTarget.getTextBeforeCaret() : ""; + + if(tempContext._kmwLength() < n) { + tempContext = Array(n-tempContext._kmwLength()+1).join("\uFFFE") + tempContext; } - /** - * Function contextMatch KCM - * Scope Public - * @param {number} n Number of characters to move back from caret - * @param {Object} Ptarg Focused element - * @param {string} val String to match - * @param {number} ln Number of characters to return - * @return {boolean} True if selected context matches val - * Description Test keyboard context for match - */ - contextMatch(n: number, outputTarget: OutputTarget, val: string, ln: number): boolean { - var cx=this.context(n, ln, outputTarget); - if(cx === val) { - return true; // I3318 - } - outputTarget.deadkeys().resetMatched(); // I3318 - return false; + return tempContext._kmwSubstr(-n)._kmwSubstr(0,ln); + } + + /** + * Function nul KN + * Scope Public + * @param {number} n Length of context to check + * @param {Object} Ptarg Element to work with (must be currently focused element) + * @return {boolean} True if length of context is less than or equal to n + * Description Test length of context, return true if the length of the context is less than or equal to n + * + * Example [abc|def] as INPUT, with the caret position marked by |: + * KN(3,Pelem) == TRUE + * KN(2,Pelem) == FALSE + * KN(4,Pelem) == TRUE + */ + nul(n: number, outputTarget: OutputTarget): boolean { + var cx=this.context(n+1, 1, outputTarget); + + // With #31, the result will be a replacement character if context is empty. + return cx === "\uFFFE"; + } + + /** + * Function contextMatch KCM + * Scope Public + * @param {number} n Number of characters to move back from caret + * @param {Object} Ptarg Focused element + * @param {string} val String to match + * @param {number} ln Number of characters to return + * @return {boolean} True if selected context matches val + * Description Test keyboard context for match + */ + contextMatch(n: number, outputTarget: OutputTarget, val: string, ln: number): boolean { + var cx=this.context(n, ln, outputTarget); + if(cx === val) { + return true; // I3318 } + outputTarget.deadkeys().resetMatched(); // I3318 + return false; + } - /** - * Builds the *cached or uncached* keyboard context for a specified range, relative to caret - * - * @param {number} n Number of characters to move back from caret - * @param {number} ln Number of characters to return - * @param {Object} Pelem Element to work with (must be currently focused element) - * @return {Array} Context array (of strings and numbers) - */ - private _BuildExtendedContext(n: number, ln: number, outputTarget: OutputTarget): CachedExEntry { - var cache: CachedExEntry = this.cachedContextEx.get(n, ln); - if(cache !== null) { - return cache; - } else { - // By far the easiest way to correctly build what we want is to start from the right and work to what we need. - // We may have done it for a similar cursor position before. - cache = this.cachedContextEx.get(n, n); - if(cache === null) { - // First, let's make sure we have a cloned, sorted copy of the deadkey array. - let unmatchedDeadkeys = outputTarget.deadkeys().toSortedArray(); // Is reverse-order sorted for us already. - - // Time to build from scratch! - var index = 0; - cache = { valContext: [], deadContext: []}; - while(cache.valContext.length < n) { - // As adapted from `deadkeyMatch`. - var sp = outputTarget.getDeadkeyCaret(); - var deadPos = sp - index; - if(unmatchedDeadkeys.length > 0 && unmatchedDeadkeys[0].p > deadPos) { - // We have deadkeys at the right-hand side of the caret! They don't belong in the context, so pop 'em off. - unmatchedDeadkeys.splice(0, 1); - continue; - } else if(unmatchedDeadkeys.length > 0 && unmatchedDeadkeys[0].p == deadPos) { - // Take the deadkey. - cache.deadContext[n-cache.valContext.length-1] = unmatchedDeadkeys[0]; - cache.valContext = ([unmatchedDeadkeys[0].d] as (string|number)[]).concat(cache.valContext); - unmatchedDeadkeys.splice(0, 1); - } else { - // Take the character. We get "\ufffe" if it doesn't exist. - var kc = this.context(++index, 1, outputTarget); - cache.valContext = ([kc] as (string|number)[]).concat(cache.valContext); - } + /** + * Builds the *cached or uncached* keyboard context for a specified range, relative to caret + * + * @param {number} n Number of characters to move back from caret + * @param {number} ln Number of characters to return + * @param {Object} Pelem Element to work with (must be currently focused element) + * @return {Array} Context array (of strings and numbers) + */ + private _BuildExtendedContext(n: number, ln: number, outputTarget: OutputTarget): CachedExEntry { + var cache: CachedExEntry = this.cachedContextEx.get(n, ln); + if(cache !== null) { + return cache; + } else { + // By far the easiest way to correctly build what we want is to start from the right and work to what we need. + // We may have done it for a similar cursor position before. + cache = this.cachedContextEx.get(n, n); + if(cache === null) { + // First, let's make sure we have a cloned, sorted copy of the deadkey array. + let unmatchedDeadkeys = outputTarget.deadkeys().toSortedArray(); // Is reverse-order sorted for us already. + + // Time to build from scratch! + var index = 0; + cache = { valContext: [], deadContext: []}; + while(cache.valContext.length < n) { + // As adapted from `deadkeyMatch`. + var sp = outputTarget.getDeadkeyCaret(); + var deadPos = sp - index; + if(unmatchedDeadkeys.length > 0 && unmatchedDeadkeys[0].p > deadPos) { + // We have deadkeys at the right-hand side of the caret! They don't belong in the context, so pop 'em off. + unmatchedDeadkeys.splice(0, 1); + continue; + } else if(unmatchedDeadkeys.length > 0 && unmatchedDeadkeys[0].p == deadPos) { + // Take the deadkey. + cache.deadContext[n-cache.valContext.length-1] = unmatchedDeadkeys[0]; + cache.valContext = ([unmatchedDeadkeys[0].d] as (string|number)[]).concat(cache.valContext); + unmatchedDeadkeys.splice(0, 1); + } else { + // Take the character. We get "\ufffe" if it doesn't exist. + var kc = this.context(++index, 1, outputTarget); + cache.valContext = ([kc] as (string|number)[]).concat(cache.valContext); } - this.cachedContextEx.set(n, n, cache); } + this.cachedContextEx.set(n, n, cache); + } - // Now that we have the cache... - var subCache = cache; - subCache.valContext = subCache.valContext.slice(0, ln); - for(var i=0; i < subCache.valContext.length; i++) { - if(subCache[i] == '\ufffe') { - subCache.valContext.splice(0, 1); - subCache.deadContext.splice(0, 1); - } + // Now that we have the cache... + var subCache = cache; + subCache.valContext = subCache.valContext.slice(0, ln); + for(var i=0; i < subCache.valContext.length; i++) { + if(subCache[i] == '\ufffe') { + subCache.valContext.splice(0, 1); + subCache.deadContext.splice(0, 1); } + } - if(subCache.valContext.length == 0) { - subCache.valContext = ['\ufffe']; - subCache.deadContext = []; - } + if(subCache.valContext.length == 0) { + subCache.valContext = ['\ufffe']; + subCache.deadContext = []; + } - this.cachedContextEx.set(n, ln, subCache); + this.cachedContextEx.set(n, ln, subCache); - return subCache; - } + return subCache; } + } - /** - * Function fullContextMatch KFCM - * Scope Private - * @param {number} n Number of characters to move back from caret - * @param {Object} Ptarg Focused element - * @param {Array} rule An array of ContextEntries to match. - * @return {boolean} True if the fully-specified rule context matches the current KMW state. - * - * A KMW 10+ function designed to bring KMW closer to Keyman Desktop functionality, - * near-directly modeling (externally) the compiled form of Desktop rules' context section. - */ - fullContextMatch(n: number, outputTarget: OutputTarget, rule: ContextEntry[]): boolean { - // Stage one: build the context index map. - var fullContext = this._BuildExtendedContext(n, rule.length, outputTarget); - this.ruleContextEx = this.cachedContextEx.clone(); - var context = fullContext.valContext; - var deadContext = fullContext.deadContext; - - var mismatch = false; - - // This symbol internally indicates lack of context in a position. (See KC_) - const NUL_CONTEXT = "\uFFFE"; - - var assertNever = function(x: never): never { - // Could be accessed by improperly handwritten calls to `fullContextMatch`. - throw new Error("Unexpected object in fullContextMatch specification: " + x); - } + /** + * Function fullContextMatch KFCM + * Scope Private + * @param {number} n Number of characters to move back from caret + * @param {Object} Ptarg Focused element + * @param {Array} rule An array of ContextEntries to match. + * @return {boolean} True if the fully-specified rule context matches the current KMW state. + * + * A KMW 10+ function designed to bring KMW closer to Keyman Desktop functionality, + * near-directly modeling (externally) the compiled form of Desktop rules' context section. + */ + fullContextMatch(n: number, outputTarget: OutputTarget, rule: ContextEntry[]): boolean { + // Stage one: build the context index map. + var fullContext = this._BuildExtendedContext(n, rule.length, outputTarget); + this.ruleContextEx = this.cachedContextEx.clone(); + var context = fullContext.valContext; + var deadContext = fullContext.deadContext; + + var mismatch = false; + + // This symbol internally indicates lack of context in a position. (See KC_) + const NUL_CONTEXT = "\uFFFE"; + + var assertNever = function(x: never): never { + // Could be accessed by improperly handwritten calls to `fullContextMatch`. + throw new Error("Unexpected object in fullContextMatch specification: " + x); + } - // Stage two: time to match against the rule specified. - for(var i=0; i < rule.length; i++) { - if(typeof rule[i] == 'string') { - var str = rule[i] as string; - if(str !== context[i]) { - mismatch = true; + // Stage two: time to match against the rule specified. + for(var i=0; i < rule.length; i++) { + if(typeof rule[i] == 'string') { + var str = rule[i] as string; + if(str !== context[i]) { + mismatch = true; + break; + } + } else { + // TypeScript needs a cast to this intermediate type to do its discriminated union magic. + var r = rule[i] as ContextNonCharEntry; + switch(r.t) { + case 'd': + // We still need to set a flag here; + if(r['d'] !== context[i]) { + mismatch = true; + } else { + deadContext[i].set(); + } break; - } - } else { - // TypeScript needs a cast to this intermediate type to do its discriminated union magic. - var r = rule[i] as ContextNonCharEntry; - switch(r.t) { - case 'd': - // We still need to set a flag here; - if(r['d'] !== context[i]) { - mismatch = true; - } else { - deadContext[i].set(); - } - break; - case 'a': - var lookup: KeyboardStoreElement; - - if(typeof context[i] == 'string') { - lookup = context[i] as string; - } else { - lookup = {'t': 'd', 'd': context[i] as number}; - } + case 'a': + var lookup: KeyboardStoreElement; - var result = this.any(i, lookup, r.a); - - if(!r.n) { // If it's a standard 'any'... - if(!result) { - mismatch = true; - } else if(deadContext[i] !== undefined) { - // It's a deadkey match, so indicate that. - deadContext[i].set(); - } - // 'n' for 'notany'. - // - if `result === true`, `any` would match: this should thus fail. - // - if `context[i] === NUL_CONTEXT`, `notany` should not match. - } else if(r.n && (result || context[i] === NUL_CONTEXT)) { - mismatch = true; - } - break; - case 'i': - // The context will never hold a 'beep.' - var ch = this._Index(r.i, r.o) as string | RuleDeadkey; + if(typeof context[i] == 'string') { + lookup = context[i] as string; + } else { + lookup = {'t': 'd', 'd': context[i] as number}; + } - if(ch !== undefined && (typeof(ch) == 'string' ? ch : ch.d) !== context[i]) { - mismatch = true; - } else if(deadContext[i] !== undefined) { - deadContext[i].set(); - } - break; - case 'c': - if(context[r.c - 1] !== context[i]) { + var result = this.any(i, lookup, r.a); + + if(!r.n) { // If it's a standard 'any'... + if(!result) { mismatch = true; } else if(deadContext[i] !== undefined) { + // It's a deadkey match, so indicate that. deadContext[i].set(); } - break; - case 'n': - // \uFFFE is the internal 'no context here sentinel'. - if(context[i] != NUL_CONTEXT) { - mismatch = true; - } - break; - default: - assertNever(r); - } + // 'n' for 'notany'. + // - if `result === true`, `any` would match: this should thus fail. + // - if `context[i] === NUL_CONTEXT`, `notany` should not match. + } else if(r.n && (result || context[i] === NUL_CONTEXT)) { + mismatch = true; + } + break; + case 'i': + // The context will never hold a 'beep.' + var ch = this._Index(r.i, r.o) as string | RuleDeadkey; + + if(ch !== undefined && (typeof(ch) == 'string' ? ch : ch.d) !== context[i]) { + mismatch = true; + } else if(deadContext[i] !== undefined) { + deadContext[i].set(); + } + break; + case 'c': + if(context[r.c - 1] !== context[i]) { + mismatch = true; + } else if(deadContext[i] !== undefined) { + deadContext[i].set(); + } + break; + case 'n': + // \uFFFE is the internal 'no context here sentinel'. + if(context[i] != NUL_CONTEXT) { + mismatch = true; + } + break; + default: + assertNever(r); } } + } - if(mismatch) { - // Reset the matched 'any' indices, if any. - outputTarget.deadkeys().resetMatched(); - this._AnyIndices = []; - } - - return !mismatch; + if(mismatch) { + // Reset the matched 'any' indices, if any. + outputTarget.deadkeys().resetMatched(); + this._AnyIndices = []; } - /** - * Function KIK - * Scope Public - * @param {Object} e keystroke event - * @return {boolean} true if keypress event - * Description Test if event as a keypress event - */ - isKeypress(e: KeyEvent): boolean { - if(this.activeKeyboard.isMnemonic) { // I1380 - support KIK for positional layouts - return !e.LisVirtualKey; // will now return true for U_xxxx keys, but not for T_xxxx keys - } else { - return KeyMapping._USKeyCodeToCharCode(e) ? true : false; // I1380 - support KIK for positional layouts - } + return !mismatch; + } + + /** + * Function KIK + * Scope Public + * @param {Object} e keystroke event + * @return {boolean} true if keypress event + * Description Test if event as a keypress event + */ + isKeypress(e: KeyEvent): boolean { + if(this.activeKeyboard.isMnemonic) { // I1380 - support KIK for positional layouts + return !e.LisVirtualKey; // will now return true for U_xxxx keys, but not for T_xxxx keys + } else { + return KeyMapping._USKeyCodeToCharCode(e) ? true : false; // I1380 - support KIK for positional layouts } + } - /** - * Maps a KeyEvent's modifiers to their appropriate value for key-rule evaluation - * based on the rule's specified target modifier set. - * - * Mostly used to correct chiral OSK-keys targeting non-chiral rules. - * @param e The source KeyEvent - * @returns - */ - private static matchModifiersToRuleChirality(eventModifiers: number, targetModifierMask: number): number { - const CHIRAL_ALT = Codes.modifierCodes["LALT"] | Codes.modifierCodes["RALT"]; - const CHIRAL_CTRL = Codes.modifierCodes["LCTRL"] | Codes.modifierCodes["RCTRL"]; - - let modifiers = eventModifiers; - - // If the target rule does not use chiral alt... - if(!(targetModifierMask & CHIRAL_ALT)) { - const altIntersection = modifiers & CHIRAL_ALT; - - if(altIntersection) { - // Undo the chiral part and replace with non-chiral. - modifiers ^= altIntersection | Codes.modifierCodes["ALT"]; - } - } + /** + * Maps a KeyEvent's modifiers to their appropriate value for key-rule evaluation + * based on the rule's specified target modifier set. + * + * Mostly used to correct chiral OSK-keys targeting non-chiral rules. + * @param e The source KeyEvent + * @returns + */ + private static matchModifiersToRuleChirality(eventModifiers: number, targetModifierMask: number): number { + const CHIRAL_ALT = Codes.modifierCodes["LALT"] | Codes.modifierCodes["RALT"]; + const CHIRAL_CTRL = Codes.modifierCodes["LCTRL"] | Codes.modifierCodes["RCTRL"]; - // If the target rule does not use chiral ctrl... - if(!(targetModifierMask & CHIRAL_CTRL)) { - const ctrlIntersection = modifiers & CHIRAL_CTRL; + let modifiers = eventModifiers; - if(ctrlIntersection) { - // Undo the chiral part and replace with non-chiral. - modifiers ^= ctrlIntersection | Codes.modifierCodes["CTRL"]; - } - } + // If the target rule does not use chiral alt... + if(!(targetModifierMask & CHIRAL_ALT)) { + const altIntersection = modifiers & CHIRAL_ALT; - return modifiers; + if(altIntersection) { + // Undo the chiral part and replace with non-chiral. + modifiers ^= altIntersection | Codes.modifierCodes["ALT"]; + } } - /** - * Function keyMatch KKM - * Scope Public - * @param {Object} e keystroke event - * @param {number} Lruleshift - * @param {number} Lrulekey - * @return {boolean} True if key matches rule - * Description Test keystroke with modifiers against rule - */ - keyMatch(e: KeyEvent, Lruleshift:number, Lrulekey:number): boolean { - var retVal = false; // I3318 - var keyCode = (e.Lcode == 173 ? 189 : e.Lcode); //I3555 (Firefox hyphen issue) - - let bitmask = this.activeKeyboard.modifierBitmask; - let Codes = com.keyman.text.Codes; - var modifierBitmask = bitmask & Codes.modifierBitmasks["ALL"]; - var stateBitmask = bitmask & Codes.stateBitmasks["ALL"]; - - const eventModifiers = KeyboardInterface.matchModifiersToRuleChirality(e.Lmodifiers, Lruleshift); - - if(e.vkCode > 255) { - keyCode = e.vkCode; // added to support extended (touch-hold) keys for mnemonic layouts - } + // If the target rule does not use chiral ctrl... + if(!(targetModifierMask & CHIRAL_CTRL)) { + const ctrlIntersection = modifiers & CHIRAL_CTRL; - if(e.LisVirtualKey || keyCode > 255) { - if((Lruleshift & 0x4000) == 0x4000 || (keyCode > 255)) { // added keyCode test to support extended keys - retVal = ((Lrulekey == keyCode) && ((Lruleshift & modifierBitmask) == eventModifiers)); //I3318, I3555 - retVal = retVal && this.stateMatch(e, Lruleshift & stateBitmask); - } - } else if((Lruleshift & 0x4000) == 0) { - retVal = (keyCode == Lrulekey); // I3318, I3555 - } - if(!retVal) { - this.activeTargetOutput.deadkeys().resetMatched(); // I3318 + if(ctrlIntersection) { + // Undo the chiral part and replace with non-chiral. + modifiers ^= ctrlIntersection | Codes.modifierCodes["CTRL"]; } - return retVal; // I3318 - }; - - /** - * Function stateMatch KSM - * Scope Public - * @param {Object} e keystroke event - * @param {number} Lstate - * Description Test keystroke against state key rules - */ - stateMatch(e: KeyEvent, Lstate: number) { - return ((Lstate & e.Lstates) == Lstate); } - /** - * Function keyInformation KKI - * Scope Public - * @param {Object} e - * @return {Object} Object with event's virtual key flag, key code, and modifiers - * Description Get object with extended key event information - */ - keyInformation(e: KeyEvent): KeyInformation { - var ei = new KeyInformation(); - ei['vk'] = e.LisVirtualKey; - ei['code'] = e.Lcode; - ei['modifiers'] = e.Lmodifiers; - return ei; - }; - - /** - * Function deadkeyMatch KDM - * Scope Public - * @param {number} n offset from current cursor position - * @param {Object} Ptarg target element - * @param {number} d deadkey - * @return {boolean} True if deadkey found selected context matches val - * Description Match deadkey at current cursor position - */ - deadkeyMatch(n: number, outputTarget: OutputTarget, d: number): boolean { - return outputTarget.hasDeadkeyMatch(n, d); + return modifiers; + } + + /** + * Function keyMatch KKM + * Scope Public + * @param {Object} e keystroke event + * @param {number} Lruleshift + * @param {number} Lrulekey + * @return {boolean} True if key matches rule + * Description Test keystroke with modifiers against rule + */ + keyMatch(e: KeyEvent, Lruleshift:number, Lrulekey:number): boolean { + var retVal = false; // I3318 + var keyCode = (e.Lcode == 173 ? 189 : e.Lcode); //I3555 (Firefox hyphen issue) + + let bitmask = this.activeKeyboard.modifierBitmask; + var modifierBitmask = bitmask & Codes.modifierBitmasks["ALL"]; + var stateBitmask = bitmask & Codes.stateBitmasks["ALL"]; + + const eventModifiers = KeyboardInterface.matchModifiersToRuleChirality(e.Lmodifiers, Lruleshift); + + if(e.vkCode > 255) { + keyCode = e.vkCode; // added to support extended (touch-hold) keys for mnemonic layouts } - /** - * Function beep KB - * Scope Public - * @param {Object} Pelem element to flash - * Description Flash body as substitute for audible beep; notify embedded device to vibrate - */ - beep(outputTarget: OutputTarget): void { - this.resetContextCache(); - - // Denote as part of the matched rule's behavior. - this.ruleBehavior.beep = true; + if(e.LisVirtualKey || keyCode > 255) { + if((Lruleshift & 0x4000) == 0x4000 || (keyCode > 255)) { // added keyCode test to support extended keys + retVal = ((Lrulekey == keyCode) && ((Lruleshift & modifierBitmask) == eventModifiers)); //I3318, I3555 + retVal = retVal && this.stateMatch(e, Lruleshift & stateBitmask); + } + } else if((Lruleshift & 0x4000) == 0) { + retVal = (keyCode == Lrulekey); // I3318, I3555 + } + if(!retVal) { + this.activeTargetOutput.deadkeys().resetMatched(); // I3318 } + return retVal; // I3318 + }; - _ExplodeStore(store: KeyboardStore): ComplexKeyboardStore { - if(typeof(store) == 'string') { - let cachedStores = this.activeKeyboard.explodedStores; + /** + * Function stateMatch KSM + * Scope Public + * @param {Object} e keystroke event + * @param {number} Lstate + * Description Test keystroke against state key rules + */ + stateMatch(e: KeyEvent, Lstate: number) { + return ((Lstate & e.Lstates) == Lstate); + } - // Is the result cached? - if(cachedStores[store]) { - return cachedStores[store]; - } + /** + * Function keyInformation KKI + * Scope Public + * @param {Object} e + * @return {Object} Object with event's virtual key flag, key code, and modifiers + * Description Get object with extended key event information + */ + keyInformation(e: KeyEvent): KeyInformation { + var ei = new KeyInformation(); + ei['vk'] = e.LisVirtualKey; + ei['code'] = e.Lcode; + ei['modifiers'] = e.Lmodifiers; + return ei; + }; - // Nope, so let's build its cache. - var result: ComplexKeyboardStore = []; - for(var i=0; i < store._kmwLength(); i++) { - result.push(store._kmwCharAt(i)); - } + /** + * Function deadkeyMatch KDM + * Scope Public + * @param {number} n offset from current cursor position + * @param {Object} Ptarg target element + * @param {number} d deadkey + * @return {boolean} True if deadkey found selected context matches val + * Description Match deadkey at current cursor position + */ + deadkeyMatch(n: number, outputTarget: OutputTarget, d: number): boolean { + return outputTarget.hasDeadkeyMatch(n, d); + } - // Cache the result for later! - cachedStores[store] = result; - return result; - } else { - return store; + /** + * Function beep KB + * Scope Public + * @param {Object} Pelem element to flash + * Description Flash body as substitute for audible beep; notify embedded device to vibrate + */ + beep(outputTarget: OutputTarget): void { + this.resetContextCache(); + + // Denote as part of the matched rule's behavior. + this.ruleBehavior.beep = true; + } + + _ExplodeStore(store: KeyboardStore): ComplexKeyboardStore { + if(typeof(store) == 'string') { + let cachedStores = this.activeKeyboard.explodedStores; + + // Is the result cached? + if(cachedStores[store]) { + return cachedStores[store]; } - } - /** - * Function any KA - * Scope Public - * @param {number} n character position (index) - * @param {string} ch character to find in string - * @param {string} s 'any' string - * @return {boolean} True if character found in 'any' string, sets index accordingly - * Description Test for character matching - */ - any(n: number, ch: KeyboardStoreElement, s: KeyboardStore): boolean { - if(ch == '') { - return false; + // Nope, so let's build its cache. + var result: ComplexKeyboardStore = []; + for(var i=0; i < store._kmwLength(); i++) { + result.push(store._kmwCharAt(i)); } - s = this._ExplodeStore(s); - var Lix = -1; - for(var i=0; i < s.length; i++) { - if(typeof(s[i]) == 'string') { - if(s[i] == ch) { - Lix = i; - break; - } - } else if(s[i]['d'] === ch['d']) { + // Cache the result for later! + cachedStores[store] = result; + return result; + } else { + return store; + } + } + + /** + * Function any KA + * Scope Public + * @param {number} n character position (index) + * @param {string} ch character to find in string + * @param {string} s 'any' string + * @return {boolean} True if character found in 'any' string, sets index accordingly + * Description Test for character matching + */ + any(n: number, ch: KeyboardStoreElement, s: KeyboardStore): boolean { + if(ch == '') { + return false; + } + + s = this._ExplodeStore(s); + var Lix = -1; + for(var i=0; i < s.length; i++) { + if(typeof(s[i]) == 'string') { + if(s[i] == ch) { Lix = i; break; } + } else if(s[i]['d'] === ch['d']) { + Lix = i; + break; } - this._AnyIndices[n] = Lix; - return Lix >= 0; } + this._AnyIndices[n] = Lix; + return Lix >= 0; + } - /** - * Function _Index - * Scope Public - * @param {string} Ps string - * @param {number} Pn index - * Description Returns the character from a store string according to the offset in the index array - */ - _Index(Ps: KeyboardStore, Pn: number): KeyboardStoreElement { - Ps = this._ExplodeStore(Ps); - - if(this._AnyIndices[Pn-1] < Ps.length) { //I3319 - return Ps[this._AnyIndices[Pn-1]]; - } else { - /* Should not be possible for a compiled keyboard, but may arise - * during the development of handwritten keyboards. - */ - console.warn("Unmatched contextual index() statement detected in rule with index " + Pn + "!"); - return ""; - } + /** + * Function _Index + * Scope Public + * @param {string} Ps string + * @param {number} Pn index + * Description Returns the character from a store string according to the offset in the index array + */ + _Index(Ps: KeyboardStore, Pn: number): KeyboardStoreElement { + Ps = this._ExplodeStore(Ps); + + if(this._AnyIndices[Pn-1] < Ps.length) { //I3319 + return Ps[this._AnyIndices[Pn-1]]; + } else { + /* Should not be possible for a compiled keyboard, but may arise + * during the development of handwritten keyboards. + */ + console.warn("Unmatched contextual index() statement detected in rule with index " + Pn + "!"); + return ""; } + } - /** - * Function indexOutput KIO - * Scope Public - * @param {number} Pdn no of character to overwrite (delete) - * @param {string} Ps string - * @param {number} Pn index - * @param {Object} Pelem element to output to - * Description Output a character selected from the string according to the offset in the index array - */ - indexOutput(Pdn: number, Ps: KeyboardStore, Pn: number, outputTarget: OutputTarget): void { - this.resetContextCache(); - - var assertNever = function(x: never): never { - // Could be accessed by improperly handwritten calls to `fullContextMatch`. - throw new Error("Unexpected object in fullContextMatch specification: " + x); - } + /** + * Function indexOutput KIO + * Scope Public + * @param {number} Pdn no of character to overwrite (delete) + * @param {string} Ps string + * @param {number} Pn index + * @param {Object} Pelem element to output to + * Description Output a character selected from the string according to the offset in the index array + */ + indexOutput(Pdn: number, Ps: KeyboardStore, Pn: number, outputTarget: OutputTarget): void { + this.resetContextCache(); - var indexChar = this._Index(Ps, Pn); - if(indexChar !== "") { - if(typeof indexChar == 'string' ) { - this.output(Pdn, outputTarget, indexChar); //I3319 - } else if(indexChar['t']) { - var storeEntry = indexChar as StoreNonCharEntry; - - switch(storeEntry.t) { - case 'b': // Beep commands may appear within stores. - this.beep(outputTarget); - break; - case 'd': - this.deadkeyOutput(Pdn, outputTarget, indexChar['d']); - break; - default: - assertNever(storeEntry); - } - } else { // For keyboards developed during 10.0's alpha phase - t:'d' was assumed. - this.deadkeyOutput(Pdn, outputTarget, indexChar['d']); + var assertNever = function(x: never): never { + // Could be accessed by improperly handwritten calls to `fullContextMatch`. + throw new Error("Unexpected object in fullContextMatch specification: " + x); + } + + var indexChar = this._Index(Ps, Pn); + if(indexChar !== "") { + if(typeof indexChar == 'string' ) { + this.output(Pdn, outputTarget, indexChar); //I3319 + } else if(indexChar['t']) { + var storeEntry = indexChar as StoreNonCharEntry; + + switch(storeEntry.t) { + case 'b': // Beep commands may appear within stores. + this.beep(outputTarget); + break; + case 'd': + this.deadkeyOutput(Pdn, outputTarget, indexChar['d']); + break; + default: + assertNever(storeEntry); } + } else { // For keyboards developed during 10.0's alpha phase - t:'d' was assumed. + this.deadkeyOutput(Pdn, outputTarget, indexChar['d']); } } + } - /** - * Function deleteContext KDC - * Scope Public - * @param {number} dn number of context entries to overwrite - * @param {Object} Pelem element to output to - * @param {string} s string to output - * Description Keyboard output - */ - deleteContext(dn: number, outputTarget: OutputTarget): void { - var context: CachedExEntry; - - // We want to control exactly which deadkeys get removed. - if(dn > 0) { - context = this._BuildExtendedContext(dn, dn, outputTarget); - let nulCount = 0; - - for(var i=0; i < context.valContext.length; i++) { - var dk = context.deadContext[i]; - - if(dk) { - // Remove deadkey in context. - outputTarget.deadkeys().remove(dk); - - // Reduce our reported context size. - dn--; - } else if(context.valContext[i] == "\uFFFE") { - // Count any `nul` sentinels that would contribute to our deletion count. - nulCount++; - } + /** + * Function deleteContext KDC + * Scope Public + * @param {number} dn number of context entries to overwrite + * @param {Object} Pelem element to output to + * @param {string} s string to output + * Description Keyboard output + */ + deleteContext(dn: number, outputTarget: OutputTarget): void { + var context: CachedExEntry; + + // We want to control exactly which deadkeys get removed. + if(dn > 0) { + context = this._BuildExtendedContext(dn, dn, outputTarget); + let nulCount = 0; + + for(var i=0; i < context.valContext.length; i++) { + var dk = context.deadContext[i]; + + if(dk) { + // Remove deadkey in context. + outputTarget.deadkeys().remove(dk); + + // Reduce our reported context size. + dn--; + } else if(context.valContext[i] == "\uFFFE") { + // Count any `nul` sentinels that would contribute to our deletion count. + nulCount++; } + } - // Prevent attempts to delete nul sentinels, as they don't exist in the actual context. - // (Addresses regression from KMW v 12.0 paired with Developer bug through same version) - let contextLength = context.valContext.length - nulCount; - if(dn > contextLength) { - dn = contextLength; - } + // Prevent attempts to delete nul sentinels, as they don't exist in the actual context. + // (Addresses regression from KMW v 12.0 paired with Developer bug through same version) + let contextLength = context.valContext.length - nulCount; + if(dn > contextLength) { + dn = contextLength; } + } - // If a matched deadkey hasn't been deleted, we don't WANT to delete it. - outputTarget.deadkeys().resetMatched(); + // If a matched deadkey hasn't been deleted, we don't WANT to delete it. + outputTarget.deadkeys().resetMatched(); - // Why reinvent the wheel? Delete the remaining characters by 'inserting a blank string'. - this.output(dn, outputTarget, ''); - } + // Why reinvent the wheel? Delete the remaining characters by 'inserting a blank string'. + this.output(dn, outputTarget, ''); + } - /** - * Function output KO - * Scope Public - * @param {number} dn number of characters to overwrite - * @param {Object} Pelem element to output to - * @param {string} s string to output - * Description Keyboard output - */ - output(dn: number, outputTarget: OutputTarget, s:string): void { - this.resetContextCache(); - - outputTarget.saveProperties(); - outputTarget.clearSelection(); - outputTarget.deadkeys().deleteMatched(); // I3318 - if(dn >= 0) { - // Automatically manages affected deadkey positions. Does not delete deadkeys b/c legacy behavior support. - outputTarget.deleteCharsBeforeCaret(dn); - } - // Automatically manages affected deadkey positions. - outputTarget.insertTextBeforeCaret(s); - outputTarget.restoreProperties(); + /** + * Function output KO + * Scope Public + * @param {number} dn number of characters to overwrite + * @param {Object} Pelem element to output to + * @param {string} s string to output + * Description Keyboard output + */ + output(dn: number, outputTarget: OutputTarget, s:string): void { + this.resetContextCache(); + + outputTarget.saveProperties(); + outputTarget.clearSelection(); + outputTarget.deadkeys().deleteMatched(); // I3318 + if(dn >= 0) { + // Automatically manages affected deadkey positions. Does not delete deadkeys b/c legacy behavior support. + outputTarget.deleteCharsBeforeCaret(dn); } + // Automatically manages affected deadkey positions. + outputTarget.insertTextBeforeCaret(s); + outputTarget.restoreProperties(); + } - /** - * `contextExOutput` function emits the character or object at `contextOffset` from the - * current matched rule's context. Introduced in Keyman 14.0, in order to resolve a - * gap between desktop and web core functionality for context(n) matching on notany(). - * See #917 for additional detail. - * @alias KCXO - * @public - * @param {number} Pdn number of characters to delete left of cursor - * @param {OutputTarget} outputTarget target to output to - * @param {number} contextLength length of current rule context to retrieve - * @param {number} contextOffset offset from start of current rule context, 1-based - */ - contextExOutput(Pdn: number, outputTarget: OutputTarget, contextLength: number, contextOffset: number): void { - this.resetContextCache(); - - if(Pdn >= 0) { - this.output(Pdn, outputTarget, ""); - } + /** + * `contextExOutput` function emits the character or object at `contextOffset` from the + * current matched rule's context. Introduced in Keyman 14.0, in order to resolve a + * gap between desktop and web core functionality for context(n) matching on notany(). + * See #917 for additional detail. + * @alias KCXO + * @public + * @param {number} Pdn number of characters to delete left of cursor + * @param {OutputTarget} outputTarget target to output to + * @param {number} contextLength length of current rule context to retrieve + * @param {number} contextOffset offset from start of current rule context, 1-based + */ + contextExOutput(Pdn: number, outputTarget: OutputTarget, contextLength: number, contextOffset: number): void { + this.resetContextCache(); - const context = this.ruleContextEx.get(contextLength, contextLength); - const dk = context.deadContext[contextOffset-1], vc = context.valContext[contextOffset-1]; - if(dk) { - outputTarget.insertDeadkeyBeforeCaret(dk.d); - } else if(typeof vc == 'string') { - this.output(-1, outputTarget, vc); - } else { - throw new Error("contextExOutput: should never be a numeric valContext with no corresponding deadContext"); - } + if(Pdn >= 0) { + this.output(Pdn, outputTarget, ""); } - /** - * Function deadkeyOutput KDO - * Scope Public - * @param {number} Pdn no of character to overwrite (delete) - * @param {Object} Pelem element to output to - * @param {number} Pd deadkey id - * Description Record a deadkey at current cursor position, deleting Pdn characters first - */ - deadkeyOutput(Pdn: number, outputTarget: OutputTarget, Pd: number): void { - this.resetContextCache(); - - if(Pdn >= 0) { - this.output(Pdn, outputTarget,""); //I3318 corrected to >= - } - - outputTarget.insertDeadkeyBeforeCaret(Pd); - // _DebugDeadKeys(Pelem, 'KDeadKeyOutput: dn='+Pdn+'; deadKey='+Pd); + const context = this.ruleContextEx.get(contextLength, contextLength); + const dk = context.deadContext[contextOffset-1], vc = context.valContext[contextOffset-1]; + if(dk) { + outputTarget.insertDeadkeyBeforeCaret(dk.d); + } else if(typeof vc == 'string') { + this.output(-1, outputTarget, vc); + } else { + throw new Error("contextExOutput: should never be a numeric valContext with no corresponding deadContext"); } + } - /** - * KIFS compares the content of a system store with a string value - * - * @param {number} systemId ID of the system store to test (only TSS_LAYER currently supported) - * @param {string} strValue String value to compare to - * @param {Object} Pelem Currently active element (may be needed by future tests) - * @return {boolean} True if the test succeeds - */ - ifStore(systemId: number, strValue: string, outputTarget: OutputTarget): boolean { - var result=true; - let store = this.systemStores[systemId]; - if(store) { - result = store.matches(strValue); - } - return result; //Moved from previous line, now supports layer selection, Build 350 + /** + * Function deadkeyOutput KDO + * Scope Public + * @param {number} Pdn no of character to overwrite (delete) + * @param {Object} Pelem element to output to + * @param {number} Pd deadkey id + * Description Record a deadkey at current cursor position, deleting Pdn characters first + */ + deadkeyOutput(Pdn: number, outputTarget: OutputTarget, Pd: number): void { + this.resetContextCache(); + + if(Pdn >= 0) { + this.output(Pdn, outputTarget,""); //I3318 corrected to >= } - /** - * KSETS sets the value of a system store to a string - * - * @param {number} systemId ID of the system store to set (only TSS_LAYER currently supported) - * @param {string} strValue String to set as the system store content - * @param {Object} Pelem Currently active element (may be needed in future tests) - * @return {boolean} True if command succeeds - * (i.e. for TSS_LAYER, if the layer is successfully selected) - * - * Note that option/variable stores are instead set within keyboard script code, as they only - * affect keyboard behavior. - */ - setStore(systemId: number, strValue: string, outputTarget: OutputTarget): boolean { - this.resetContextCache(); - // Unique case: we only allow set(&layer) ops from keyboard rules triggered by touch OSKs. - if(systemId == KeyboardInterface.TSS_LAYER && this.activeDevice.touchable) { - // Denote the changed store as part of the matched rule's behavior. - this.ruleBehavior.setStore[systemId] = strValue; - } else { - return false; - } + outputTarget.insertDeadkeyBeforeCaret(Pd); + // _DebugDeadKeys(Pelem, 'KDeadKeyOutput: dn='+Pdn+'; deadKey='+Pd); + } + + /** + * KIFS compares the content of a system store with a string value + * + * @param {number} systemId ID of the system store to test (only TSS_LAYER currently supported) + * @param {string} strValue String value to compare to + * @param {Object} Pelem Currently active element (may be needed by future tests) + * @return {boolean} True if the test succeeds + */ + ifStore(systemId: number, strValue: string, outputTarget: OutputTarget): boolean { + var result=true; + let store = this.systemStores[systemId]; + if(store) { + result = store.matches(strValue); } + return result; //Moved from previous line, now supports layer selection, Build 350 + } - /** - * Load an option store value from a cookie or default value - * - * @param {string} kbdName keyboard internal name - * @param {string} storeName store (option) name, embedded in cookie name - * @param {string} dfltValue default value - * @return {string} current or default option value - * - * This will only ever be called when the keyboard is loaded, as it is used by keyboards - * to initialize a store value on the keyboard's script object. - */ - loadStore(kbdName: string, storeName:string, dfltValue:string): string { - this.resetContextCache(); - if(this.variableStoreSerializer) { - let cValue = this.variableStoreSerializer.loadStore(kbdName, storeName); - return cValue[storeName] || dfltValue; - } else { - return dfltValue; - } + /** + * KSETS sets the value of a system store to a string + * + * @param {number} systemId ID of the system store to set (only TSS_LAYER currently supported) + * @param {string} strValue String to set as the system store content + * @param {Object} Pelem Currently active element (may be needed in future tests) + * @return {boolean} True if command succeeds + * (i.e. for TSS_LAYER, if the layer is successfully selected) + * + * Note that option/variable stores are instead set within keyboard script code, as they only + * affect keyboard behavior. + */ + setStore(systemId: number, strValue: string, outputTarget: OutputTarget): boolean { + this.resetContextCache(); + // Unique case: we only allow set(&layer) ops from keyboard rules triggered by touch OSKs. + if(systemId == SystemStoreIDs.TSS_LAYER && this.activeDevice.touchable) { + // Denote the changed store as part of the matched rule's behavior. + this.ruleBehavior.setStore[systemId] = strValue; + } else { + return false; } + } - /** - * Save an option store value to a cookie - * - * @param {string} storeName store (option) name, embedded in cookie name - * @param {string} optValue option value to save - * @return {boolean} true if save successful - * - * Note that a keyboard will freely manipulate the value of its variable stores on the - * script object within its own code. This function's use is merely to _persist_ that - * value across sessions, providing a custom user default for later uses of the keyboard. - */ - saveStore(storeName:string, optValue:string): boolean { - this.resetContextCache(); - var kbd=this.activeKeyboard; - if(!kbd || typeof kbd.id == 'undefined' || kbd.id == '') { - return false; - } + /** + * Load an option store value from a cookie or default value + * + * @param {string} kbdName keyboard internal name + * @param {string} storeName store (option) name, embedded in cookie name + * @param {string} dfltValue default value + * @return {string} current or default option value + * + * This will only ever be called when the keyboard is loaded, as it is used by keyboards + * to initialize a store value on the keyboard's script object. + */ + loadStore(kbdName: string, storeName:string, dfltValue:string): string { + this.resetContextCache(); + if(this.variableStoreSerializer) { + let cValue = this.variableStoreSerializer.loadStore(kbdName, storeName); + return cValue[storeName] || dfltValue; + } else { + return dfltValue; + } + } - // And the lookup under that entry looks for the value under the store name, again. - let valueObj: VariableStore = {}; - valueObj[storeName] = optValue; + /** + * Save an option store value to a cookie + * + * @param {string} storeName store (option) name, embedded in cookie name + * @param {string} optValue option value to save + * @return {boolean} true if save successful + * + * Note that a keyboard will freely manipulate the value of its variable stores on the + * script object within its own code. This function's use is merely to _persist_ that + * value across sessions, providing a custom user default for later uses of the keyboard. + */ + saveStore(storeName:string, optValue:string): boolean { + this.resetContextCache(); + var kbd=this.activeKeyboard; + if(!kbd || typeof kbd.id == 'undefined' || kbd.id == '') { + return false; + } - // Null-check in case of invocation during unit-test - if(this.ruleBehavior) { - this.ruleBehavior.saveStore[storeName] = valueObj; - } else { - // We're in a unit-test environment, directly invoking this method from outside of a keyboard. - // In this case, we should immediately commit the change. - this.variableStoreSerializer.saveStore(this.activeKeyboard.id, storeName, valueObj); - } - return true; + // And the lookup under that entry looks for the value under the store name, again. + let valueObj: VariableStore = {}; + valueObj[storeName] = optValue; + + // Null-check in case of invocation during unit-test + if(this.ruleBehavior) { + this.ruleBehavior.saveStore[storeName] = valueObj; + } else { + // We're in a unit-test environment, directly invoking this method from outside of a keyboard. + // In this case, we should immediately commit the change. + this.variableStoreSerializer.saveStore(this.activeKeyboard.id, storeName, valueObj); } + return true; + } + + resetContextCache(): void { + this.cachedContext.reset(); + this.cachedContextEx.reset(); + } - resetContextCache(): void { - this.cachedContext.reset(); - this.cachedContextEx.reset(); + defaultBackspace(outputTarget: OutputTarget) { + if(outputTarget.isSelectionEmpty()) { + // Delete the character left of the caret + this.output(1, outputTarget, ""); + } else { + // Delete just the selection + this.output(0, outputTarget, ""); } + } - defaultBackspace(outputTarget: OutputTarget) { - if(outputTarget.isSelectionEmpty()) { - // Delete the character left of the caret - this.output(1, outputTarget, ""); - } else { - // Delete just the selection - this.output(0, outputTarget, ""); - } + /** + * Function processNewContextEvent + * Scope Private + * @param {Object} outputTarget The target receiving input + * @param {Object} keystroke The input keystroke (with its properties) to be mapped by the keyboard. + * Description Calls the keyboard's `begin newContext` group + * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin NewContext` + */ + processNewContextEvent(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { + if(!this.activeKeyboard) { + throw "No active keyboard for keystroke processing!"; } + return this.process(this.activeKeyboard.processNewContextEvent.bind(this.activeKeyboard), outputTarget, keystroke, true); + } - /** - * Function processNewContextEvent - * Scope Private - * @param {Object} outputTarget The target receiving input - * @param {Object} keystroke The input keystroke (with its properties) to be mapped by the keyboard. - * Description Calls the keyboard's `begin newContext` group - * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin NewContext` - */ - processNewContextEvent(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { - if(!this.activeKeyboard) { - throw "No active keyboard for keystroke processing!"; - } - return this.process(this.activeKeyboard.processNewContextEvent.bind(this.activeKeyboard), outputTarget, keystroke, true); + /** + * Function processPostKeystroke + * Scope Private + * @param {Object} outputTarget The target receiving input + * @param {Object} keystroke The input keystroke with relevant properties to be mapped by the keyboard. + * Description Calls the keyboard's `begin postKeystroke` group + * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin PostKeystroke` + */ + processPostKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { + if(!this.activeKeyboard) { + throw "No active keyboard for keystroke processing!"; } + return this.process(this.activeKeyboard.processPostKeystroke.bind(this.activeKeyboard), outputTarget, keystroke, true); + } - /** - * Function processPostKeystroke - * Scope Private - * @param {Object} outputTarget The target receiving input - * @param {Object} keystroke The input keystroke with relevant properties to be mapped by the keyboard. - * Description Calls the keyboard's `begin postKeystroke` group - * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin PostKeystroke` - */ - processPostKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { - if(!this.activeKeyboard) { - throw "No active keyboard for keystroke processing!"; - } - return this.process(this.activeKeyboard.processPostKeystroke.bind(this.activeKeyboard), outputTarget, keystroke, true); + /** + * Function processKeystroke + * Scope Private + * @param {Object} outputTarget The target receiving input + * @param {Object} keystroke The input keystroke (with its properties) to be mapped by the keyboard. + * Description Encapsulates calls to keyboard input processing. + * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin Unicode` + */ + processKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { + if(!this.activeKeyboard) { + throw "No active keyboard for keystroke processing!"; } + return this.process(this.activeKeyboard.process.bind(this.activeKeyboard), outputTarget, keystroke, false); + } - /** - * Function processKeystroke - * Scope Private - * @param {Object} outputTarget The target receiving input - * @param {Object} keystroke The input keystroke (with its properties) to be mapped by the keyboard. - * Description Encapsulates calls to keyboard input processing. - * @returns {RuleBehavior} Record of commands and state changes that result from executing `begin Unicode` - */ - processKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior { - if(!this.activeKeyboard) { - throw "No active keyboard for keystroke processing!"; - } - return this.process(this.activeKeyboard.process.bind(this.activeKeyboard), outputTarget, keystroke, false); + private process(callee, outputTarget: OutputTarget, keystroke: KeyEvent, readonly: boolean): RuleBehavior { + // Clear internal state tracking data from prior keystrokes. + if(!outputTarget) { + throw "No target specified for keyboard output!"; + } else if(!this.activeKeyboard) { + throw "No active keyboard for keystroke processing!"; + } else if(!callee) { + throw "No callee for keystroke processing!"; } - private process(callee, outputTarget: OutputTarget, keystroke: KeyEvent, readonly: boolean): RuleBehavior { - // Clear internal state tracking data from prior keystrokes. - if(!outputTarget) { - throw "No target specified for keyboard output!"; - } else if(!this.activeKeyboard) { - throw "No active keyboard for keystroke processing!"; - } else if(!callee) { - throw "No callee for keystroke processing!"; - } + outputTarget.invalidateSelection(); - outputTarget.invalidateSelection(); + outputTarget.deadkeys().resetMatched(); // I3318 + this.resetContextCache(); - outputTarget.deadkeys().resetMatched(); // I3318 - this.resetContextCache(); + // Capture the initial state of the OutputTarget before any rules are matched. + let preInput = Mock.from(outputTarget, true); - // Capture the initial state of the OutputTarget before any rules are matched. - let preInput = Mock.from(outputTarget, true); + // Capture the initial state of any variable stores + const cachedVariableStores = this.activeKeyboard.variableStores; - // Capture the initial state of any variable stores - const cachedVariableStores = this.activeKeyboard.variableStores; + // Establishes the results object, allowing corresponding commands to set values here as appropriate. + this.ruleBehavior = new RuleBehavior(); - // Establishes the results object, allowing corresponding commands to set values here as appropriate. - this.ruleBehavior = new RuleBehavior(); + // Ensure the settings are in place so that KIFS/ifState activates and deactivates + // the appropriate rule(s) for the modeled device. + this.activeDevice = keystroke.device; - // Ensure the settings are in place so that KIFS/ifState activates and deactivates - // the appropriate rule(s) for the modeled device. - this.activeDevice = keystroke.device; + // Calls the start-group of the active keyboard. + this.activeTargetOutput = outputTarget; + var matched = callee(outputTarget, keystroke); + this.activeTargetOutput = null; - // Calls the start-group of the active keyboard. - this.activeTargetOutput = outputTarget; - var matched = callee(outputTarget, keystroke); - this.activeTargetOutput = null; + // Finalize the rule's results. + this.ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(preInput, keystroke, readonly); - // Finalize the rule's results. - this.ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(preInput, keystroke, readonly); + // We always backup the changes to variable stores to the RuleBehavior, to + // be applied during finalization, then restore them to the cached initial + // values to avoid side-effects with predictive text mocks. + this.ruleBehavior.variableStores = this.activeKeyboard.variableStores; + this.activeKeyboard.variableStores = cachedVariableStores; - // We always backup the changes to variable stores to the RuleBehavior, to - // be applied during finalization, then restore them to the cached initial - // values to avoid side-effects with predictive text mocks. - this.ruleBehavior.variableStores = this.activeKeyboard.variableStores; - this.activeKeyboard.variableStores = cachedVariableStores; + // `matched` refers to whether or not the FINAL rule (from any group) matched, rather than + // whether or not ANY rule matched. If the final rule doesn't match, we trigger the key's + // default behavior (if appropriate). + // + // See https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852 + this.ruleBehavior.triggerKeyDefault = !matched; - // `matched` refers to whether or not the FINAL rule (from any group) matched, rather than - // whether or not ANY rule matched. If the final rule doesn't match, we trigger the key's - // default behavior (if appropriate). - // - // See https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852 - this.ruleBehavior.triggerKeyDefault = !matched; + // Clear our result-tracking variable to prevent any possible pollution for future processing. + let behavior = this.ruleBehavior; + this.ruleBehavior = null; - // Clear our result-tracking variable to prevent any possible pollution for future processing. - let behavior = this.ruleBehavior; - this.ruleBehavior = null; + return behavior; + } - return behavior; - } + /** + * Applies the dictionary of variable store values to the active keyboard + * + * Has no effect on keyboards compiled with 14.0 or earlier; system store + * names are not exposed unless compiled with Developer 15.0 or later. + * + * @param stores A dictionary of stores which should be found in the + * keyboard + */ + applyVariableStores(stores: VariableStoreDictionary): void { + this.activeKeyboard.variableStores = stores; + } - /** - * Applies the dictionary of variable store values to the active keyboard - * - * Has no effect on keyboards compiled with 14.0 or earlier; system store - * names are not exposed unless compiled with Developer 15.0 or later. - * - * @param stores A dictionary of stores which should be found in the - * keyboard - */ - applyVariableStores(stores: com.keyman.keyboards.VariableStoreDictionary): void { - this.activeKeyboard.variableStores = stores; - } + /** + * Publishes the KeyboardInterface's shorthand API names. As this assigns the current functions + * held by the longform versions, note that this should be called after replacing any of them via + * JS method extension. + * + * DOM-aware KeymanWeb should call this after its domKbdInterface.ts code is loaded, as it replaces + * a few. (This is currently done within its kmwapi.ts.) + */ + static __publishShorthandAPI() { + // Keyboard callbacks + let prototype = this.prototype; - /** - * Publishes the KeyboardInterface's shorthand API names. As this assigns the current functions - * held by the longform versions, note that this should be called after replacing any of them via - * JS method extension. - * - * DOM-aware KeymanWeb should call this after its domKbdInterface.ts code is loaded, as it replaces - * a few. (This is currently done within its kmwapi.ts.) - */ - static __publishShorthandAPI() { - // Keyboard callbacks - let prototype = this.prototype; - - var exportKBCallback = function(miniName: string, longName: string) { + var exportKBCallback = function(miniName: string, longName: string) { + if(prototype[longName]) { prototype[miniName] = prototype[longName]; } - - exportKBCallback('KSF', 'saveFocus'); - exportKBCallback('KBR', 'beepReset'); - exportKBCallback('KT', 'insertText'); - exportKBCallback('KR', 'registerKeyboard'); - exportKBCallback('KRS', 'registerStub'); - exportKBCallback('KC', 'context'); - exportKBCallback('KN', 'nul'); - exportKBCallback('KCM', 'contextMatch'); - exportKBCallback('KFCM', 'fullContextMatch'); - exportKBCallback('KIK', 'isKeypress'); - exportKBCallback('KKM', 'keyMatch'); - exportKBCallback('KSM', 'stateMatch'); - exportKBCallback('KKI', 'keyInformation'); - exportKBCallback('KDM', 'deadkeyMatch'); - exportKBCallback('KB', 'beep'); - exportKBCallback('KA', 'any'); - exportKBCallback('KDC', 'deleteContext'); - exportKBCallback('KO', 'output'); - exportKBCallback('KDO', 'deadkeyOutput'); - exportKBCallback('KCXO', 'contextExOutput'); - exportKBCallback('KIO', 'indexOutput'); - exportKBCallback('KIFS', 'ifStore'); - exportKBCallback('KSETS', 'setStore'); - exportKBCallback('KLOAD', 'loadStore'); - exportKBCallback('KSAVE', 'saveStore'); } + + exportKBCallback('KSF', 'saveFocus'); + exportKBCallback('KBR', 'beepReset'); + exportKBCallback('KT', 'insertText'); + exportKBCallback('KR', 'registerKeyboard'); + exportKBCallback('KRS', 'registerStub'); + exportKBCallback('KC', 'context'); + exportKBCallback('KN', 'nul'); + exportKBCallback('KCM', 'contextMatch'); + exportKBCallback('KFCM', 'fullContextMatch'); + exportKBCallback('KIK', 'isKeypress'); + exportKBCallback('KKM', 'keyMatch'); + exportKBCallback('KSM', 'stateMatch'); + exportKBCallback('KKI', 'keyInformation'); + exportKBCallback('KDM', 'deadkeyMatch'); + exportKBCallback('KB', 'beep'); + exportKBCallback('KA', 'any'); + exportKBCallback('KDC', 'deleteContext'); + exportKBCallback('KO', 'output'); + exportKBCallback('KDO', 'deadkeyOutput'); + exportKBCallback('KCXO', 'contextExOutput'); + exportKBCallback('KIO', 'indexOutput'); + exportKBCallback('KIFS', 'ifStore'); + exportKBCallback('KSETS', 'setStore'); + exportKBCallback('KLOAD', 'loadStore'); + exportKBCallback('KSAVE', 'saveStore'); } +} - (function() { - // This will be the only call within the keyboard-processor module. - KeyboardInterface.__publishShorthandAPI(); - }()); -} \ No newline at end of file +(function() { + // This will be the only call within the keyboard-processor module. + KeyboardInterface.__publishShorthandAPI(); +}()); \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/keyEvent.ts b/common/web/keyboard-processor/src/text/keyEvent.ts index 1a16dc94bc7a51e068b0347092cf170c3be864fe..719de488b11b243b0ea9bf1362d02059fe6b59ae 100644 --- a/common/web/keyboard-processor/src/text/keyEvent.ts +++ b/common/web/keyboard-processor/src/text/keyEvent.ts @@ -1,57 +1,188 @@ -/// +// TODO: Move to separate folder: 'codes' +// We should start splitting off code needed by keyboards even without a KeyboardProcessor active. +// There's an upcoming `/common/web/types` package that 'codes' and 'keyboards' may fit well within. -namespace com.keyman.text { - // Represents a probability distribution over a keyboard's keys. - // Defined here to avoid compilation issues. - export type KeyDistribution = {keyId: string, p: number}[]; +// KeyEvent may be a _little_ bit of pollution, but this IS what the Web OSK currently generates to signal +// a key event. The most straightforward way to integrate Web OSK events on other platforms is to have +// other platforms recognize and utilize this type. + +import type Keyboard from "../keyboards/keyboard.js"; +import { type DeviceSpec } from "@keymanapp/web-utils"; + +import Codes from './codes.js'; +import DefaultRules from './defaultRules.js'; + +// Represents a probability distribution over a keyboard's keys. +// Defined here to avoid compilation issues. +export type KeyDistribution = {keyId: string, p: number}[]; + +/** + * A simple instance of the standard 'default rules' for keystroke processing from the + * DefaultRules base class. + */ +const BASE_DEFAULT_RULES = new DefaultRules(); + +export interface KeyEventSpec { + + Lcode: number; + Lstates: number; + LmodifierChange?: boolean; + Lmodifiers: number; + LisVirtualKey?: boolean; + vkCode?: number; + kName: string; + kLayer?: string; // The key's layer property + kbdLayer?: string; // The virtual keyboard's active layer + kNextLayer?: string; + + /** + * Marks the active keyboard at the time that this KeyEvent was generated by the user. + * + * Note: this is NOT equivalent to the active keyboard at the time that the event handler begins + * processing! It should be set via closure (or similar) on the event handler that can 100% + * guarantee that the keyboard instance known to the handler has not changed during JS execution + * since the user's interaction that raised the event. + */ + srcKeyboard?: Keyboard; + + // Holds relevant event properties leading to construction of this KeyEvent. + source?: any; // Technically, KeyEvent|MouseEvent|Touch - but those are DOM types that must be kept out of headless mode. + // Holds a generated fat-finger distribution (when appropriate) + keyDistribution?: KeyDistribution; + + /** + * The device model for web-core to follow when processing the keystroke. + */ + device: DeviceSpec; + + /** + * `true` if this event was produced by sources other than a DOM-based KeyboardEvent. + */ + isSynthetic?: boolean; +} + +/** + * This class is defined within its own file so that it can be loaded by code outside of KMW without + * having to actually load the entirety of KMW. + */ +export default class KeyEvent implements KeyEventSpec { + Lcode: number; + Lstates: number; + LmodifierChange?: boolean; + Lmodifiers: number; + LisVirtualKey?: boolean; + vkCode?: number; + kName: string; + kLayer?: string; // The key's layer property + kbdLayer?: string; // The virtual keyboard's active layer + kNextLayer?: string; /** - * This class is defined within its own file so that it can be loaded by code outside of KMW without - * having to actually load the entirety of KMW. + * Marks the active keyboard at the time that this KeyEvent was generated by the user. + * + * Note: this is NOT equivalent to the active keyboard at the time that the event handler begins + * processing! It should be set via closure (or similar) on the event handler that can 100% + * guarantee that the keyboard instance known to the handler has not changed during JS execution + * since the user's interaction that raised the event. */ - export class KeyEvent { - Lcode: number; - Lstates: number; - LmodifierChange?: boolean; - Lmodifiers: number; - LisVirtualKey: boolean; - vkCode: number; - kName: string; - kLayer?: string; // The key's layer property - kbdLayer?: string; // The virtual keyboard's active layer - kNextLayer?: string; - - /** - * Marks the active keyboard at the time that this KeyEvent was generated by the user. - * - * Note: this is NOT equivalent to the active keyboard at the time that the event handler begins - * processing! It should be set via closure (or similar) on the event handler that can 100% - * guarantee that the keyboard instance known to the handler has not changed during JS execution - * since the user's interaction that raised the event. - */ - srcKeyboard?: keyboards.Keyboard; - - // Holds relevant event properties leading to construction of this KeyEvent. - source?: any; // Technically, KeyEvent|MouseEvent|Touch - but those are DOM types that must be kept out of headless mode. - // Holds a generated fat-finger distribution (when appropriate) - keyDistribution?: KeyDistribution; - - /** - * The device model for web-core to follow when processing the keystroke. - */ - device: utils.DeviceSpec; - - /** - * `true` if this event was produced by sources other than a DOM-based KeyboardEvent. - */ - isSynthetic: boolean = true; - - public static constructNullKeyEvent(device: utils.DeviceSpec): KeyEvent { - const keyEvent = new KeyEvent(); - keyEvent.Lcode = 0; - keyEvent.kName = ''; - keyEvent.device = device; - return keyEvent; + srcKeyboard?: Keyboard; + + // Holds relevant event properties leading to construction of this KeyEvent. + source?: any; // Technically, KeyEvent|MouseEvent|Touch - but those are DOM types that must be kept out of headless mode. + // Holds a generated fat-finger distribution (when appropriate) + keyDistribution?: KeyDistribution; + + /** + * The device model for web-core to follow when processing the keystroke. + */ + device: DeviceSpec; + + /** + * `true` if this event was produced by sources other than a DOM-based KeyboardEvent. + */ + isSynthetic: boolean = true; + + public constructor(keyEventSpec: KeyEventSpec) { + for(let key in keyEventSpec) { + if(keyEventSpec[key] !== undefined) { + this[key] = keyEventSpec[key]; + } + } + } + + public static constructNullKeyEvent(device: DeviceSpec): KeyEvent { + const keyEvent = new KeyEvent({ + Lcode: 0, + kName: '', + device: device, + Lstates: undefined, + Lmodifiers: undefined, + vkCode: undefined, + LisVirtualKey: undefined + }); + return keyEvent; + } + + get isModifier(): boolean { + switch(this.Lcode) { + case 16: //"K_SHIFT":16,"K_CONTROL":17,"K_ALT":18 + case 17: + case 18: + case 20: //"K_CAPS":20, "K_NUMLOCK":144,"K_SCROLL":145 + case 144: + case 145: + return true; + default: + return false; + } + } + + // FIXME: makes some bad assumptions. + setMnemonicCode(shifted: boolean, capsActive: boolean) { + // K_SPACE is not handled by defaultKeyOutput for physical keystrokes unless using touch-aliased elements. + // It's also a "exception required, March 2013" for clickKey, so at least they both have this requirement. + if(this.Lcode != Codes.keyCodes['K_SPACE']) { + // So long as the key name isn't prefixed with 'U_', we'll get a default mapping based on the Lcode value. + // We need to determine the mnemonic base character - for example, SHIFT + K_PERIOD needs to map to '>'. + let mappingEvent: KeyEvent = new KeyEvent(this); + for(let key in (this as KeyEvent)) { + mappingEvent[key] = this[key]; + } + + // To facilitate storing relevant commands, we should probably reverse-lookup + // the actual keyname instead. + mappingEvent.kName = 'K_xxxx'; + mappingEvent.Lmodifiers = (shifted ? 0x10 : 0); // mnemonic lookups only exist for default & shift layers. + var mappedChar: string = BASE_DEFAULT_RULES.forAny(mappingEvent, true); + + /* First, save a backup of the original code. This one won't needlessly trigger keyboard + * rules, but allows us to replicate/emulate commands after rule processing if needed. + * (Like backspaces) + */ + this.vkCode = this.Lcode; + if(mappedChar) { + // Will return 96 for 'a', which is a keycode corresponding to Codes.keyCodes('K_NP1') - a numpad key. + // That stated, we're in mnemonic mode - this keyboard's rules are based on the char codes. + this.Lcode = mappedChar.charCodeAt(0); + } else { + // Don't let command-type keys (like K_DEL, which will output '.' otherwise!) + // trigger keyboard rules. + // + // However, DO make sure modifier keys pass through safely. + // (https://github.com/keymanapp/keyman/issues/3744) + if(!this.isModifier) { + delete this.Lcode; + } + } + } + + if(capsActive) { + // TODO: Needs fixing - does not properly mirror physical keystrokes, as Lcode range 96-111 corresponds + // to numpad keys! (Physical keyboard section has its own issues here.) + if((this.Lcode >= 65 && this.Lcode <= 90) /* 'A' - 'Z' */ || (this.Lcode >= 97 && this.Lcode <= 122) /* 'a' - 'z' */) { + this.Lmodifiers ^= 0x10; // Flip the 'shifted' bit, so it'll act as the opposite key. + this.Lcode ^= 0x20; // Flips the 'upper' vs 'lower' bit for the base 'a'-'z' ASCII alphabetics. + } } - }; -} \ No newline at end of file + } +}; \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/keyMapping.ts b/common/web/keyboard-processor/src/text/keyMapping.ts index ba7cc09868802a61e6dc09141553aea1099169f3..663eb5457f7a7bb3c0221008923f1aa11da930a4 100644 --- a/common/web/keyboard-processor/src/text/keyMapping.ts +++ b/common/web/keyboard-processor/src/text/keyMapping.ts @@ -2,185 +2,186 @@ KeymanWeb 11.0 Copyright 2019 SIL International ***/ -namespace com.keyman { - class KeyMap { - [keycode: string]: number; +import type KeyEvent from "./keyEvent.js"; +import { KeyEventSpec } from "./keyEvent.js"; + +class KeyMap { + [keycode: string]: number; +} + +class BrowserKeyMaps { + FF: KeyMap = new KeyMap(); + Safari: KeyMap = new KeyMap(); + Opera: KeyMap = new KeyMap(); + + constructor() { + // All three have been around since at least May 2014 / FF 29. + // It'd hard to find precise history, but at least that much has been confirmed. + // https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode, on Feb 26 2021. + this.FF['k61'] = 187; // = // FF 2.0 + this.FF['k59'] = 186; // ; + this.FF['k173'] = 189; // -/_ } - - class BrowserKeyMaps { - FF: KeyMap = new KeyMap(); - Safari: KeyMap = new KeyMap(); - Opera: KeyMap = new KeyMap(); - - constructor() { - // All three have been around since at least May 2014 / FF 29. - // It'd hard to find precise history, but at least that much has been confirmed. - // https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode, on Feb 26 2021. - this.FF['k61'] = 187; // = // FF 2.0 - this.FF['k59'] = 186; // ; - this.FF['k173'] = 189; // -/_ - } +} + +class LanguageKeyMaps { + [languageCode: string]: KeyMap; + + // // Here are some old legacy definitions that were no longer referenced but are likely related: + // static _BaseLayoutEuro: {[code: string]: string} = { + // 'se': '\u00a71234567890+´~~~QWERTYUIOP\u00c5\u00a8\'~~~ASDFGHJKL\u00d6\u00c4~~~~~ ` ~ + this['uk']['k192'] = 222; // ' @ => ' " + this['uk']['k222'] = 226; // # ~ => K_oE2 // I1504 - UK keyboard mixup #, \ + this['uk']['k220'] = 220; // \ | => \ | // I1504 - UK keyboard mixup #, \ } +} - class LanguageKeyMaps { - [languageCode: string]: KeyMap; - - // // Here are some old legacy definitions that were no longer referenced but are likely related: - // static _BaseLayoutEuro: {[code: string]: string} = { - // 'se': '\u00a71234567890+´~~~QWERTYUIOP\u00c5\u00a8\'~~~ASDFGHJKL\u00d6\u00c4~~~~~ ` ~ - this['uk']['k192'] = 222; // ' @ => ' " - this['uk']['k222'] = 226; // # ~ => K_oE2 // I1504 - UK keyboard mixup #, \ - this['uk']['k220'] = 220; // \ | => \ | // I1504 - UK keyboard mixup #, \ - } - } +export default class KeyMapping { + static readonly browserMap: BrowserKeyMaps = new BrowserKeyMaps(); + static readonly languageMap: LanguageKeyMaps = new LanguageKeyMaps(); - export class KeyMapping { - static readonly browserMap: BrowserKeyMaps = new BrowserKeyMaps(); - static readonly languageMap: LanguageKeyMaps = new LanguageKeyMaps(); + private static _usCharCodes: KeyMap[]; - private static _usCharCodes: KeyMap[]; + private constructor() { + // Do not construct this class. + } - private constructor() { - // Do not construct this class. - } + private static _usCodeInit() { + var s0=new KeyMap(),s1=new KeyMap(); + + s0['k192'] = 96; + s0['k49'] = 49; + s0['k50'] = 50; + s0['k51'] = 51; + s0['k52'] = 52; + s0['k53'] = 53; + s0['k54'] = 54; + s0['k55'] = 55; + s0['k56'] = 56; + s0['k57'] = 57; + s0['k48'] = 48; + s0['k189'] = 45; + s0['k187'] = 61; + s0['k81'] = 113; + s0['k87'] = 119; + s0['k69'] = 101; + s0['k82'] = 114; + s0['k84'] = 116; + s0['k89'] = 121; + s0['k85'] = 117; + s0['k73'] = 105; + s0['k79'] = 111; + s0['k80'] = 112; + s0['k219'] = 91; + s0['k221'] = 93; + s0['k220'] = 92; + s0['k65'] = 97; + s0['k83'] = 115; + s0['k68'] = 100; + s0['k70'] = 102; + s0['k71'] = 103; + s0['k72'] = 104; + s0['k74'] = 106; + s0['k75'] = 107; + s0['k76'] = 108; + s0['k186'] = 59; + s0['k222'] = 39; + s0['k90'] = 122; + s0['k88'] = 120; + s0['k67'] = 99; + s0['k86'] = 118; + s0['k66'] = 98; + s0['k78'] = 110; + s0['k77'] = 109; + s0['k188'] = 44; + s0['k190'] = 46; + s0['k191'] = 47; + + s1['k192'] = 126; + s1['k49'] = 33; + s1['k50'] = 64; + s1['k51'] = 35; + s1['k52'] = 36; + s1['k53'] = 37; + s1['k54'] = 94; + s1['k55'] = 38; + s1['k56'] = 42; + s1['k57'] = 40; + s1['k48'] = 41; + s1['k189'] = 95; + s1['k187'] = 43; + s1['k81'] = 81; + s1['k87'] = 87; + s1['k69'] = 69; + s1['k82'] = 82; + s1['k84'] = 84; + s1['k89'] = 89; + s1['k85'] = 85; + s1['k73'] = 73; + s1['k79'] = 79; + s1['k80'] = 80; + s1['k219'] = 123; + s1['k221'] = 125; + s1['k220'] = 124; + s1['k65'] = 65; + s1['k83'] = 83; + s1['k68'] = 68; + s1['k70'] = 70; + s1['k71'] = 71; + s1['k72'] = 72; + s1['k74'] = 74; + s1['k75'] = 75; + s1['k76'] = 76; + s1['k186'] = 58; + s1['k222'] = 34; + s1['k90'] = 90; + s1['k88'] = 88; + s1['k67'] = 67; + s1['k86'] = 86; + s1['k66'] = 66; + s1['k78'] = 78; + s1['k77'] = 77; + s1['k188'] = 60; + s1['k190'] = 62; + s1['k191'] = 63; + + KeyMapping._usCharCodes = [s0,s1]; + } - private static _usCodeInit() { - var s0=new KeyMap(),s1=new KeyMap(); - - s0['k192'] = 96; - s0['k49'] = 49; - s0['k50'] = 50; - s0['k51'] = 51; - s0['k52'] = 52; - s0['k53'] = 53; - s0['k54'] = 54; - s0['k55'] = 55; - s0['k56'] = 56; - s0['k57'] = 57; - s0['k48'] = 48; - s0['k189'] = 45; - s0['k187'] = 61; - s0['k81'] = 113; - s0['k87'] = 119; - s0['k69'] = 101; - s0['k82'] = 114; - s0['k84'] = 116; - s0['k89'] = 121; - s0['k85'] = 117; - s0['k73'] = 105; - s0['k79'] = 111; - s0['k80'] = 112; - s0['k219'] = 91; - s0['k221'] = 93; - s0['k220'] = 92; - s0['k65'] = 97; - s0['k83'] = 115; - s0['k68'] = 100; - s0['k70'] = 102; - s0['k71'] = 103; - s0['k72'] = 104; - s0['k74'] = 106; - s0['k75'] = 107; - s0['k76'] = 108; - s0['k186'] = 59; - s0['k222'] = 39; - s0['k90'] = 122; - s0['k88'] = 120; - s0['k67'] = 99; - s0['k86'] = 118; - s0['k66'] = 98; - s0['k78'] = 110; - s0['k77'] = 109; - s0['k188'] = 44; - s0['k190'] = 46; - s0['k191'] = 47; - - s1['k192'] = 126; - s1['k49'] = 33; - s1['k50'] = 64; - s1['k51'] = 35; - s1['k52'] = 36; - s1['k53'] = 37; - s1['k54'] = 94; - s1['k55'] = 38; - s1['k56'] = 42; - s1['k57'] = 40; - s1['k48'] = 41; - s1['k189'] = 95; - s1['k187'] = 43; - s1['k81'] = 81; - s1['k87'] = 87; - s1['k69'] = 69; - s1['k82'] = 82; - s1['k84'] = 84; - s1['k89'] = 89; - s1['k85'] = 85; - s1['k73'] = 73; - s1['k79'] = 79; - s1['k80'] = 80; - s1['k219'] = 123; - s1['k221'] = 125; - s1['k220'] = 124; - s1['k65'] = 65; - s1['k83'] = 83; - s1['k68'] = 68; - s1['k70'] = 70; - s1['k71'] = 71; - s1['k72'] = 72; - s1['k74'] = 74; - s1['k75'] = 75; - s1['k76'] = 76; - s1['k186'] = 58; - s1['k222'] = 34; - s1['k90'] = 90; - s1['k88'] = 88; - s1['k67'] = 67; - s1['k86'] = 86; - s1['k66'] = 66; - s1['k78'] = 78; - s1['k77'] = 77; - s1['k188'] = 60; - s1['k190'] = 62; - s1['k191'] = 63; - - KeyMapping._usCharCodes = [s0,s1]; + /** + * Function _USKeyCodeToCharCode + * Scope Private + * @param {Event} Levent KMW event object + * @return {number} Character code + * Description Translate keyboard codes to standard US layout codes + */ + static _USKeyCodeToCharCode(Levent: KeyEvent | KeyEventSpec) { + return KeyMapping.usCharCodes[Levent.Lmodifiers & 0x10 ? 1 : 0]['k'+Levent.Lcode]; + }; + + public static get usCharCodes() { + if(!KeyMapping._usCharCodes) { + KeyMapping._usCodeInit(); } - /** - * Function _USKeyCodeToCharCode - * Scope Private - * @param {Event} Levent KMW event object - * @return {number} Character code - * Description Translate keyboard codes to standard US layout codes - */ - static _USKeyCodeToCharCode(Levent: com.keyman.text.KeyEvent) { - return KeyMapping.usCharCodes[Levent.Lmodifiers & 0x10 ? 1 : 0]['k'+Levent.Lcode]; - }; - - public static get usCharCodes() { - if(!KeyMapping._usCharCodes) { - KeyMapping._usCodeInit(); - } - - return KeyMapping._usCharCodes; - } + return KeyMapping._usCharCodes; } } \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/keyboardProcessor.ts b/common/web/keyboard-processor/src/text/keyboardProcessor.ts index a1ad0699f03a73e4f3d633a1807895e8212654fb..ac49021d207e3a8addc5a327f308a9c2a962a45c 100644 --- a/common/web/keyboard-processor/src/text/keyboardProcessor.ts +++ b/common/web/keyboard-processor/src/text/keyboardProcessor.ts @@ -1,780 +1,598 @@ -// Establishes key-code definitions. -/// -// Defines our generalized "KeyEvent" class. -/// -// Defines the RuleBehavior keyboard-processing return object. -/// -// Defines default key handling behaviors. -/// -// Defines the keyboard wrapper object. -/// -// Defines built-in keymapping. -/// +// #region Big ol' list of imports -// Also relies on @keymanapp/web-utils, which is included via tsconfig.json. - -namespace com.keyman.text { - export type BeepHandler = (outputTarget: OutputTarget) => void; - export type LogMessageHandler = (str: string) => void; +import EventEmitter from 'eventemitter3'; - export interface VariableStoreSerializer { - loadStore(keyboardID: string, storeName: string): VariableStore; - saveStore(keyboardID: string, storeName: string, storeMap: VariableStore); - } +import Codes from "./codes.js"; +import type Keyboard from "../keyboards/keyboard.js"; +import { MinimalKeymanGlobal } from '../keyboards/keyboardHarness.js'; +import KeyEvent from "./keyEvent.js"; +import { Layouts } from "../keyboards/defaultLayouts.js"; +import type { MutableSystemStore } from "./systemStores.js"; - export interface ProcessorInitOptions { - baseLayout?: string; - variableStoreSerializer?: VariableStoreSerializer; - } +import DefaultRules, { EmulationKeystrokes } from "./defaultRules.js"; +import type OutputTarget from "./outputTarget.js"; +import { Mock } from "./outputTarget.js"; - export class KeyboardProcessor { - public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = { - baseLayout: 'us' - } +import KeyboardInterface, { SystemStoreIDs, VariableStore } from "./kbdInterface.js"; +import RuleBehavior from "./ruleBehavior.js"; - // Tracks the simulated value for supported state keys, allowing the OSK to mirror a physical keyboard for them. - // Using the exact keyCode name from the Codes definitions will allow for certain optimizations elsewhere in the code. - stateKeys = { - "K_CAPS":false, - "K_NUMLOCK":false, - "K_SCROLL":false - }; - - // Tracks the most recent modifier state information in order to quickly detect changes - // in keyboard state not otherwise captured by the hosting page in the browser. - // Needed for AltGr simulation. - modStateFlags: number = 0; - - keyboardInterface: KeyboardInterface; - - /** - * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` - * entry points. - */ - contextDevice: utils.DeviceSpec; - - baseLayout: string; - - // Callbacks for various feedback types - beepHandler?: BeepHandler; - warningLogger?: LogMessageHandler; - errorLogger?: LogMessageHandler; - - constructor(device: utils.DeviceSpec, options?: ProcessorInitOptions) { - if(!options) { - options = KeyboardProcessor.DEFAULT_OPTIONS; - } +import { DeviceSpec, globalObject } from "@keymanapp/web-utils"; - this.contextDevice = device; +// #endregion - this.baseLayout = options.baseLayout || KeyboardProcessor.DEFAULT_OPTIONS.baseLayout; - this.keyboardInterface = new KeyboardInterface(options.variableStoreSerializer); - this.installInterface(); - } - - private installInterface() { - // We must ensure that the keyboard can find the API functions at the expected place. - let globalThis = utils.getGlobalObject(); - globalThis[KeyboardInterface.GLOBAL_NAME] = this.keyboardInterface; - - // Ensure that the active keyboard is set on the keyboard interface object. - if(this.activeKeyboard) { - this.keyboardInterface.activeKeyboard = this.activeKeyboard; - } - } +// Also relies on @keymanapp/web-utils, which is included via tsconfig.json. - public get activeKeyboard(): keyboards.Keyboard { - return this.keyboardInterface.activeKeyboard; - } +export type BeepHandler = (outputTarget: OutputTarget) => void; +export type LogMessageHandler = (str: string) => void; - public set activeKeyboard(keyboard: keyboards.Keyboard) { - this.keyboardInterface.activeKeyboard = keyboard; +export interface VariableStoreSerializer { + loadStore(keyboardID: string, storeName: string): VariableStore; + saveStore(keyboardID: string, storeName: string, storeMap: VariableStore); +} - // All old deadkeys and keyboard-specific cache should immediately be invalidated - // on a keyboard change. - this.resetContext(); - } +export interface ProcessorInitOptions { + baseLayout?: string; + keyboardInterface?: KeyboardInterface; + defaultOutputRules?: DefaultRules; // Takes the class def object, not an instance thereof. +} - get layerStore(): MutableSystemStore { - return this.keyboardInterface.systemStores[KeyboardInterface.TSS_LAYER] as MutableSystemStore; - } +interface EventMap { + statekeyChange: (stateKeys: typeof KeyboardProcessor.prototype.stateKeys) => void; +} - public get newLayerStore(): MutableSystemStore { - return this.keyboardInterface.systemStores[KeyboardInterface.TSS_NEWLAYER] as MutableSystemStore; - } +export default class KeyboardProcessor extends EventEmitter { + public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = { + baseLayout: 'us', + defaultOutputRules: new DefaultRules() + }; - public get oldLayerStore(): MutableSystemStore { - return this.keyboardInterface.systemStores[KeyboardInterface.TSS_OLDLAYER] as MutableSystemStore; - } + // Tracks the simulated value for supported state keys, allowing the OSK to mirror a physical keyboard for them. + // Using the exact keyCode name from the Codes definitions will allow for certain optimizations elsewhere in the code. + stateKeys = { + "K_CAPS":false, + "K_NUMLOCK":false, + "K_SCROLL":false + }; - public get layerId(): string { - return this.layerStore.value; - } + // Tracks the most recent modifier state information in order to quickly detect changes + // in keyboard state not otherwise captured by the hosting page in the browser. + // Needed for AltGr simulation. + modStateFlags: number = 0; - // Note: will trigger an 'event' callback designed to notify the OSK of layer changes. - public set layerId(value: string) { - this.layerStore.set(value); - } + keyboardInterface: KeyboardInterface; - /** - * Get the default RuleBehavior for the specified key, attempting to mimic standard browser defaults - * where and when appropriate. - * - * @param {object} Lkc The pre-analyzed KeyEvent object - * @param {boolean} outputTarget The OutputTarget receiving the KeyEvent - * @return {string} - */ - defaultRuleBehavior(Lkc: KeyEvent, outputTarget: OutputTarget, readonly: boolean): RuleBehavior { - let preInput = Mock.from(outputTarget, readonly); - let ruleBehavior = new RuleBehavior(); - - let matched = false; - var char = ''; - var special: EmulationKeystrokes; - if(Lkc.isSynthetic || outputTarget.isSynthetic) { - matched = true; // All the conditions below result in matches until the final else, which restores the expected default - // if no match occurs. - - if(DefaultOutput.isCommand(Lkc)) { - // Note this in the rule behavior, return successfully. We'll consider applying it later. - ruleBehavior.triggersDefaultCommand = true; - - // We'd rather let the browser handle these keys, but we're using emulated keystrokes, forcing KMW - // to emulate default behavior here. - } else if((special = DefaultOutput.forSpecialEmulation(Lkc)) != null) { - switch(special) { - case EmulationKeystrokes.Backspace: - this.keyboardInterface.defaultBackspace(outputTarget); - break; - case EmulationKeystrokes.Enter: - outputTarget.handleNewlineAtCaret(); - break; - // case '\u007f': // K_DEL - // // For (possible) future implementation. - // // Would recommend (conceptually) equaling K_RIGHT + K_BKSP, the former of which would technically be a 'command'. - default: - // In case we extend the allowed set, but forget to implement its handling case above. - ruleBehavior.errorLog = "Unexpected 'special emulation' character (\\u" + (special as String).kmwCharCodeAt(0).toString(16) + ") went unhandled!"; - } - } else { - // Back to the standard default, pending normal matching. - matched = false; - } - } + /** + * Indicates the device (platform) to be used for non-keystroke events, + * such as those sent to `begin postkeystroke` and `begin newcontext` + * entry points. + */ + contextDevice: DeviceSpec; - let isMnemonic = this.activeKeyboard && this.activeKeyboard.isMnemonic; + baseLayout: string; - if(!matched) { - if((char = DefaultOutput.forAny(Lkc, isMnemonic)) != null) { - special = DefaultOutput.forSpecialEmulation(Lkc) - if(special == EmulationKeystrokes.Backspace) { - // A browser's default backspace may fail to delete both parts of an SMP character. - this.keyboardInterface.defaultBackspace(outputTarget); - } else if(special || DefaultOutput.isCommand(Lkc)) { // Filters out 'commands' like TAB. - // We only do the "for special emulation" cases under the condition above... aside from backspace - // Let the browser handle those. - return null; - } else { - this.keyboardInterface.output(0, outputTarget, char); - } - } else { - // No match, no default RuleBehavior. - return null; - } - } + defaultRules: DefaultRules; - // Shortcut things immediately if there were issues generating this rule behavior. - if(ruleBehavior.errorLog) { - return ruleBehavior; - } + // Callbacks for various feedback types + beepHandler?: BeepHandler; + warningLogger?: LogMessageHandler; + errorLogger?: LogMessageHandler; - let transcription = outputTarget.buildTranscriptionFrom(preInput, Lkc, readonly); - ruleBehavior.transcription = transcription; + constructor(device: DeviceSpec, options?: ProcessorInitOptions) { + super(); - return ruleBehavior; + if(!options) { + options = KeyboardProcessor.DEFAULT_OPTIONS; } - setSyntheticEventDefaults(Lkc: text.KeyEvent) { - // Set the flags for the state keys - for desktop devices. For touch - // devices, the only state key in use currently is Caps Lock, which is set - // when the 'caps' layer is active in ActiveKey::constructBaseKeyEvent. - if(!Lkc.device.touchable) { - Lkc.Lstates |= this.stateKeys['K_CAPS'] ? Codes.modifierCodes['CAPS'] : Codes.modifierCodes['NO_CAPS']; - Lkc.Lstates |= this.stateKeys['K_NUMLOCK'] ? Codes.modifierCodes['NUM_LOCK'] : Codes.modifierCodes['NO_NUM_LOCK']; - Lkc.Lstates |= this.stateKeys['K_SCROLL'] ? Codes.modifierCodes['SCROLL_LOCK'] : Codes.modifierCodes['NO_SCROLL_LOCK']; - } - - // Set LisVirtualKey to false to ensure that nomatch rule does fire for U_xxxx keys - if(Lkc.kName && Lkc.kName.substr(0,2) == 'U_') { - Lkc.LisVirtualKey=false; - } + this.contextDevice = device; - // Get code for non-physical keys (T_KOKAI, U_05AB etc) - if(typeof Lkc.Lcode == 'undefined') { - Lkc.Lcode = this.getVKDictionaryCode(Lkc.kName);// Updated for Build 347 - if(!Lkc.Lcode) { - // Special case for U_xxxx keys. This vk code will never be used - // in a keyboard, so we use this to ensure that keystroke processing - // occurs for the key. - Lkc.Lcode = 1; - } - } + this.baseLayout = options.baseLayout || KeyboardProcessor.DEFAULT_OPTIONS.baseLayout; + this.keyboardInterface = options.keyboardInterface || new KeyboardInterface(globalObject(), MinimalKeymanGlobal); + this.defaultRules = options.defaultOutputRules || KeyboardProcessor.DEFAULT_OPTIONS.defaultOutputRules; + } - // Handles modifier states when the OSK is emulating rightalt through the leftctrl-leftalt layer. - if((Lkc.Lmodifiers & Codes.modifierBitmasks['ALT_GR_SIM']) == Codes.modifierBitmasks['ALT_GR_SIM'] && this.activeKeyboard.emulatesAltGr) { - Lkc.Lmodifiers &= ~Codes.modifierBitmasks['ALT_GR_SIM']; - Lkc.Lmodifiers |= Codes.modifierCodes['RALT']; - } - } + public get activeKeyboard(): Keyboard { + return this.keyboardInterface.activeKeyboard; + } - constructNullKeyEvent(device: utils.DeviceSpec): KeyEvent { - const keyEvent = KeyEvent.constructNullKeyEvent(device); - this.setSyntheticEventDefaults(keyEvent); - return keyEvent; - } + public set activeKeyboard(keyboard: Keyboard) { + this.keyboardInterface.activeKeyboard = keyboard; - processNewContextEvent(device: utils.DeviceSpec, outputTarget: OutputTarget): RuleBehavior { - return this.activeKeyboard ? - this.keyboardInterface.processNewContextEvent(outputTarget, this.constructNullKeyEvent(device)) : - null; - } + // All old deadkeys and keyboard-specific cache should immediately be invalidated + // on a keyboard change. + this.resetContext(); + } - processPostKeystroke(device: utils.DeviceSpec, outputTarget: OutputTarget): RuleBehavior { - return this.activeKeyboard ? - this.keyboardInterface.processPostKeystroke(outputTarget, this.constructNullKeyEvent(device)) : - null; - } + get layerStore(): MutableSystemStore { + return this.keyboardInterface.systemStores[SystemStoreIDs.TSS_LAYER] as MutableSystemStore; + } - processKeystroke(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { - var matchBehavior: RuleBehavior; - - // Pass this key code and state to the keyboard program - if(this.activeKeyboard && keyEvent.Lcode != 0) { - /* - * The `this.installInterface()` call is insurance against something I've seen in unit tests when things break a bit. - * - * Currently, when a KMW shutdown doesn't go through properly or completely, sometimes we end up with parallel - * versions of KMW running, and an old, partially-shutdown one will "snipe" a command meant for the most-recent - * one's test. So, installing here ensures that the active Processor has its matching KeyboardInterface ready, - * even should that occur. - */ - this.installInterface(); - matchBehavior = this.keyboardInterface.processKeystroke(outputTarget, keyEvent); - } + public get newLayerStore(): MutableSystemStore { + return this.keyboardInterface.systemStores[SystemStoreIDs.TSS_NEWLAYER] as MutableSystemStore; + } - if(!matchBehavior || matchBehavior.triggerKeyDefault) { - // Restore the virtual key code if a mnemonic keyboard is being used - // If no vkCode value was stored, maintain the original Lcode value. - keyEvent.Lcode=keyEvent.vkCode || keyEvent.Lcode; - - // Handle unmapped keys, including special keys - // The following is physical layout dependent, so should be avoided if possible. All keys should be mapped. - this.keyboardInterface.activeTargetOutput = outputTarget; - - // Match against the 'default keyboard' - rules to mimic the default string output when typing in a browser. - // Many keyboards rely upon these 'implied rules'. - let defaultBehavior = this.defaultRuleBehavior(keyEvent, outputTarget, false); - if(defaultBehavior) { - if(!matchBehavior) { - matchBehavior = defaultBehavior; - } else { - matchBehavior.mergeInDefaults(defaultBehavior); - } - matchBehavior.triggerKeyDefault = false; // We've triggered it successfully. - } // If null, we must rely on something else (like the browser, in DOM-aware code) to fulfill the default. + public get oldLayerStore(): MutableSystemStore { + return this.keyboardInterface.systemStores[SystemStoreIDs.TSS_OLDLAYER] as MutableSystemStore; + } - this.keyboardInterface.activeTargetOutput = null; - } + public get layerId(): string { + return this.layerStore.value; + } - return matchBehavior; - } + // Note: will trigger an 'event' callback designed to notify the OSK of layer changes. + public set layerId(value: string) { + this.layerStore.set(value); + } - // FIXME: makes some bad assumptions. - static setMnemonicCode(Lkc: KeyEvent, shifted: boolean, capsActive: boolean) { - // K_SPACE is not handled by defaultKeyOutput for physical keystrokes unless using touch-aliased elements. - // It's also a "exception required, March 2013" for clickKey, so at least they both have this requirement. - if(Lkc.Lcode != Codes.keyCodes['K_SPACE']) { - // So long as the key name isn't prefixed with 'U_', we'll get a default mapping based on the Lcode value. - // We need to determine the mnemonic base character - for example, SHIFT + K_PERIOD needs to map to '>'. - let mappingEvent: KeyEvent = new KeyEvent(); - for(var key in Lkc) { - mappingEvent[key] = Lkc[key]; + /** + * Get the default RuleBehavior for the specified key, attempting to mimic standard browser defaults + * where and when appropriate. + * + * @param {object} Lkc The pre-analyzed KeyEvent object + * @param {boolean} outputTarget The OutputTarget receiving the KeyEvent + * @return {string} + */ + defaultRuleBehavior(Lkc: KeyEvent, outputTarget: OutputTarget, readonly: boolean): RuleBehavior { + let preInput = Mock.from(outputTarget, readonly); + let ruleBehavior = new RuleBehavior(); + + let matched = false; + var char = ''; + var special: EmulationKeystrokes; + if(Lkc.isSynthetic || outputTarget.isSynthetic) { + matched = true; // All the conditions below result in matches until the final else, which restores the expected default + // if no match occurs. + + if(this.defaultRules.isCommand(Lkc)) { + // Note this in the rule behavior, return successfully. We'll consider applying it later. + ruleBehavior.triggersDefaultCommand = true; + + // We'd rather let the browser handle these keys, but we're using emulated keystrokes, forcing KMW + // to emulate default behavior here. + } else if((special = this.defaultRules.forSpecialEmulation(Lkc)) != null) { + switch(special) { + case EmulationKeystrokes.Backspace: + this.keyboardInterface.defaultBackspace(outputTarget); + break; + case EmulationKeystrokes.Enter: + outputTarget.handleNewlineAtCaret(); + break; + // case '\u007f': // K_DEL + // // For (possible) future implementation. + // // Would recommend (conceptually) equaling K_RIGHT + K_BKSP, the former of which would technically be a 'command'. + default: + // In case we extend the allowed set, but forget to implement its handling case above. + ruleBehavior.errorLog = "Unexpected 'special emulation' character (\\u" + (special as String).kmwCharCodeAt(0).toString(16) + ") went unhandled!"; } + } else { + // Back to the standard default, pending normal matching. + matched = false; + } + } - // To facilitate storing relevant commands, we should probably reverse-lookup - // the actual keyname instead. - mappingEvent.kName = 'K_xxxx'; - mappingEvent.Lmodifiers = (shifted ? 0x10 : 0); // mnemonic lookups only exist for default & shift layers. - var mappedChar: string = DefaultOutput.forAny(mappingEvent, true); - - /* First, save a backup of the original code. This one won't needlessly trigger keyboard - * rules, but allows us to replicate/emulate commands after rule processing if needed. - * (Like backspaces) - */ - Lkc.vkCode = Lkc.Lcode; - if(mappedChar) { - // Will return 96 for 'a', which is a keycode corresponding to Codes.keyCodes('K_NP1') - a numpad key. - // That stated, we're in mnemonic mode - this keyboard's rules are based on the char codes. - Lkc.Lcode = mappedChar.charCodeAt(0); + let isMnemonic = this.activeKeyboard && this.activeKeyboard.isMnemonic; + + if(!matched) { + if((char = this.defaultRules.forAny(Lkc, isMnemonic)) != null) { + special = this.defaultRules.forSpecialEmulation(Lkc) + if(special == EmulationKeystrokes.Backspace) { + // A browser's default backspace may fail to delete both parts of an SMP character. + this.keyboardInterface.defaultBackspace(outputTarget); + } else if(special || this.defaultRules.isCommand(Lkc)) { // Filters out 'commands' like TAB. + // We only do the "for special emulation" cases under the condition above... aside from backspace + // Let the browser handle those. + return null; } else { - // Don't let command-type keys (like K_DEL, which will output '.' otherwise!) - // trigger keyboard rules. - // - // However, DO make sure modifier keys pass through safely. - // (https://github.com/keymanapp/keyman/issues/3744) - if(!KeyboardProcessor.isModifier(Lkc)) { - delete Lkc.Lcode; - } + this.keyboardInterface.output(0, outputTarget, char); } + } else { + // No match, no default RuleBehavior. + return null; } + } - if(capsActive) { - // TODO: Needs fixing - does not properly mirror physical keystrokes, as Lcode range 96-111 corresponds - // to numpad keys! (Physical keyboard section has its own issues here.) - if((Lkc.Lcode >= 65 && Lkc.Lcode <= 90) /* 'A' - 'Z' */ || (Lkc.Lcode >= 97 && Lkc.Lcode <= 122) /* 'a' - 'z' */) { - Lkc.Lmodifiers ^= 0x10; // Flip the 'shifted' bit, so it'll act as the opposite key. - Lkc.Lcode ^= 0x20; // Flips the 'upper' vs 'lower' bit for the base 'a'-'z' ASCII alphabetics. - } - } + // Shortcut things immediately if there were issues generating this rule behavior. + if(ruleBehavior.errorLog) { + return ruleBehavior; } - /** - * Get modifier key state from layer id - * - * @param {string} layerId layer id (e.g. ctrlshift) - * @return {number} modifier key state (desktop keyboards) - */ - static getModifierState(layerId: string): number { - var modifier=0; - if(layerId.indexOf('shift') >= 0) { - modifier |= Codes.modifierCodes['SHIFT']; - } + let transcription = outputTarget.buildTranscriptionFrom(preInput, Lkc, readonly); + ruleBehavior.transcription = transcription; - // The chiral checks must not be directly exclusive due each other to visual OSK feedback. - var ctrlMatched=false; - if(layerId.indexOf('leftctrl') >= 0) { - modifier |= Codes.modifierCodes['LCTRL']; - ctrlMatched=true; - } - if(layerId.indexOf('rightctrl') >= 0) { - modifier |= Codes.modifierCodes['RCTRL']; - ctrlMatched=true; - } - if(layerId.indexOf('ctrl') >= 0 && !ctrlMatched) { - modifier |= Codes.modifierCodes['CTRL']; - } + return ruleBehavior; + } - var altMatched=false; - if(layerId.indexOf('leftalt') >= 0) { - modifier |= Codes.modifierCodes['LALT']; - altMatched=true; - } - if(layerId.indexOf('rightalt') >= 0) { - modifier |= Codes.modifierCodes['RALT']; - altMatched=true; - } - if(layerId.indexOf('alt') >= 0 && !altMatched) { - modifier |= Codes.modifierCodes['ALT']; - } + processNewContextEvent(device: DeviceSpec, outputTarget: OutputTarget): RuleBehavior { + return this.activeKeyboard ? + this.keyboardInterface.processNewContextEvent(outputTarget, this.activeKeyboard.constructNullKeyEvent(device, this.stateKeys)) : + null; + } - return modifier; - } + processPostKeystroke(device: DeviceSpec, outputTarget: OutputTarget): RuleBehavior { + return this.activeKeyboard ? + this.keyboardInterface.processPostKeystroke(outputTarget, this.activeKeyboard.constructNullKeyEvent(device, this.stateKeys)) : + null; + } - /** - * Get state key state from layer id - * - * @param {string} layerId layer id (e.g. caps) - * @return {number} modifier key state (desktop keyboards) - */ - static getStateFromLayer(layerId: string): number { - var modifier=0; - - if(layerId.indexOf('caps') >= 0) { - modifier |= Codes.modifierCodes['CAPS']; - } else { - modifier |= Codes.modifierCodes['NO_CAPS']; - } + processKeystroke(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { + var matchBehavior: RuleBehavior; - return modifier; + // Pass this key code and state to the keyboard program + if(this.activeKeyboard && keyEvent.Lcode != 0) { + matchBehavior = this.keyboardInterface.processKeystroke(outputTarget, keyEvent); } - /** - * @summary Look up a custom virtual key code in the virtual key code dictionary KVKD. On first run, will build the dictionary. - * - * `VKDictionary` is constructed from the keyboard's `KVKD` member. This list is constructed - * at compile-time and is a list of 'additional' virtual key codes, starting at 256 (i.e. - * outside the range of standard virtual key codes). These additional codes are both - * `[T_xxx]` and `[U_xxxx]` custom key codes from the Keyman keyboard language. However, - * `[U_xxxx]` keys only generate an entry in `KVKD` if there is a corresponding rule that - * is associated with them in the keyboard rules. If the `[U_xxxx]` key code is only - * referenced as the id of a key in the touch layout, then it does not get an entry in - * the `KVKD` property. - * - * @private - * @param {string} keyName custom virtual key code to lookup in the dictionary - * @return {number} key code > 255 on success, or 0 if not found - */ - getVKDictionaryCode(keyName: string) { - var activeKeyboard = this.activeKeyboard; - if(!activeKeyboard.scriptObject['VKDictionary']) { - var a=[]; - if(typeof activeKeyboard.scriptObject['KVKD'] == 'string') { - // Build the VK dictionary - // TODO: Move the dictionary build into the compiler -- so compiler generates code such as following. - // Makes the VKDictionary member unnecessary. - // this.KVKD={"K_ABC":256,"K_DEF":257,...}; - var s=activeKeyboard.scriptObject['KVKD'].split(' '); - for(var i=0; i -// Defines the KeyEvent type. -/// /// +import { extendString } from "@keymanapp/web-utils"; -// Also relies on string-extensions provided by the web-utils package. - -namespace com.keyman.text { - export class TextTransform implements Transform { - readonly insert: string; - readonly deleteLeft: number; - readonly deleteRight?: number; +extendString(); - constructor(insert: string, deleteLeft: number, deleteRight?: number) { - this.insert = insert; - this.deleteLeft = deleteLeft; - this.deleteRight = deleteRight || 0; - } +// Defines deadkey management in a manner attachable to each element interface. +import type KeyEvent from "./keyEvent.js"; +import { Deadkey, DeadkeyTracker } from "./deadkeys.js"; - public static readonly nil = new TextTransform('', 0, 0); +// Also relies on string-extensions provided by the web-utils package. - public isNoOp(): boolean { - return this.insert === '' && this.deleteLeft === 0 && this.deleteRight === 0; - } +export function isEmptyTransform(transform: Transform) { + if(!transform) { + return false; + } + return transform.insert === '' && transform.deleteLeft === 0 && (transform.deleteRight ?? 0) === 0; +} + +export class TextTransform implements Transform { + readonly insert: string; + readonly deleteLeft: number; + readonly deleteRight?: number; + + constructor(insert: string, deleteLeft: number, deleteRight?: number) { + this.insert = insert; + this.deleteLeft = deleteLeft; + this.deleteRight = deleteRight || 0; } - export class Transcription { - readonly token: number; - readonly keystroke: KeyEvent; - readonly transform: Transform; - alternates: Alternate[]; // constructed after the rest of the transcription. - readonly preInput: Mock; + public static readonly nil = new TextTransform('', 0, 0); +} - private static tokenSeed: number = 0; +export class Transcription { + readonly token: number; + readonly keystroke: KeyEvent; + readonly transform: Transform; + alternates: Alternate[]; // constructed after the rest of the transcription. + readonly preInput: Mock; - constructor(keystroke: KeyEvent, transform: Transform, preInput: Mock, alternates?: Alternate[]/*, removedDks: Deadkey[], insertedDks: Deadkey[]*/) { - let token = this.token = Transcription.tokenSeed++; + private static tokenSeed: number = 0; - this.keystroke = keystroke; - this.transform = transform; - this.alternates = alternates; - this.preInput = preInput; + constructor(keystroke: KeyEvent, transform: Transform, preInput: Mock, alternates?: Alternate[]/*, removedDks: Deadkey[], insertedDks: Deadkey[]*/) { + let token = this.token = Transcription.tokenSeed++; - this.transform.id = this.token; + this.keystroke = keystroke; + this.transform = transform; + this.alternates = alternates; + this.preInput = preInput; - // Assign the ID to each alternate, as well. - if(alternates) { - alternates.forEach(function(alt) { - alt.sample.id = token; - }); - } + this.transform.id = this.token; + + // Assign the ID to each alternate, as well. + if(alternates) { + alternates.forEach(function(alt) { + alt.sample.id = token; + }); } } +} - export type Alternate = ProbabilityMass; +export type Alternate = ProbabilityMass; - export abstract class OutputTarget { - private _dks: text.DeadkeyTracker; +export default abstract class OutputTarget { + private _dks: DeadkeyTracker; - constructor() { - this._dks = new text.DeadkeyTracker(); - } + constructor() { + this._dks = new DeadkeyTracker(); + } - /** - * Signifies that this OutputTarget has no default key processing behaviors. This should be false - * for OutputTargets backed by web elements like HTMLInputElement or HTMLTextAreaElement. - */ - get isSynthetic(): boolean { - return true; - } + /** + * Signifies that this OutputTarget has no default key processing behaviors. This should be false + * for OutputTargets backed by web elements like HTMLInputElement or HTMLTextAreaElement. + */ + get isSynthetic(): boolean { + return true; + } - resetContext(): void { - this.deadkeys().clear(); - } + resetContext(): void { + this.deadkeys().clear(); + } - deadkeys(): text.DeadkeyTracker { - return this._dks; - } + deadkeys(): DeadkeyTracker { + return this._dks; + } - hasDeadkeyMatch(n: number, d: number): boolean { - return this.deadkeys().isMatch(this.getDeadkeyCaret(), n, d); - } + hasDeadkeyMatch(n: number, d: number): boolean { + return this.deadkeys().isMatch(this.getDeadkeyCaret(), n, d); + } - insertDeadkeyBeforeCaret(d: number) { - var dk: Deadkey = new Deadkey(this.getDeadkeyCaret(), d); - this.deadkeys().add(dk); - } + insertDeadkeyBeforeCaret(d: number) { + var dk: Deadkey = new Deadkey(this.getDeadkeyCaret(), d); + this.deadkeys().add(dk); + } - /** - * Should be called by each output target immediately before text mutation operations occur. - * - * Maintains solutions to old issues: I3318,I3319 - * @param {number} delta Use negative values if characters were deleted, positive if characters were added. - */ - protected adjustDeadkeys(delta: number) { - this.deadkeys().adjustPositions(this.getDeadkeyCaret(), delta); - } + /** + * Should be called by each output target immediately before text mutation operations occur. + * + * Maintains solutions to old issues: I3318,I3319 + * @param {number} delta Use negative values if characters were deleted, positive if characters were added. + */ + protected adjustDeadkeys(delta: number) { + this.deadkeys().adjustPositions(this.getDeadkeyCaret(), delta); + } - /** - * Needed to properly clone deadkeys for use with Mock element interfaces toward predictive text purposes. - * @param {object} dks An existing set of deadkeys to deep-copy for use by this element interface. - */ - protected setDeadkeys(dks: text.DeadkeyTracker) { - this._dks = dks.clone(); - } + /** + * Needed to properly clone deadkeys for use with Mock element interfaces toward predictive text purposes. + * @param {object} dks An existing set of deadkeys to deep-copy for use by this element interface. + */ + protected setDeadkeys(dks: DeadkeyTracker) { + this._dks = dks.clone(); + } - /** - * Determines the basic operations needed to reconstruct the current OutputTarget's text from the prior state specified - * by another OutputTarget based on their text and caret positions. - * - * This is designed for use as a "before and after" comparison to determine the effect of a single keyboard rule at a time. - * As such, it assumes that the caret is immediately after any inserted text. - * @param from An output target (preferably a Mock) representing the prior state of the input/output system. - */ - buildTransformFrom(original: OutputTarget): Transform { - let to = this.getText(); - let from = original.getText(); - - let fromCaret = original.getDeadkeyCaret(); - let toCaret = this.getDeadkeyCaret(); - - // Step 1: Determine the number of left-deletions. - let maxSMPLeftMatch = fromCaret < toCaret ? fromCaret : toCaret; - - // We need the corresponding non-SMP caret location in order to binary-search efficiently. - // (Examining code units is much more computationally efficient.) - let maxLeftMatch = to._kmwCodePointToCodeUnit(maxSMPLeftMatch); - - // 1.1: use a non-SMP-aware binary search to determine the divergence point. - let start = 0; - let end = maxLeftMatch; // the index AFTER the last possible matching char. - - // This search is O(maxLeftMatch). 1/2 + 1/4 + 1/8 + ... converges to = 1. - while(start < end) { - let mid = Math.floor((end+start+1) / 2); // round up (compare more) - let fromLeft = from.substr(start, mid-start); - let toLeft = to.substr(start, mid-start); - - if(fromLeft == toLeft) { - start = mid; - } else { - end = mid - 1; - } + /** + * Determines the basic operations needed to reconstruct the current OutputTarget's text from the prior state specified + * by another OutputTarget based on their text and caret positions. + * + * This is designed for use as a "before and after" comparison to determine the effect of a single keyboard rule at a time. + * As such, it assumes that the caret is immediately after any inserted text. + * @param from An output target (preferably a Mock) representing the prior state of the input/output system. + */ + buildTransformFrom(original: OutputTarget): Transform { + let to = this.getText(); + let from = original.getText(); + + let fromCaret = original.getDeadkeyCaret(); + let toCaret = this.getDeadkeyCaret(); + + // Step 1: Determine the number of left-deletions. + let maxSMPLeftMatch = fromCaret < toCaret ? fromCaret : toCaret; + + // We need the corresponding non-SMP caret location in order to binary-search efficiently. + // (Examining code units is much more computationally efficient.) + let maxLeftMatch = to._kmwCodePointToCodeUnit(maxSMPLeftMatch); + + // 1.1: use a non-SMP-aware binary search to determine the divergence point. + let start = 0; + let end = maxLeftMatch; // the index AFTER the last possible matching char. + + // This search is O(maxLeftMatch). 1/2 + 1/4 + 1/8 + ... converges to = 1. + while(start < end) { + let mid = Math.floor((end+start+1) / 2); // round up (compare more) + let fromLeft = from.substr(start, mid-start); + let toLeft = to.substr(start, mid-start); + + if(fromLeft == toLeft) { + start = mid; + } else { + end = mid - 1; } + } - // At the loop's end: `end` now holds the non-SMP-aware divergence point. - // The 'caret' is after the last matching code unit. - - // 1.2: detect a possible surrogate-pair split scenario, correcting for it - // (by moving the split before the high-surrogate) if detected. - - // If the split location is precisely on either end of the context, we can't - // have split a surrogate pair. - if(end > 0 && end < maxLeftMatch) { - let potentialHigh = from.charCodeAt(end-1); - let potentialFromLow = from.charCodeAt(end); - let potentialToLow = to.charCodeAt(end); - - // if potentialHigh is a possible high surrogate... - if(potentialHigh >= 0xD800 && potentialHigh <= 0xDBFF) { - // and at least one potential 'low' is a possible low surrogate... - let flag = potentialFromLow >= 0xDC00 && potentialFromLow <= 0xDFFF; - flag = flag || (potentialToLow >= 0XDC00 && potentialToLow <= 0xDFFF); - - // Correct the split location, moving it 'before' the high surrogate. - if(flag) { - end = end - 1; - } - } - } + // At the loop's end: `end` now holds the non-SMP-aware divergence point. + // The 'caret' is after the last matching code unit. - // 1.3: take substring from start to the split point; determine SMP-aware length. - // This yields the SMP-aware divergence index, which gives the number of left-deletes. - let newCaret = from._kmwCodeUnitToCodePoint(end); - let deletedLeft = fromCaret - newCaret; - - // Step 2: Determine the other properties. - // Since the 'after' OutputTarget's caret indicates the end of any inserted text, we - // can easily calculate the rest. - let insertedLength = toCaret - newCaret; - let delta = to._kmwSubstr(newCaret, insertedLength); - - let undeletedRight = to._kmwLength() - toCaret; - let originalRight = from._kmwLength() - fromCaret; - let deletedRight = originalRight - undeletedRight; - - // May occur when reverting a suggestion that had been applied mid-word. - if(deletedRight < 0) { - // Restores deleteRight characters. - delta = delta + to._kmwSubstr(toCaret, -deletedRight); - deletedRight = 0; - } + // 1.2: detect a possible surrogate-pair split scenario, correcting for it + // (by moving the split before the high-surrogate) if detected. + + // If the split location is precisely on either end of the context, we can't + // have split a surrogate pair. + if(end > 0 && end < maxLeftMatch) { + let potentialHigh = from.charCodeAt(end-1); + let potentialFromLow = from.charCodeAt(end); + let potentialToLow = to.charCodeAt(end); - return new TextTransform(delta, deletedLeft, deletedRight); + // if potentialHigh is a possible high surrogate... + if(potentialHigh >= 0xD800 && potentialHigh <= 0xDBFF) { + // and at least one potential 'low' is a possible low surrogate... + let flag = potentialFromLow >= 0xDC00 && potentialFromLow <= 0xDFFF; + flag = flag || (potentialToLow >= 0XDC00 && potentialToLow <= 0xDFFF); + + // Correct the split location, moving it 'before' the high surrogate. + if(flag) { + end = end - 1; + } + } } - buildTranscriptionFrom(original: OutputTarget, keyEvent: KeyEvent, readonly: boolean, alternates?: Alternate[]): Transcription { - let transform = this.buildTransformFrom(original); + // 1.3: take substring from start to the split point; determine SMP-aware length. + // This yields the SMP-aware divergence index, which gives the number of left-deletes. + let newCaret = from._kmwCodeUnitToCodePoint(end); + let deletedLeft = fromCaret - newCaret; - // If we ever decide to re-add deadkey tracking, this is the place for it. + // Step 2: Determine the other properties. + // Since the 'after' OutputTarget's caret indicates the end of any inserted text, we + // can easily calculate the rest. + let insertedLength = toCaret - newCaret; + let delta = to._kmwSubstr(newCaret, insertedLength); - return new Transcription(keyEvent, transform, Mock.from(original, readonly), alternates); - } + let undeletedRight = to._kmwLength() - toCaret; + let originalRight = from._kmwLength() - fromCaret; + let deletedRight = originalRight - undeletedRight; - /** - * Restores the `OutputTarget` to the indicated state. Designed for use with `Transcription.preInput`. - * @param original An `OutputTarget` (usually a `Mock`). - */ - restoreTo(original: OutputTarget) { - // - this.setTextBeforeCaret(original.getTextBeforeCaret()); - this.setTextAfterCaret(original.getTextAfterCaret()); - - // Also, restore the deadkeys! - this._dks = original._dks.clone(); + // May occur when reverting a suggestion that had been applied mid-word. + if(deletedRight < 0) { + // Restores deleteRight characters. + delta = delta + to._kmwSubstr(toCaret, -deletedRight); + deletedRight = 0; } - apply(transform: Transform) { - if(transform.deleteRight) { - this.setTextAfterCaret(this.getTextAfterCaret()._kmwSubstr(transform.deleteRight)); - } + return new TextTransform(delta, deletedLeft, deletedRight); + } - if(transform.deleteLeft) { - this.deleteCharsBeforeCaret(transform.deleteLeft); - } + buildTranscriptionFrom(original: OutputTarget, keyEvent: KeyEvent, readonly: boolean, alternates?: Alternate[]): Transcription { + let transform = this.buildTransformFrom(original); - if(transform.insert) { - this.insertTextBeforeCaret(transform.insert); - } + // If we ever decide to re-add deadkey tracking, this is the place for it. - // We assume that all deadkeys are invalidated after applying a Transform, since - // prediction implies we'll be completing a word, post-deadkeys. - this._dks.clear(); - } + return new Transcription(keyEvent, transform, Mock.from(original, readonly), alternates); + } + + /** + * Restores the `OutputTarget` to the indicated state. Designed for use with `Transcription.preInput`. + * @param original An `OutputTarget` (usually a `Mock`). + */ + restoreTo(original: OutputTarget) { + // + this.setTextBeforeCaret(original.getTextBeforeCaret()); + this.setTextAfterCaret(original.getTextAfterCaret()); + + // Also, restore the deadkeys! + this._dks = original._dks.clone(); + } - /** - * Helper to `restoreTo` - allows directly setting the 'before' context to that of another - * `OutputTarget`. - * @param s - */ - protected setTextBeforeCaret(s: string): void { - // This one's easy enough to provide a default implementation for. - this.deleteCharsBeforeCaret(this.getTextBeforeCaret()._kmwLength()); - this.insertTextBeforeCaret(s); + apply(transform: Transform) { + if(transform.deleteRight) { + this.setTextAfterCaret(this.getTextAfterCaret()._kmwSubstr(transform.deleteRight)); } - /** - * Helper to `restoreTo` - allows directly setting the 'after' context to that of another - * `OutputTarget`. - * @param s - */ - protected abstract setTextAfterCaret(s: string): void; - - /** - * Clears any selected text within the wrapper's element(s). - * Silently does nothing if no such text exists. - */ - abstract clearSelection(): void; - - /** - * Clears any cached selection-related state values. - */ - abstract invalidateSelection(): void; - - /** - * Indicates whether or not the underlying element has its own selection (input, textarea) - * or is part of (or possesses) the DOM's active selection. Don't confuse with isSelectionEmpty(). - * - * TODO: rename to supportsOwnSelection - */ - abstract hasSelection(): boolean; - - /** - * Returns true if there is no current selection -- that is, the selection range is empty - */ - abstract isSelectionEmpty(): boolean; - - /** - * Returns an index corresponding to the caret's position for use with deadkeys. - */ - abstract getDeadkeyCaret(): number; - - /** - * Relative to the caret, gets the current context within the wrapper's element. - */ - abstract getTextBeforeCaret(): string; - - /** - * Relative to the caret (and/or active selection), gets the element's text after the caret, - * excluding any actively selected text that would be immediately replaced upon text entry. - */ - abstract getTextAfterCaret(): string; - - /** - * Gets the element's full text, including any text that is actively selected. - */ - abstract getText(): string; - - /** - * Performs context deletions (from the left of the caret) as needed by the KeymanWeb engine and - * corrects the location of any affected deadkeys. - * - * Does not delete deadkeys (b/c KMW 1 & 2 behavior maintenance). - * @param dn The number of characters to delete. If negative, context will be left unchanged. - */ - abstract deleteCharsBeforeCaret(dn: number): void; - - /** - * Inserts text immediately before the caret's current position, moving the caret after the - * newly inserted text in the process along with any affected deadkeys. - * - * @param s Text to insert before the caret's current position. - */ - abstract insertTextBeforeCaret(s: string): void; - - /** - * Allows element-specific handling for ENTER key inputs. Conceptually, this should usually - * correspond to `insertTextBeforeCaret('\n'), but actual implementation will vary greatly among - * elements. - */ - abstract handleNewlineAtCaret(): void; - - /** - * Saves element-specific state properties prone to mutation, enabling restoration after - * text-output operations. - */ - saveProperties() { - // Most element interfaces won't need anything here. + if(transform.deleteLeft) { + this.deleteCharsBeforeCaret(transform.deleteLeft); } - /** - * Restores previously-saved element-specific state properties. Designed for use after text-output - * ops to facilitate more-seamless web-dev and user interactions. - */ - restoreProperties(){ - // Most element interfaces won't need anything here. + if(transform.insert) { + this.insertTextBeforeCaret(transform.insert); } - /** - * Generates a synthetic event on the underlying element, signalling that its value has changed. - */ - abstract doInputEvent(): void; + // We assume that all deadkeys are invalidated after applying a Transform, since + // prediction implies we'll be completing a word, post-deadkeys. + this._dks.clear(); } - // Due to some interesting requirements on compile ordering in TS, - // this needs to be in the same file as OutputTarget now. - export class Mock extends OutputTarget { - text: string; - caretIndex: number; - - constructor(text?: string, caretPos?: number) { - super(); + /** + * Helper to `restoreTo` - allows directly setting the 'before' context to that of another + * `OutputTarget`. + * @param s + */ + protected setTextBeforeCaret(s: string): void { + // This one's easy enough to provide a default implementation for. + this.deleteCharsBeforeCaret(this.getTextBeforeCaret()._kmwLength()); + this.insertTextBeforeCaret(s); + } - this.text = text ? text : ""; - var defaultLength = this.text._kmwLength(); - // Ensures that `caretPos == 0` is handled correctly. - this.caretIndex = typeof caretPos == "number" ? caretPos : defaultLength; - } + /** + * Helper to `restoreTo` - allows directly setting the 'after' context to that of another + * `OutputTarget`. + * @param s + */ + protected abstract setTextAfterCaret(s: string): void; + + /** + * Clears any selected text within the wrapper's element(s). + * Silently does nothing if no such text exists. + */ + abstract clearSelection(): void; + + /** + * Clears any cached selection-related state values. + */ + abstract invalidateSelection(): void; + + /** + * Indicates whether or not the underlying element has its own selection (input, textarea) + * or is part of (or possesses) the DOM's active selection. Don't confuse with isSelectionEmpty(). + * + * TODO: rename to supportsOwnSelection + */ + abstract hasSelection(): boolean; + + /** + * Returns true if there is no current selection -- that is, the selection range is empty + */ + abstract isSelectionEmpty(): boolean; + + /** + * Returns an index corresponding to the caret's position for use with deadkeys. + */ + abstract getDeadkeyCaret(): number; + + /** + * Relative to the caret, gets the current context within the wrapper's element. + */ + abstract getTextBeforeCaret(): string; + + /** + * Gets the element's-currently selected text. + */ + abstract getSelectedText(): string; + + /** + * Relative to the caret (and/or active selection), gets the element's text after the caret, + * excluding any actively selected text that would be immediately replaced upon text entry. + */ + abstract getTextAfterCaret(): string; + + /** + * Gets the element's full text, including any text that is actively selected. + */ + abstract getText(): string; + + /** + * Performs context deletions (from the left of the caret) as needed by the KeymanWeb engine and + * corrects the location of any affected deadkeys. + * + * Does not delete deadkeys (b/c KMW 1 & 2 behavior maintenance). + * @param dn The number of characters to delete. If negative, context will be left unchanged. + */ + abstract deleteCharsBeforeCaret(dn: number): void; + + /** + * Inserts text immediately before the caret's current position, moving the caret after the + * newly inserted text in the process along with any affected deadkeys. + * + * @param s Text to insert before the caret's current position. + */ + abstract insertTextBeforeCaret(s: string): void; + + /** + * Allows element-specific handling for ENTER key inputs. Conceptually, this should usually + * correspond to `insertTextBeforeCaret('\n'), but actual implementation will vary greatly among + * elements. + */ + abstract handleNewlineAtCaret(): void; + + /** + * Saves element-specific state properties prone to mutation, enabling restoration after + * text-output operations. + */ + saveProperties() { + // Most element interfaces won't need anything here. + } - // Clones the state of an existing EditableElement, creating a Mock version of its state. - static from(outputTarget: OutputTarget, readonly: boolean) { - let clone: Mock; + /** + * Restores previously-saved element-specific state properties. Designed for use after text-output + * ops to facilitate more-seamless web-dev and user interactions. + */ + restoreProperties(){ + // Most element interfaces won't need anything here. + } - if(outputTarget instanceof Mock) { - // Avoids the need to run expensive kmwstring.ts / `_kmwLength()` - // calculations when deep-copying Mock instances. - let priorMock = outputTarget as Mock; - clone = new Mock(priorMock.text, priorMock.caretIndex); - } else { - // If we're 'cloning' a different OutputTarget type, we don't have a - // guaranteed way to more efficiently get these values; these are the - // best methods specified by the abstraction. - - if(readonly) { - // for NewContext and PostOutput, we want the whole text - let text = outputTarget.getText(); - let afterText = outputTarget.getTextAfterCaret(); - let caretIndex = text._kmwLength() - afterText._kmwLength(); - clone = new Mock(text, caretIndex); - } else { - // We choose to ignore (rather, pre-emptively remove) any actively-selected text, - // as since it's always removed instantly during any text mutation operations. - let preText = outputTarget.getTextBeforeCaret(); - let caretIndex = preText._kmwLength(); - clone = new Mock(preText + outputTarget.getTextAfterCaret(), caretIndex); - } - } + /** + * Generates a synthetic event on the underlying element, signalling that its value has changed. + */ + abstract doInputEvent(): void; +} - // Also duplicate deadkey state! (Needed for fat-finger ops.) - clone.setDeadkeys(outputTarget.deadkeys()); +// Due to some interesting requirements on compile ordering in TS, +// this needs to be in the same file as OutputTarget now. +export class Mock extends OutputTarget { + text: string; - return clone; - } + selStart: number; + selEnd: number; + selForward: boolean = true; - clearSelection(): void { - return; - } + constructor(text?: string, caretPos?: number); + constructor(text?: string, selStart?: number, selEnd?: number); + constructor(text?: string, selStart?: number, selEnd?: number) { + super(); - invalidateSelection(): void { - return; - } + this.text = text ? text : ""; + var defaultLength = this.text._kmwLength(); - isSelectionEmpty(): boolean { - // TODO: consider if we need to maintain selection information in Mocks - return true; - } + // Ensures that `caretPos == 0` is handled correctly. + this.selStart = typeof selStart == "number" ? selStart : defaultLength; - hasSelection(): boolean { - return true; - } + // If no selection-end is set, selection length is implied to be 0. + this.selEnd = typeof selEnd == "number" ? selEnd : this.selStart; - getDeadkeyCaret(): number { - return this.caretIndex; - } + this.selForward = this.selEnd >= this.selStart; + } - setDeadkeyCaret(index: number) { - if(index < 0 || index > this.text._kmwLength()) { - throw new Error("Provided caret index is out of range."); + // Clones the state of an existing EditableElement, creating a Mock version of its state. + static from(outputTarget: OutputTarget, readonly?: boolean) { + let clone: Mock; + + if(outputTarget instanceof Mock) { + // Avoids the need to run expensive kmwstring.ts / `_kmwLength()` + // calculations when deep-copying Mock instances. + let priorMock = outputTarget as Mock; + clone = new Mock(priorMock.text, priorMock.selStart, priorMock.selEnd); + } else { + const text = outputTarget.getText(); + const textLen = text._kmwLength(); + + // If !hasSelection() + let selectionStart: number = textLen; + let selectionEnd: number = 0; + + if(outputTarget.hasSelection()) { + let beforeText = outputTarget.getTextBeforeCaret(); + let afterText = outputTarget.getTextAfterCaret(); + selectionStart = beforeText._kmwLength(); + selectionEnd = textLen - afterText._kmwLength(); } - this.caretIndex = index; - } - getTextBeforeCaret(): string { - return this.text.kmwSubstr(0, this.caretIndex); + // readonly group or not, the returned Mock remains the same. + // New-context events should act as if the caret were at the earlier-in-context + // side of the selection, same as standard keyboard rules. + clone = new Mock(text, selectionStart, selectionEnd); } - getTextAfterCaret(): string { - return this.text.kmwSubstr(this.caretIndex); - } + // Also duplicate deadkey state! (Needed for fat-finger ops.) + clone.setDeadkeys(outputTarget.deadkeys()); + + return clone; + } + + clearSelection(): void { + this.text = this.getTextBeforeCaret() + this.getTextAfterCaret(); + this.selEnd = this.selStart; + this.selForward = true; + } + + invalidateSelection(): void { + return; + } + + isSelectionEmpty(): boolean { + return this.selStart == this.selEnd; + } - getText(): string { - return this.text; + hasSelection(): boolean { + return true; + } + + getDeadkeyCaret(): number { + return this.selStart; + } + + setSelection(start: number, end?: number) { + this.selStart = start; + this.selEnd = typeof end == 'number' ? end : start; + + this.selForward = end >= start; + if(!this.selForward) { + let temp = this.selStart; + this.selStart = this.selEnd; + this.selEnd = temp; } + } - deleteCharsBeforeCaret(dn: number): void { - if(dn >= 0) { - if(dn > this.caretIndex) { - dn = this.caretIndex; - } - this.adjustDeadkeys(-dn); - this.text = this.text.kmwSubstr(0, this.caretIndex - dn) + this.getTextAfterCaret(); - this.caretIndex -= dn; + getTextBeforeCaret(): string { + return this.text.kmwSubstr(0, this.selStart); + } + + getSelectedText(): string { + return this.text.kmwSubstr(this.selStart, this.selEnd - this.selStart); + } + + getTextAfterCaret(): string { + return this.text.kmwSubstr(this.selEnd); + } + + getText(): string { + return this.text; + } + + deleteCharsBeforeCaret(dn: number): void { + if(dn >= 0) { + if(dn > this.selStart) { + dn = this.selStart; } + this.adjustDeadkeys(-dn); + this.text = this.text.kmwSubstr(0, this.selStart - dn) + this.text.kmwSubstr(this.selStart); + this.selStart -= dn; + this.selEnd -= dn; } + } - insertTextBeforeCaret(s: string): void { - this.adjustDeadkeys(s._kmwLength()); - this.text = this.getTextBeforeCaret() + s + this.getTextAfterCaret(); - this.caretIndex += s.kmwLength(); - } + insertTextBeforeCaret(s: string): void { + this.adjustDeadkeys(s._kmwLength()); + this.text = this.getTextBeforeCaret() + s + this.text.kmwSubstr(this.selStart); + this.selStart += s.kmwLength(); + this.selEnd += s.kmwLength(); + } - handleNewlineAtCaret(): void { - this.insertTextBeforeCaret('\n'); - } + handleNewlineAtCaret(): void { + this.insertTextBeforeCaret('\n'); + } - protected setTextAfterCaret(s: string): void { - this.text = this.getTextBeforeCaret() + s; - } + protected setTextAfterCaret(s: string): void { + this.text = this.getTextBeforeCaret() + s; + } - doInputEvent() { - // Mock isn't backed by an element, so it won't have any event listeners. - } + doInputEvent() { + // Mock isn't backed by an element, so it won't have any event listeners. } } \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/ruleBehavior.ts b/common/web/keyboard-processor/src/text/ruleBehavior.ts index e1bc700cf1bec75185142591a875dfa1d289fb8f..e5d669e9092dd862154524ba00baee3ae59d5682 100644 --- a/common/web/keyboard-processor/src/text/ruleBehavior.ts +++ b/common/web/keyboard-processor/src/text/ruleBehavior.ts @@ -1,133 +1,138 @@ -namespace com.keyman.text { +/// + +import KeyboardProcessor from "./keyboardProcessor.js"; +import OutputTarget, { Mock, type Transcription } from "./outputTarget.js"; +import { VariableStoreDictionary } from "../keyboards/keyboard.js"; +import type { VariableStore } from "./kbdInterface.js"; + +/** + * Represents the commands and state changes that result from a matched keyboard rule. + */ +export default class RuleBehavior { /** - * Represents the commands and state changes that result from a matched keyboard rule. + * The before-and-after Transform from matching a keyboard rule. May be `null` + * if no keyboard rules were matched for the keystroke. */ - export class RuleBehavior { - /** - * The before-and-after Transform from matching a keyboard rule. May be `null` - * if no keyboard rules were matched for the keystroke. - */ - transcription: Transcription = null; - - /** - * Indicates whether or not a BEEP command was issued by the matched keyboard rule. - */ - beep?: boolean; - - /** - * A set of changed store values triggered by the matched keyboard rule. - */ - setStore: {[id: number]: string} = {}; - - /** - * A set of variable stores with save requests triggered by the matched keyboard rule - */ - saveStore: {[name: string]: VariableStore} = {}; - - /** - * A set of variable stores with possible changes to be applied during finalization. - */ - variableStores: keyboards.VariableStoreDictionary = {}; - - /** - * Denotes a non-output default behavior; this should be evaluated later, against the true keystroke. - */ - triggersDefaultCommand: boolean = false; - - /** - * Denotes error log messages generated when attempting to generate this behavior. - */ - errorLog?: string; - - /** - * Denotes warning log messages generated when attempting to generate this behavior. - */ - warningLog?: string; - - /** - * If predictive text is active, contains a Promise returning predictive Suggestions. - */ - predictionPromise?: Promise; - - /** - * In reference to https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852: - * - * If the final group processed is a context and keystroke group (using keys), - * and there is no nomatch rule, and the keystroke is not matched in the group, - * the keystroke's default behavior should trigger, regardless of whether or not any - * rules in prior groups matched. - */ - triggerKeyDefault?: boolean; - - finalize(processor: KeyboardProcessor, outputTarget: OutputTarget, readonly: boolean) { - if(!this.transcription) { - throw "Cannot finalize a RuleBehavior with no transcription."; - } + transcription: Transcription = null; - if(processor.beepHandler && this.beep) { - processor.beepHandler(outputTarget); - } + /** + * Indicates whether or not a BEEP command was issued by the matched keyboard rule. + */ + beep?: boolean; + + /** + * A set of changed store values triggered by the matched keyboard rule. + */ + setStore: {[id: number]: string} = {}; + + /** + * A set of variable stores with save requests triggered by the matched keyboard rule + */ + saveStore: {[name: string]: VariableStore} = {}; + + /** + * A set of variable stores with possible changes to be applied during finalization. + */ + variableStores: VariableStoreDictionary = {}; + + /** + * Denotes a non-output default behavior; this should be evaluated later, against the true keystroke. + */ + triggersDefaultCommand: boolean = false; + + /** + * Denotes error log messages generated when attempting to generate this behavior. + */ + errorLog?: string; + + /** + * Denotes warning log messages generated when attempting to generate this behavior. + */ + warningLog?: string; + + /** + * If predictive text is active, contains a Promise returning predictive Suggestions. + */ + predictionPromise?: Promise; + + /** + * In reference to https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852: + * + * If the final group processed is a context and keystroke group (using keys), + * and there is no nomatch rule, and the keystroke is not matched in the group, + * the keystroke's default behavior should trigger, regardless of whether or not any + * rules in prior groups matched. + */ + triggerKeyDefault?: boolean; + + finalize(processor: KeyboardProcessor, outputTarget: OutputTarget, readonly: boolean) { + if(!this.transcription) { + throw "Cannot finalize a RuleBehavior with no transcription."; + } - for(let storeID in this.setStore) { - let sysStore = processor.keyboardInterface.systemStores[storeID]; - if(sysStore) { - try { - sysStore.set(this.setStore[storeID]); - } catch (error) { - if(processor.errorLogger) { - processor.errorLogger("Rule attempted to perform illegal operation - 'platform' may not be changed."); - } + if(processor.beepHandler && this.beep) { + processor.beepHandler(outputTarget); + } + + for(let storeID in this.setStore) { + let sysStore = processor.keyboardInterface.systemStores[storeID]; + if(sysStore) { + try { + sysStore.set(this.setStore[storeID]); + } catch (error) { + if(processor.errorLogger) { + processor.errorLogger("Rule attempted to perform illegal operation - 'platform' may not be changed."); } - } else if(processor.warningLogger) { - processor.warningLogger("Unknown store affected by keyboard rule: " + storeID); } + } else if(processor.warningLogger) { + processor.warningLogger("Unknown store affected by keyboard rule: " + storeID); } + } - processor.keyboardInterface.applyVariableStores(this.variableStores); + processor.keyboardInterface.applyVariableStores(this.variableStores); - if(processor.keyboardInterface.variableStoreSerializer) { - for(let storeID in this.saveStore) { - processor.keyboardInterface.variableStoreSerializer.saveStore(processor.activeKeyboard.id, storeID, this.saveStore[storeID]); - } + if(processor.keyboardInterface.variableStoreSerializer) { + for(let storeID in this.saveStore) { + processor.keyboardInterface.variableStoreSerializer.saveStore(processor.activeKeyboard.id, storeID, this.saveStore[storeID]); } + } - if(this.triggersDefaultCommand) { - let keyEvent = this.transcription.keystroke; - DefaultOutput.applyCommand(keyEvent, outputTarget); - } + if(this.triggersDefaultCommand) { + let keyEvent = this.transcription.keystroke; + processor.defaultRules.applyCommand(keyEvent, outputTarget); + } - if(processor.warningLogger && this.warningLog) { - processor.warningLogger(this.warningLog); - } else if(processor.errorLogger && this.errorLog) { - processor.errorLogger(this.errorLog); - } + if(processor.warningLogger && this.warningLog) { + processor.warningLogger(this.warningLog); + } else if(processor.errorLogger && this.errorLog) { + processor.errorLogger(this.errorLog); } + } - /** - * Merges default-related behaviors from another RuleBehavior into this one. Assumes that the current instance - * "came first" chronologically. Both RuleBehaviors must be sourced from the same keystroke. - * - * Intended use: merging rule-based behavior with default key behavior during scenarios like those described - * at https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852. - * - * This function does not attempt a "complete" merge for two fully-constructed RuleBehaviors! Things - * WILL break for unintended uses. - * @param other - */ - mergeInDefaults(other: RuleBehavior) { - let keystroke = this.transcription.keystroke; - let keyFromOther = other.transcription.keystroke; - if(keystroke.Lcode != keyFromOther.Lcode || keystroke.Lmodifiers != keyFromOther.Lmodifiers) { - throw "RuleBehavior default-merge not supported unless keystrokes are identical!"; - } + /** + * Merges default-related behaviors from another RuleBehavior into this one. Assumes that the current instance + * "came first" chronologically. Both RuleBehaviors must be sourced from the same keystroke. + * + * Intended use: merging rule-based behavior with default key behavior during scenarios like those described + * at https://github.com/keymanapp/keyman/pull/4350#issuecomment-768753852. + * + * This function does not attempt a "complete" merge for two fully-constructed RuleBehaviors! Things + * WILL break for unintended uses. + * @param other + */ + mergeInDefaults(other: RuleBehavior) { + let keystroke = this.transcription.keystroke; + let keyFromOther = other.transcription.keystroke; + if(keystroke.Lcode != keyFromOther.Lcode || keystroke.Lmodifiers != keyFromOther.Lmodifiers) { + throw "RuleBehavior default-merge not supported unless keystrokes are identical!"; + } - this.triggersDefaultCommand = this.triggersDefaultCommand || other.triggersDefaultCommand; + this.triggersDefaultCommand = this.triggersDefaultCommand || other.triggersDefaultCommand; - let mergingMock = Mock.from(this.transcription.preInput, false); - mergingMock.apply(this.transcription.transform); - mergingMock.apply(other.transcription.transform); + let mergingMock = Mock.from(this.transcription.preInput, false); + mergingMock.apply(this.transcription.transform); + mergingMock.apply(other.transcription.transform); - this.transcription = mergingMock.buildTranscriptionFrom(this.transcription.preInput, keystroke, false, this.transcription.alternates); - } + this.transcription = mergingMock.buildTranscriptionFrom(this.transcription.preInput, keystroke, false, this.transcription.alternates); } } \ No newline at end of file diff --git a/common/web/keyboard-processor/src/text/systemStores.ts b/common/web/keyboard-processor/src/text/systemStores.ts index 00be331b72bc341a6c96d593b50aa4723abffb97..13184baafec2f65df3fec1525631bb42d660547c 100644 --- a/common/web/keyboard-processor/src/text/systemStores.ts +++ b/common/web/keyboard-processor/src/text/systemStores.ts @@ -1,133 +1,134 @@ -namespace com.keyman.text { - /** - * Defines common behaviors associated with system stores. - */ - export abstract class SystemStore { - public readonly id: number; - - constructor(id: number) { - this.id = id; - } +import type KeyboardInterface from "./kbdInterface.js"; +import { SystemStoreIDs } from "./kbdInterface.js"; - abstract matches(value: string): boolean; +/** + * Defines common behaviors associated with system stores. + */ +export abstract class SystemStore { + public readonly id: number; - set(value: string): void { - throw new Error("System store with ID " + this.id + " may not be directly set."); - } + constructor(id: number) { + this.id = id; } - /** - * A handler designed to receive feedback whenever a system store's value is changed. - * @param source The system store being mutated, before the value change occurs. - * @param newValue The new value being set - * @returns `false` / `undefined` to allow the change, `true` to block the change. - */ - export type SystemStoreMutationHandler = (source: MutableSystemStore, newValue: string) => boolean; - - export class MutableSystemStore extends SystemStore { - private _value: string; - handler?: SystemStoreMutationHandler = null; - - constructor(id: number, defaultValue: string) { - super(id); - this._value = defaultValue; - } + abstract matches(value: string): boolean; - get value() { - return this._value; - } + set(value: string): void { + throw new Error("System store with ID " + this.id + " may not be directly set."); + } +} + +/** + * A handler designed to receive feedback whenever a system store's value is changed. + * @param source The system store being mutated, before the value change occurs. + * @param newValue The new value being set + * @returns `false` / `undefined` to allow the change, `true` to block the change. + */ +export type SystemStoreMutationHandler = (source: MutableSystemStore, newValue: string) => boolean; + +export class MutableSystemStore extends SystemStore { + private _value: string; + handler?: SystemStoreMutationHandler = null; + + constructor(id: number, defaultValue: string) { + super(id); + this._value = defaultValue; + } - matches(value: string) { - return this._value == value; - } + get value() { + return this._value; + } - set(value: string) { - // Even if things stay the same, we should still signal this. - // It's important for tracking if a rule directly set the layer - // versus if it passively remained. - if(this.handler) { - if(this.handler(this, value)) { - return; - } - } + matches(value: string) { + return this._value == value; + } - this._value = value; + set(value: string) { + // Even if things stay the same, we should still signal this. + // It's important for tracking if a rule directly set the layer + // versus if it passively remained. + if(this.handler) { + if(this.handler(this, value)) { + return; + } } + + this._value = value; } +} - /** - * Handles checks against the current platform. - */ - export class PlatformSystemStore extends SystemStore { - private readonly kbdInterface: KeyboardInterface; +/** + * Handles checks against the current platform. + */ +export class PlatformSystemStore extends SystemStore { + private readonly kbdInterface: KeyboardInterface; - constructor(keyboardInterface: KeyboardInterface) { - super(KeyboardInterface.TSS_PLATFORM); + constructor(keyboardInterface: KeyboardInterface) { + super(SystemStoreIDs.TSS_PLATFORM); - this.kbdInterface = keyboardInterface; - } + this.kbdInterface = keyboardInterface; + } - matches(value: string) { - var i,constraint,constraints=value.split(' '); - let device = this.kbdInterface.activeDevice; - - for(i=0; i { + if(window.KeymanWeb) { + window.KeymanWeb.uninstall(); + } + }) + + it('`window`, disabled rule processing', async () => { + const harness = new KeyboardHarness(window, MinimalKeymanGlobal); + let keyboardLoader = new DOMKeyboardLoader(harness); + let keyboard = await keyboardLoader.loadKeyboardFromPath('/resources/keyboards/khmer_angkor.js'); + + assert.isOk(keyboard); + assert.equal(keyboard.id, 'Keyboard_khmer_angkor'); + assert.isTrue(keyboard.isChiral); + assert.isFalse(keyboard.isCJK); + assert.isOk(window.KeymanWeb); + assert.isOk(window.keyman); + + // Should be not be modified by the keyboard load; it is not activated by default. + assert.isNotOk(harness.activeKeyboard); + + // Should be cleared post-keyboard-load. + assert.isNotOk(harness.loadedKeyboard); + }); + + it('`window`, enabled rule processing', async () => { + const harness = new KeyboardInterface(window, MinimalKeymanGlobal); + const keyboardLoader = new DOMKeyboardLoader(harness); + const keyboard = await keyboardLoader.loadKeyboardFromPath('/resources/keyboards/khmer_angkor.js'); + harness.activeKeyboard = keyboard; + + assert.isOk(keyboard); + assert.equal(keyboard.id, 'Keyboard_khmer_angkor'); + assert.isTrue(keyboard.isChiral); + assert.isFalse(keyboard.isCJK); + assert.isOk(window.KeymanWeb); + assert.isOk(window.keyman); + + // TODO: verify actual rule processing. + const nullKeyEvent = keyboard.constructNullKeyEvent(device); + const mock = new Mock(); + const result = harness.processKeystroke(mock, nullKeyEvent); + + assert.isOk(result); + assert.isOk(window.KeymanWeb); + assert.isOk(window.keyman); + + // Should be cleared post-keyboard-load. + assert.isNotOk(harness.loadedKeyboard); + }); + + it('load keyboards successfully in parallel without side effects', async () => { + let harness = new KeyboardInterface(window, MinimalKeymanGlobal); + let keyboardLoader = new DOMKeyboardLoader(harness); + + // Preload a keyboard and make it active. + const test_kbd = await keyboardLoader.loadKeyboardFromPath('/resources/keyboards/test_917.js'); + harness.activeKeyboard = test_kbd; + assert.isNotOk(harness.loadedKeyboard); + + // With an active keyboard, load three keyboards but activate none of them. + const lao_keyboard_promise = keyboardLoader.loadKeyboardFromPath('/resources/keyboards/lao_2008_basic.js'); + const khmer_keyboard_promise = keyboardLoader.loadKeyboardFromPath('/resources/keyboards/khmer_angkor.js'); + const chiral_keyboard_promise = keyboardLoader.loadKeyboardFromPath('/resources/keyboards/test_chirality.js'); + + // Sure, why not `await` out of order? + const chiral_keyboard = await chiral_keyboard_promise; + const lao_keyboard = await lao_keyboard_promise; + const khmer_keyboard = await khmer_keyboard_promise; + + assert.strictEqual(test_kbd, harness.activeKeyboard); + assert.isNotOk(harness.loadedKeyboard); + + assert.isOk(lao_keyboard); + assert.isOk(chiral_keyboard); + assert.isOk(khmer_keyboard); + + // This part provides assurance that the keyboard properly loaded. + assert.equal(lao_keyboard.id, "Keyboard_lao_2008_basic"); + assert.equal(khmer_keyboard.id, "Keyboard_khmer_angkor"); + assert.equal(chiral_keyboard.id, "Keyboard_test_chirality"); + + harness.activeKeyboard = lao_keyboard; + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/manual.conf.js b/common/web/keyboard-processor/tests/dom/manual.conf.cjs similarity index 79% rename from web/src/test/auto/manual.conf.js rename to common/web/keyboard-processor/tests/dom/manual.conf.cjs index e9ba5c97a9bf851a57fdc06f0551a75792245d17..37d982e9b5a70c983c7b64cb766adb1f00b9adfd 100644 --- a/web/src/test/auto/manual.conf.js +++ b/common/web/keyboard-processor/tests/dom/manual.conf.cjs @@ -1,5 +1,5 @@ module.exports = function(config) { - var base = require("./base.conf.js"); + var base = require("./base.conf.cjs"); var specifics = { // test results reporter to use @@ -9,7 +9,7 @@ module.exports = function(config) { // start these browsers // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['Firefox', 'Chrome', 'Edge'], // Can be specified at run-time instead! + browsers: ['Chrome'], // Can be specified at run-time instead! // Future note for us: https://www.npmjs.com/package/karma-browserstack-launcher // Concurrency level @@ -18,7 +18,7 @@ module.exports = function(config) { // level of logging // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG - // Can't set in base.conf.js b/c of constant's definition style. + // Can't set in base.conf.cjs b/c of constant's definition style. logLevel: config.LOG_INFO }; diff --git a/common/web/keyboard-processor/tests/dom/readme.md b/common/web/keyboard-processor/tests/dom/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..153b7f4a1e8bb278a6b3c74ec94740fca518a157 --- /dev/null +++ b/common/web/keyboard-processor/tests/dom/readme.md @@ -0,0 +1,5 @@ +Automated tests in this subfolder and its children are designed to facilitate simple, browser-independent +unit tests that are DOM-reliant. + +Tests for anything that may reasonably vary depending upon the browser used to run the code should go under +the "integrated" folder instead. \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/basic-engine.js b/common/web/keyboard-processor/tests/node/basic-engine.js new file mode 100644 index 0000000000000000000000000000000000000000..15fd1efc5d8449cd4e9101e19a8984674299ae72 --- /dev/null +++ b/common/web/keyboard-processor/tests/node/basic-engine.js @@ -0,0 +1,52 @@ +import { assert } from 'chai'; +import fs from 'fs'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardInterface, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; +import { KeyboardTest, NodeProctor } from '@keymanapp/recorder-core'; + +describe('Engine - Basic Simulation', function() { + let testJSONtext = fs.readFileSync(require.resolve('@keymanapp/common-test-resources/json/engine_tests/basic_lao_simulation.json')); + // Common test suite setup. + let testSuite = new KeyboardTest(JSON.parse(testJSONtext)); + + let keyboardWithHarness; + + let device = { + formFactor: 'desktop', + OS: 'windows', + browser: 'native' + } + + before(async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + let keyboard = await keyboardLoader.loadKeyboardFromPath('../../test/' + testSuite.keyboard.filename); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; + + assert.equal(keyboard.id, "Keyboard_" + testSuite.keyboard.id); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // This part provides extra assurance that the keyboard properly loaded. + assert.equal(keyboard.id, "Keyboard_lao_2008_basic"); + }); + + // Converts each test set into its own Mocha-level test. + for(let set of testSuite.inputTestSets) { + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + + if(!proctor.compatibleWithSuite(testSuite)) { + it.skip(set.toTestName() + " - Cannot run this test suite on Node."); + } else { + it(set.toTestName(), function() { + // Refresh the proctor instance at runtime. + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + set.test(proctor); + }); + } + } +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/cases/basic-init.js b/common/web/keyboard-processor/tests/node/basic-init.js similarity index 63% rename from common/web/keyboard-processor/tests/cases/basic-init.js rename to common/web/keyboard-processor/tests/node/basic-init.js index 9fb576cd9058f651a7120ed6d74ff2f6ac88a55d..8f04a6a6e8b9a70ca8ddf6f40f0be87649a66206 100644 --- a/common/web/keyboard-processor/tests/cases/basic-init.js +++ b/common/web/keyboard-processor/tests/node/basic-init.js @@ -1,11 +1,11 @@ -var assert = require('chai').assert; -var fs = require("fs"); -var vm = require("vm"); +import { assert } from 'chai'; -let KeyboardProcessor = require('../../build/index.bundled.js'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardProcessor } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. @@ -30,16 +30,16 @@ describe('KeyboardProcessor', function() { }); describe('activeKeyboard', function() { - it('is automatically set (in headless) on keyboard load', function () { + it('is automatically set (in headless) on keyboard load', async function () { let kp = new KeyboardProcessor(); // These two lines will load a keyboard from its file; headless-mode `registerKeyboard` will // automatically set the keyboard as active. - var script = new vm.Script(fs.readFileSync('../../test/resources/keyboards/khmer_angkor.js')); - script.runInThisContext(); + let keyboardLoader = new NodeKeyboardLoader(); + let keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/khmer_angkor.js')); - assert.isDefined(kp.activeKeyboard, 'Keyboard failed to register on script load'); - assert.equal('Keyboard_khmer_angkor', kp.activeKeyboard.id, 'Unexpected keyboard id found after script load'); + assert.isDefined(keyboard, 'Keyboard failed to register on script load'); + assert.equal('Keyboard_khmer_angkor', keyboard.id, 'Unexpected keyboard id found after script load'); }); }); }); diff --git a/common/web/keyboard-processor/tests/node/bundled-module.js b/common/web/keyboard-processor/tests/node/bundled-module.js new file mode 100644 index 0000000000000000000000000000000000000000..bfff937b626bfbbc99b9e4332042f55671137aac --- /dev/null +++ b/common/web/keyboard-processor/tests/node/bundled-module.js @@ -0,0 +1,50 @@ +import { assert } from "chai"; +import * as Package from "../../build/lib/index.mjs"; + +// A few small tests to ensure that the ES Module bundle was successfully constructed and is usable. + +var toSupplementaryPairString = function(code){ + var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; + var L = (code - 0x10000) % 0x400 + 0xDC00; + + return String.fromCharCode(H, L); +} + +let u = toSupplementaryPairString; + +describe('Bundled ES Module', function() { + describe('KeyboardProcessor', function () { + it('should initialize without errors', function () { + let kp = new Package.KeyboardProcessor(); + assert.isNotNull(kp); + }); + }); + + describe('Mock', () => { + it('basic functionality test', () => { + let target = new Package.Mock("aple", 2); // ap | le + target.insertTextBeforeCaret('p'); + assert.equal(target.getText(), "apple"); + }); + + it('smp test', () => { + // Is installed as a _side effect_ from importing the module. + // We could disable that and require a call of `extendString()` instead. + String.kmwEnableSupplementaryPlane(true); // Declared & defined in web-utils. + try { + let target = new Package.Mock(u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be), 2); // ap | le + target.insertTextBeforeCaret(u(0x1d5c9)); + assert.equal(target.getText(), u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be)); + } finally { + String.kmwEnableSupplementaryPlane(false); + } + }); + }); + + describe("Imported `utils`", function() { + it("should include `utils` package's Version class", () => { + let v16 = new Package.Version([16, 1]); + assert.equal(v16.toString(), "16.1"); + }); + }) +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/cases/chirality.js b/common/web/keyboard-processor/tests/node/chirality.js similarity index 87% rename from common/web/keyboard-processor/tests/cases/chirality.js rename to common/web/keyboard-processor/tests/node/chirality.js index 7d7436cf2c25a07f3007148a3d3d2bcdbc2ffee4..b7f13214d48fba09237a058440d3d6abde55f021 100644 --- a/common/web/keyboard-processor/tests/cases/chirality.js +++ b/common/web/keyboard-processor/tests/node/chirality.js @@ -1,38 +1,33 @@ -var assert = require('chai').assert; -let fs = require('fs'); -let vm = require('vm'); +import { assert } from 'chai'; +import fs from 'fs'; -let KeyboardProcessor = require('../../build/index.bundled.js'); -let KMWRecorder = require('../../../recorder/build/nodeProctor'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. -let KeyboardInterface = com.keyman.text.KeyboardInterface; -let Codes = com.keyman.text.Codes; +import { Codes, KeyboardInterface, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; +import { KeyboardTest, NodeProctor } from '@keymanapp/recorder-core'; describe('Engine - Chirality', function() { - let testJSONtext = fs.readFileSync('../../test/resources/json/engine_tests/chirality.json'); + let testJSONtext = fs.readFileSync(require.resolve('@keymanapp/common-test-resources/json/engine_tests/chirality.json')); // Common test suite setup. - let testSuite = new KMWRecorder.KeyboardTest(JSON.parse(testJSONtext)); + let testSuite = new KeyboardTest(JSON.parse(testJSONtext)); + + let keyboardWithHarness; - var keyboard; let device = { formFactor: 'desktop', OS: 'windows', browser: 'native' } - before(function() { + before(async function() { // -- START: Standard Recorder-based unit test loading boilerplate -- - // Load the keyboard. We'll need a KeyboardProcessor instance as an intermediary. - let kp = new KeyboardProcessor(); - - // These two lines will load a keyboard from its file; headless-mode `registerKeyboard` will - // automatically set the keyboard as active. - var script = new vm.Script(fs.readFileSync('../../test/' + testSuite.keyboard.filename)); - script.runInThisContext(); + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + let keyboard = await keyboardLoader.loadKeyboardFromPath('../../test/' + testSuite.keyboard.filename); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; - keyboard = kp.activeKeyboard; assert.equal(keyboard.id, "Keyboard_" + testSuite.keyboard.id); // -- END: Standard Recorder-based unit test loading boilerplate -- @@ -42,14 +37,14 @@ describe('Engine - Chirality', function() { // Converts each test set into its own Mocha-level test. for(let set of testSuite.inputTestSets) { - let proctor = new KMWRecorder.NodeProctor(keyboard, device, assert.equal); + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); if(!proctor.compatibleWithSuite(testSuite)) { it.skip(set.toTestName() + " - Cannot run this test suite on Node."); } else if(set.constraint.target == 'hardware') { it(set.toTestName(), function() { // Refresh the proctor instance at runtime. - let proctor = new KMWRecorder.NodeProctor(keyboard, device, assert.equal); + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); set.test(proctor); }); } else { @@ -92,7 +87,7 @@ describe('Engine - Chirality', function() { // We should get the same results whether or not there actually is a corresponding modifier // expected by the rule we're examining. - mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE); + let mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE); assert.equal(targetModifiers, mappedModifiers); mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE | ALT_CODE); @@ -111,7 +106,7 @@ describe('Engine - Chirality', function() { // We should get the same results whether or not there actually is a corresponding modifier // expected by the rule we're examining. - mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE); + let mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE); assert.equal(targetModifiers, mappedModifiers); let ctrlPlusAlt = ALT_CODE | CTRL_CODE; @@ -250,7 +245,7 @@ describe('Engine - Chirality', function() { let modifierTarget = VIRTUAL_KEY_CODE | ALT_CODE | LCTRL_CODE | SHIFT_CODE; - mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE | LALT_CODE | RCTRL_CODE); + let mappedModifiers = KeyboardInterface.matchModifiersToRuleChirality(initialModifiers, VIRTUAL_KEY_CODE | LALT_CODE | RCTRL_CODE); assert.equal(modifierTarget, mappedModifiers); }); }); diff --git a/common/web/keyboard-processor/tests/node/deadkeys.js b/common/web/keyboard-processor/tests/node/deadkeys.js new file mode 100644 index 0000000000000000000000000000000000000000..c7e195bc2f7170fbf9709bbbe353c7e9641c862d --- /dev/null +++ b/common/web/keyboard-processor/tests/node/deadkeys.js @@ -0,0 +1,52 @@ +import { assert } from 'chai'; +import fs from 'fs'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardInterface, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; +import { KeyboardTest, NodeProctor } from '@keymanapp/recorder-core'; + +describe('Engine - Deadkeys', function() { + let testJSONtext = fs.readFileSync(require.resolve('@keymanapp/common-test-resources/json/engine_tests/deadkeys.json')); + // Common test suite setup. + let testSuite = new KeyboardTest(JSON.parse(testJSONtext)); + + let keyboardWithHarness; + + let device = { + formFactor: 'desktop', + OS: 'windows', + browser: 'native' + } + + before(async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + let keyboard = await keyboardLoader.loadKeyboardFromPath('../../test/' + testSuite.keyboard.filename); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; + + assert.equal(keyboard.id, "Keyboard_" + testSuite.keyboard.id); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // This part provides extra assurance that the keyboard properly loaded. + assert.equal(keyboard.id, "Keyboard_test_deadkeys"); + }); + + // Converts each test set into its own Mocha-level test. + for(let set of testSuite.inputTestSets) { + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + + if(!proctor.compatibleWithSuite(testSuite)) { + it.skip(set.toTestName() + " - Cannot run this test suite on Node."); + } else { + it(set.toTestName(), function() { + // Refresh the proctor instance at runtime. + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + set.test(proctor); + }); + } + } +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/cases/engine/context.js b/common/web/keyboard-processor/tests/node/engine/context.js similarity index 97% rename from common/web/keyboard-processor/tests/cases/engine/context.js rename to common/web/keyboard-processor/tests/node/engine/context.js index 8b8fb34e9ee579633312321a077eeade82e5f42e..a1fe5863df99630d00bb13cb2236ce8b78bf8c41 100644 --- a/common/web/keyboard-processor/tests/cases/engine/context.js +++ b/common/web/keyboard-processor/tests/node/engine/context.js @@ -1,13 +1,12 @@ -var assert = require('chai').assert; -let fs = require('fs'); -let vm = require('vm'); +import { assert } from 'chai'; -let KeyboardProcessor = require('../../../build/index.bundled.js'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. +import { KeyboardInterface, KeyboardProcessor, MinimalKeymanGlobal, Mock } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; -let KMWRecorder = require('../../../../recorder/build/nodeProctor'); +import { NodeProctor, RecordedKeystrokeSequence } from '@keymanapp/recorder-core'; /* * ABOUT THIS TEST SUITE @@ -36,7 +35,7 @@ let device = { browser: 'native' } -var keyboard; +let keyboardWithHarness; var toSupplementaryPairString = function(code){ var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; @@ -60,16 +59,16 @@ function runEngineRuleSet(ruleSet, defaultNoun) { for(var j = 0; j < matchDefs.length; j++) { // Prepare the context! var matchTest = matchDefs[j]; - var ruleSeq = new KMWRecorder.RecordedKeystrokeSequence(matchTest.sequence); - let proctor = new KMWRecorder.NodeProctor(keyboard, device, assert.equal); + var ruleSeq = new RecordedKeystrokeSequence(matchTest.sequence); + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); // We want to specify the OutputTarget for this test; our actual concern is the resulting context. - var target = new com.keyman.text.Mock(); + var target = new Mock(); ruleSeq.test(proctor, target); // Now for the real test! let processor = new KeyboardProcessor(device); - processor.activeKeyboard = keyboard; + processor.keyboardInterface = keyboardWithHarness; var res = processor.keyboardInterface.fullContextMatch(ruleDef.n, target, ruleDef.rule); var msg = matchTest.msg; @@ -81,6 +80,8 @@ function runEngineRuleSet(ruleSet, defaultNoun) { } } +//#region Test Spec Definitions + // Unfortunately, at present, this is all handwritten stuff crafted from partial Recorder use. Might should make // a page that writes this format instead. Note that I've omitted the eventSpec part here, since these tests are // Node-only. @@ -980,16 +981,14 @@ var NOTANY_NUL_RULE_SET = [ NOTANY_NUL_TEST_1, NOTANY_NUL_TEST_2, NOTANY_NUL_TES // ----------- -describe('Engine - Context Matching', function() { - before(function() { - let kp = new KeyboardProcessor(device); - - // These two lines will load a keyboard from its file; headless-mode `registerKeyboard` will - // automatically set the keyboard as active. - var script = new vm.Script(fs.readFileSync('../../test/resources/keyboards/test_simple_deadkeys.js')); - script.runInThisContext(); +//#endregion - keyboard = kp.activeKeyboard; +describe('Engine - Context Matching', function() { + before(async function() { + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_simple_deadkeys.js')); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; }); // Tests "stage 1" of fullContextMatch - ensuring that a proper context index map is built. @@ -999,16 +998,16 @@ describe('Engine - Context Matching', function() { for(var j = 0; j < matchDefs.length; j++) { // Prepare the context! var ruleDef = matchDefs[j]; - var ruleSeq = new KMWRecorder.RecordedKeystrokeSequence(ruleDef.baseSequence); - let proctor = new KMWRecorder.NodeProctor(keyboard, device, assert.equal); + var ruleSeq = new RecordedKeystrokeSequence(ruleDef.baseSequence); + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); // We want to specify the OutputTarget for this test; our actual concern is the resulting context. - var target = new com.keyman.text.Mock(); + var target = new Mock(); ruleSeq.test(proctor, target); // Now for the real test! let processor = new KeyboardProcessor(device); - processor.activeKeyboard = keyboard; + processor.keyboardInterface = keyboardWithHarness; var res = processor.keyboardInterface._BuildExtendedContext(ruleDef.n, ruleDef.ln, target); assert.sameOrderedMembers(res.valContext, ruleDef.contextCache); diff --git a/common/web/keyboard-processor/tests/cases/engine/notany_context.js b/common/web/keyboard-processor/tests/node/engine/notany_context.js similarity index 70% rename from common/web/keyboard-processor/tests/cases/engine/notany_context.js rename to common/web/keyboard-processor/tests/node/engine/notany_context.js index 2c0ba3585dd4f42c4b5f604c64035f92415968fc..2329463d465f26b137e3fd8ad3f6e08b6b00d5a9 100644 --- a/common/web/keyboard-processor/tests/cases/engine/notany_context.js +++ b/common/web/keyboard-processor/tests/node/engine/notany_context.js @@ -1,15 +1,14 @@ -const assert = require('chai').assert; -const fs = require('fs'); -const vm = require('vm'); +import { assert } from 'chai'; -let KeyboardProcessor = require('../../../build/index.bundled.js'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. -global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. - // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. +import { KeyboardInterface, MinimalKeymanGlobal, Mock } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; +import { NodeProctor, RecordedKeystrokeSequence } from '@keymanapp/recorder-core'; +import { extendString } from '@keymanapp/web-utils'; -let KMWRecorder = require('../../../../recorder/build/nodeProctor'); +extendString(); // Ensure KMW's string-extension functionality is available. // Initialize supplementary plane string extensions String.kmwEnableSupplementaryPlane(false); @@ -20,14 +19,14 @@ const device = { browser: 'native' } -let keyboard; +let keyboardWithHarness; function runEngineRuleSet(ruleSet) { for(let ruleDef of ruleSet) { // Prepare the context! - const ruleSeq = new KMWRecorder.RecordedKeystrokeSequence(ruleDef); - const proctor = new KMWRecorder.NodeProctor(keyboard, device, assert.equal); - const target = new com.keyman.text.Mock(); + const ruleSeq = new RecordedKeystrokeSequence(ruleDef); + const proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + const target = new Mock(); ruleSeq.test(proctor, target); } } @@ -55,15 +54,11 @@ function runStringRuleSet(input, output) { // ----------- describe('Engine - notany() and context()', function() { - before(function() { - const kp = new KeyboardProcessor(); - - // These two lines will load a keyboard from its file; headless-mode `registerKeyboard` will - // automatically set the keyboard as active. - const script = new vm.Script(fs.readFileSync('../../test/resources/keyboards/test_917.js')); - script.runInThisContext(); - - keyboard = kp.activeKeyboard; + before(async function() { + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_917.js')); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; }); /* diff --git a/common/web/keyboard-processor/tests/cases/engine/stores.js b/common/web/keyboard-processor/tests/node/engine/stores.js similarity index 84% rename from common/web/keyboard-processor/tests/cases/engine/stores.js rename to common/web/keyboard-processor/tests/node/engine/stores.js index 530ed0c9319a89d2874b8fa69558aaae3e628c98..ade1c2bd47112b9f8d04da30e5a4b693324791ea 100644 --- a/common/web/keyboard-processor/tests/cases/engine/stores.js +++ b/common/web/keyboard-processor/tests/node/engine/stores.js @@ -1,11 +1,9 @@ -var assert = require('chai').assert; -let fs = require('fs'); -let vm = require('vm'); +import { assert } from 'chai'; -let KeyboardProcessor = require('../../../build/index.bundled.js'); +import { Keyboard, KeyboardProcessor } from '@keymanapp/keyboard-processor'; +import { extendString } from '@keymanapp/web-utils'; -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. +extendString(); let device = { formFactor: 'desktop', @@ -25,7 +23,7 @@ describe('Engine - Stores', function() { let processor = new KeyboardProcessor(device); // A 'hollow' Keyboard that only follows default rules. That said, we need a Keyboard // instance to host cache data for our exploded store tests. - processor.activeKeyboard = new com.keyman.keyboards.Keyboard(); + processor.activeKeyboard = new Keyboard(); // Function defined at top of file; creates supplementary pairs for extended Unicode codepoints. var u = toSupplementaryPairString; diff --git a/common/web/keyboard-processor/tests/node/engine/unmatched_final_group.js b/common/web/keyboard-processor/tests/node/engine/unmatched_final_group.js new file mode 100644 index 0000000000000000000000000000000000000000..81f9765da4c38668f1271e0f7f652715ca4b75f1 --- /dev/null +++ b/common/web/keyboard-processor/tests/node/engine/unmatched_final_group.js @@ -0,0 +1,41 @@ +import { assert } from 'chai'; +import fs from 'fs'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardInterface, MinimalKeymanGlobal } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; +import { KeyboardTest, NodeProctor } from '@keymanapp/recorder-core'; + +describe('Engine - Unmatched Final Groups', function() { + let testJSONtext = fs.readFileSync(require.resolve('@keymanapp/common-test-resources/json/engine_tests/ghp_enter.json')); + // Common test suite setup. + let testSuite = new KeyboardTest(JSON.parse(testJSONtext)); + + var keyboardWithHarness; + let device = { + formFactor: 'desktop', + OS: 'windows', + browser: 'native' + } + + before(async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); + const keyboard = await keyboardLoader.loadKeyboardFromPath('../../test/' + testSuite.keyboard.filename); + keyboardWithHarness = keyboardLoader.harness; + keyboardWithHarness.activeKeyboard = keyboard; + + assert.equal(keyboard.id, "Keyboard_" + testSuite.keyboard.id); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // This part provides extra assurance that the keyboard properly loaded. + assert.equal(keyboard.id, "Keyboard_galaxie_hebrew_positional"); + }); + + it('Emits default enter AND matches rule from early group', function() { + let proctor = new NodeProctor(keyboardWithHarness, device, assert.equal); + testSuite.test(proctor); + }); +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/keyboard-loading.js b/common/web/keyboard-processor/tests/node/keyboard-loading.js new file mode 100644 index 0000000000000000000000000000000000000000..4ae04351c0aacc88bb094bb7bd4dfc9bc4cb0447 --- /dev/null +++ b/common/web/keyboard-processor/tests/node/keyboard-loading.js @@ -0,0 +1,131 @@ +import { assert } from 'chai'; +import fs from 'fs'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardHarness, KeyboardInterface, KeyboardLoaderBase, MinimalKeymanGlobal, Mock } from '@keymanapp/keyboard-processor'; +import { NodeKeyboardLoader } from '@keymanapp/keyboard-processor/node-keyboard-loader'; + +describe('Headless keyboard loading', function() { + const laoPath = require.resolve('@keymanapp/common-test-resources/keyboards/lao_2008_basic.js'); + const khmerPath = require.resolve('@keymanapp/common-test-resources/keyboards/khmer_angkor.js'); + const nonKeyboardPath = require.resolve('@keymanapp/common-test-resources/index.mjs'); + // Common test suite setup. + + let device = { + formFactor: 'desktop', + OS: 'windows', + browser: 'native' + } + + describe('Minimal harness loading', () => { + it('successfully loads', async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let harness = new KeyboardHarness({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let keyboard = await keyboardLoader.loadKeyboardFromPath(laoPath); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // Asserts that the harness's loading field is cleared once the load is complete. + assert.isNotOk(harness.loadedKeyboard); + + // Asserts that the `activeKeyboard` field was not set by the operation. + assert.isNotOk(harness.activeKeyboard); + + // This part provides assurance that the keyboard properly loaded. + assert.equal(keyboard.id, "Keyboard_lao_2008_basic"); + }); + + it('cannot evaluate rules', async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let harness = new KeyboardHarness({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let keyboard = await keyboardLoader.loadKeyboardFromPath(laoPath); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // Runs a blank KeyEvent through the keyboard's rule processing... + // but via separate harness configured with a different captured global. + let ruleHarness = new KeyboardInterface({}, MinimalKeymanGlobal); + ruleHarness.activeKeyboard = keyboard; + try { + ruleHarness.processKeystroke(new Mock(), keyboard.constructNullKeyEvent(device)); + assert.fail(); + } catch (err) { + // Drives home an important detail: the 'global' object is effectively + // closure-captured. (Similar constraints may occur when experimenting with + // 'sandboxed' keyboard loading in the DOM!) + assert.equal(err.message, 'k.KKM is not a function'); + } + }); + }); + + describe('Full harness loading', () => { + it('successfully loads', async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let harness = new KeyboardInterface({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let keyboard = await keyboardLoader.loadKeyboardFromPath(laoPath); + harness.activeKeyboard = keyboard; + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // This part provides assurance that the keyboard properly loaded. + assert.equal(keyboard.id, "Keyboard_lao_2008_basic"); + }); + + it('can evaluate rules', async function() { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let harness = new KeyboardInterface({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let keyboard = await keyboardLoader.loadKeyboardFromPath(laoPath); + harness.activeKeyboard = keyboard; + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // Runs a blank KeyEvent through the keyboard's rule processing. + harness.processKeystroke(new Mock(), keyboard.constructNullKeyEvent(device)); + }); + + it('does not change the active kehboard', async function() { + let harness = new KeyboardInterface({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + const lao_keyboard = await keyboardLoader.loadKeyboardFromPath(laoPath); + assert.isNotOk(harness.activeKeyboard); + assert.isOk(lao_keyboard); + + // This part provides assurance that the keyboard properly loaded. + assert.equal(lao_keyboard.id, "Keyboard_lao_2008_basic"); + + harness.activeKeyboard = lao_keyboard; + + const khmer_keyboard = await keyboardLoader.loadKeyboardFromPath(khmerPath); + assert.strictEqual(lao_keyboard, harness.activeKeyboard); + + assert.equal(khmer_keyboard.id, "Keyboard_khmer_angkor"); + }); + + it('throws distinct errors', async function() { + const invalidPath = 'totally_invalid_path.js'; + + let harness = new KeyboardInterface({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let missingError; + try { + await keyboardLoader.loadKeyboardFromPath(invalidPath); + } catch (err) { + missingError = err; + } + assert.isOk(missingError); + + let scriptLoadError; + try { + await keyboardLoader.loadKeyboardFromPath(nonKeyboardPath); + } catch (err) { + scriptLoadError = err; + } + assert.isOk(scriptLoadError); + + // The main test: do the errors match? + assert.notEqual(scriptLoadError.message, missingError.message); + }); + }) +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/keyboard-properties.js b/common/web/keyboard-processor/tests/node/keyboard-properties.js new file mode 100644 index 0000000000000000000000000000000000000000..b6ec2dbf8d2b7fe07f8cb376bc59d68cdf3bbbb4 --- /dev/null +++ b/common/web/keyboard-processor/tests/node/keyboard-properties.js @@ -0,0 +1,98 @@ +import { assert } from 'chai'; +import fs from 'fs'; +import path from 'path'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import { KeyboardProperties, SpacebarText } from '@keymanapp/keyboard-processor'; + +describe('Keyboard Properties', function() { + let commonResourcesPackage = '@keymanapp/common-test-resources'; + let commonStubsSubpath = 'json/keyboards'; + let rootCommonStubPath = `${commonResourcesPackage}/${commonStubsSubpath}`; + + it('initialization from KMW\'s addKeyboards() API spec', () => { + // require.resolve cannot resolve a directory directly, unfortunately. + // Needs a file to 'root' the resolution mechanism. See https://github.com/nodejs/node/issues/42219. + const resolvedResourcesPath = path.dirname(require.resolve(`${commonResourcesPackage}/index.mjs`)); + const resolvedCommonStubPath = `${resolvedResourcesPath}/${commonStubsSubpath}`; + let files = fs.readdirSync(resolvedCommonStubPath); + + for(let file of files) { + let stub = JSON.parse(fs.readFileSync(`${resolvedCommonStubPath}/${file}`)); + + let dataset = []; + if(stub.languages instanceof Array) { + dataset = KeyboardProperties.fromMultilanguageAPIStub(stub); + } else { + dataset = [new KeyboardProperties(stub)]; + } + + // do verification + for(let data of dataset) { + assert.isOk(data.id); + assert.isOk(data.name); + assert.isOk(data.langId); + assert.isOk(data.langName); + // Do not make assertions on font, oskFont - those may be undefined for some keyboards. + + // Generated dynamically from the backing source data. + assert.isOk(data.displayName); + } + } + }); + + it('generates display-name text if not directly-specified', () => { + // Could convert to run on all stubs, but... this should be fine as-is. + let stub = JSON.parse(fs.readFileSync(require.resolve(`${rootCommonStubPath}/khmer_angkor.json`))); + let propObject = new KeyboardProperties(stub); + + // Without a configured SpacebarText value, will display the keyboard name. + assert.equal(propObject.displayName, propObject.name); + + KeyboardProperties.spacebarTextMode = SpacebarText.BLANK; + assert.equal(propObject.displayName, ''); + + KeyboardProperties.spacebarTextMode = SpacebarText.KEYBOARD; + assert.equal(propObject.displayName, propObject.name); + + KeyboardProperties.spacebarTextMode = SpacebarText.LANGUAGE; + assert.equal(propObject.displayName, propObject.langName); + + KeyboardProperties.spacebarTextMode = SpacebarText.LANGUAGE_KEYBOARD; + assert.isTrue(propObject.displayName.includes(propObject.langName) && propObject.displayName.includes(propObject.name)); + }); + + it('does not override directly-specified display-name text', () => { + // Could convert to run on all stubs, but... this should be fine as-is. + let stub = JSON.parse(fs.readFileSync(require.resolve(`${rootCommonStubPath}/khmer_angkor.json`))); + + const customDisplayName = "(custom)"; + let propObject = new KeyboardProperties(stub); + + propObject.displayName = customDisplayName; + + // Without a configured SpacebarText value, will display the keyboard name. + assert.equal(propObject.displayName, customDisplayName); + + propObject.spacebarTextMode = SpacebarText.BLANK; + assert.equal(propObject.displayName, customDisplayName); + + propObject.spacebarTextMode = SpacebarText.KEYBOARD; + assert.equal(propObject.displayName, customDisplayName); + + propObject.spacebarTextMode = SpacebarText.LANGUAGE; + assert.equal(propObject.displayName, customDisplayName); + + propObject.spacebarTextMode = SpacebarText.LANGUAGE_KEYBOARD; + assert.equal(propObject.displayName, customDisplayName); + + propObject.displayName = null; // clear the value + assert.notEqual(propObject.displayName, customDisplayName); + assert.isTrue(propObject.displayName.includes(propObject.langName) && propObject.displayName.includes(propObject.name)); + + propObject.displayName = customDisplayName; + assert.equal(propObject.displayName, customDisplayName); + }); +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/mocks.js b/common/web/keyboard-processor/tests/node/mocks.js new file mode 100644 index 0000000000000000000000000000000000000000..9a84e2a8cc0f126409c9f3364dab534e5108abe9 --- /dev/null +++ b/common/web/keyboard-processor/tests/node/mocks.js @@ -0,0 +1,92 @@ +import { assert } from 'chai'; +import { Mock } from '@keymanapp/keyboard-processor'; + +describe('Mocks', function() { + describe('app|les', () => { + const testMock = new Mock('apples', 3); + + it('Cloning with .from()', () => { + assert.deepEqual(Mock.from(testMock), testMock); + assert.notStrictEqual(Mock.from(testMock), testMock); + }); + + it('getText', () => { + assert.equal(testMock.getText(), 'apples'); + }); + + it('getTextBeforeCaret', () => { + assert.equal(testMock.getTextBeforeCaret(), 'app'); + }); + + it('getTextAfterCaret', () => { + assert.equal(testMock.getTextAfterCaret(), 'les'); + }); + + it('getSelectedText', () => { + assert.equal(testMock.getSelectedText(), ''); + }); + + it('isSelectionEmpty', () => { + assert.isTrue(testMock.isSelectionEmpty()); + }); + + it('clearSelection', () => { + let editMock = Mock.from(testMock); + editMock.clearSelection(); + + assert.equal(editMock.getText(), testMock.getTextBeforeCaret() + testMock.getTextAfterCaret()); + assert.equal(editMock.getTextBeforeCaret(), testMock.getTextBeforeCaret()); + assert.equal(editMock.getTextAfterCaret(), testMock.getTextAfterCaret()); + assert.isTrue(editMock.isSelectionEmpty()); + + let postClear = Mock.from(editMock); + editMock.clearSelection(); // on same object; make sure its internal selection stuff updates correctly! + assert.notStrictEqual(postClear, editMock); + assert.deepEqual(postClear, editMock); + }); + }); + + describe('app|les and ba|nanas', () => { // selection = 'les and ba' + const testMock = new Mock('apples and bananas', 3, 13); + + it('Cloning with from()', () => { + assert.deepEqual(Mock.from(testMock), testMock); + assert.notStrictEqual(Mock.from(testMock), testMock); + }); + + it('getText', () => { + assert.equal(testMock.getText(), 'apples and bananas'); + }); + + it('getTextBeforeCaret', () => { + assert.equal(testMock.getTextBeforeCaret(), 'app'); + }); + + it('getTextAfterCaret', () => { + assert.equal(testMock.getTextAfterCaret(), 'nanas'); + }); + + it('getSelectedText', () => { + assert.equal(testMock.getSelectedText(), 'les and ba'); + }); + + it('isSelectionEmpty', () => { + assert.isFalse(testMock.isSelectionEmpty()); + }); + + it('clearSelection', () => { + let editMock = Mock.from(testMock); + editMock.clearSelection(); + + assert.equal(editMock.getText(), testMock.getTextBeforeCaret() + testMock.getTextAfterCaret()); + assert.equal(editMock.getTextBeforeCaret(), testMock.getTextBeforeCaret()); + assert.equal(editMock.getTextAfterCaret(), testMock.getTextAfterCaret()); + assert.isTrue(editMock.isSelectionEmpty()); + + let postClear = Mock.from(editMock); + editMock.clearSelection(); // on same object; make sure its internal selection stuff updates correctly! + assert.notStrictEqual(postClear, editMock); + assert.deepEqual(postClear, editMock); + }); + }); +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/cases/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js similarity index 93% rename from common/web/keyboard-processor/tests/cases/transcriptions.js rename to common/web/keyboard-processor/tests/node/transcriptions.js index 11619557206280b5fd08bbb65a90d1319cfa9df4..073e0dc6b324dc0b04d679e8fdfd4c8faa0508ec 100644 --- a/common/web/keyboard-processor/tests/cases/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -1,8 +1,9 @@ -var assert = require('chai').assert; -let KeyboardProcessor = require('../../build/index.bundled.js'); +import { assert } from 'chai'; -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. +import { Mock } from '@keymanapp/keyboard-processor'; +import { extendString } from '@keymanapp/web-utils'; + +extendString(); // Ensure KMW's string-extension functionality is available. String.kmwEnableSupplementaryPlane(false); @@ -19,8 +20,6 @@ describe("Transcriptions and Transforms", function() { let smpApple = u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be); it("does not store an alias for related OutputTargets", function() { - var Mock = com.keyman.text.Mock; - // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. var target = new Mock("apple"); @@ -38,8 +37,6 @@ describe("Transcriptions and Transforms", function() { describe("Plain text operations", function() { it("handles context-free single-char output rules", function() { - var Mock = com.keyman.text.Mock; - // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. var target = new Mock("apple"); @@ -67,11 +64,9 @@ describe("Transcriptions and Transforms", function() { }); it("handles operations with moderately long text", function() { - var Mock = com.keyman.text.Mock; - var target = new Mock("The quick brown cat jumped onto the lazy dog.", 19); var original = Mock.from(target); - target.setDeadkeyCaret(30); // 19 + 11: moves it to after "onto". + target.setSelection(30); // 19 + 11: moves it to after "onto". target.deleteCharsBeforeCaret(14); // delete: "cat jumped onto" target.insertTextBeforeCaret("fox jumped over"); @@ -86,8 +81,6 @@ describe("Transcriptions and Transforms", function() { }); it("handles operations with long text", function() { - var Mock = com.keyman.text.Mock; - // Eh... had to pick SOMETHING. let text = `Did you ever hear the Tragedy of Darth Plagueis the wise? I thought not. It's not a story the Jedi would tell you. It's a Sith legend. Darth Plagueis was a @@ -116,11 +109,9 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. }); it("handles deletions around the caret without text insertion", function() { - var Mock = com.keyman.text.Mock; - var target = new Mock("apple", 2); var original = Mock.from(target); - target.setDeadkeyCaret(3); + target.setSelection(3); target.deleteCharsBeforeCaret(2); // "ale" /* It's not exactly black box, but presently we don't NEED the keyEvent object for the method to work. @@ -136,11 +127,9 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. it("handles deletions around the caret without text insertion (SMP text)", function() { try { String.kmwEnableSupplementaryPlane(true); - var Mock = com.keyman.text.Mock; - var target = new Mock(smpApple, 2); var original = Mock.from(target); - target.setDeadkeyCaret(3); + target.setSelection(3); target.deleteCharsBeforeCaret(2); // "ale" /* It's not exactly black box, but presently we don't NEED the keyEvent object for the method to work. @@ -157,13 +146,11 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. }); it("handles deletions around the caret with text insertion", function() { - var Mock = com.keyman.text.Mock; - // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. var target = new Mock("apple", 2); var original = Mock.from(target); - target.setDeadkeyCaret(3); + target.setSelection(3); target.deleteCharsBeforeCaret(2); target.insertTextBeforeCaret("PP"); // "aPPle" @@ -180,7 +167,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock("apple", 2); var original = Mock.from(target); - target.setDeadkeyCaret(4); + target.setSelection(4); target.deleteCharsBeforeCaret(3); target.insertTextBeforeCaret("P"); // "aPe" @@ -197,7 +184,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock("apple", 2); var original = Mock.from(target); - target.setDeadkeyCaret(4); + target.setSelection(4); target.deleteCharsBeforeCaret(3); target.insertTextBeforeCaret("aaaaaaaaaaaaaa"); // "aaaaaaaaaaaaaaae" @@ -214,7 +201,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock("apple", 2); var original = Mock.from(target); - target.setDeadkeyCaret(5); + target.setSelection(5); target.deleteCharsBeforeCaret(4); target.insertTextBeforeCaret("les"); // "ales" - since we've appended a letter at the very end, the whole right-hand is indeed an insertion. @@ -232,14 +219,13 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. it("handles deletions around the caret with text insertion (SMP text)", function() { try { String.kmwEnableSupplementaryPlane(true); - var Mock = com.keyman.text.Mock; // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. var target = new Mock(smpApple, 2); let smpLE = u(0x1d5c5)+u(0x1d5be); var original = Mock.from(target); - target.setDeadkeyCaret(3); + target.setSelection(3); target.deleteCharsBeforeCaret(2); target.insertTextBeforeCaret(smpLE); // "alele" @@ -259,7 +245,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock(smpApple, 2); let smpB = u(0x1d5bb); var original = Mock.from(target); - target.setDeadkeyCaret(4); + target.setSelection(4); target.deleteCharsBeforeCaret(3); target.insertTextBeforeCaret(smpB); // "aPe" @@ -276,7 +262,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock(smpApple, 2); var original = Mock.from(target); - target.setDeadkeyCaret(4); + target.setSelection(4); target.deleteCharsBeforeCaret(3); target.insertTextBeforeCaret("aaaaaaaaaaaaaa"); // "aaaaaaaaaaaaaaae" @@ -294,7 +280,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. var target = new Mock(smpApple, 2); let smpLES = u(0x1d5c5)+u(0x1d5be)+u(0x1d5cb); var original = Mock.from(target); - target.setDeadkeyCaret(5); + target.setSelection(5); target.deleteCharsBeforeCaret(4); target.insertTextBeforeCaret(smpLES); // "ales" - since we've appended a letter at the very end, the whole right-hand is indeed an insertion. @@ -316,18 +302,17 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. /*describe("Operations with deadkeys", function() { // Just one, less nuanced/subdivided; it's not a present priority for our work, but it should provide a decent basis if/when it's needed. it("Correctly recognizes deadkey set mutations", function() { - var Mock = com.keyman.text.Mock; // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. var target = new Mock("apple"); var original = Mock.from(target); - target.setDeadkeyCaret(4); + target.setSelection(4); target.insertDeadkeyBeforeCaret(0); - target.setDeadkeyCaret(1); + target.setSelection(1); target.insertDeadkeyBeforeCaret(1); - target.setDeadkeyCaret(2); + target.setSelection(2); target.insertDeadkeyBeforeCaret(2); // 'a' dk(1) 'p' dk(2) | 'p' 'l' dk(0) 'e' var original = Mock.from(target); @@ -335,7 +320,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. target.hasDeadkeyMatch(0, 2); target.deadkeys().deleteMatched(); - target.setDeadkeyCaret(3); + target.setSelection(3); target.deleteCharsBeforeCaret(2); target.insertTextBeforeCaret("b"); target.insertDeadkeyBeforeCaret(3); // In effect: 'a' dk(1) 'b' dk(3) | 'l' dk(0) 'e' diff --git a/common/web/keyboard-processor/tsconfig-base.json b/common/web/keyboard-processor/tsconfig-base.json new file mode 100644 index 0000000000000000000000000000000000000000..43ee541cbc3058dcc83f623f7b5917d11671bad9 --- /dev/null +++ b/common/web/keyboard-processor/tsconfig-base.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "inlineSources": true, + "sourceMap": true, + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "experimentalDecorators": true, + } +} diff --git a/common/web/keyboard-processor/tsconfig.common.json b/common/web/keyboard-processor/tsconfig.common.json new file mode 100644 index 0000000000000000000000000000000000000000..30fcd19eda836e2381c025c76fd9df69e32d6470 --- /dev/null +++ b/common/web/keyboard-processor/tsconfig.common.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman/", + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "experimentalDecorators": true, + "baseUrl": "./", + "outDir": "build/obj/", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" + }, + "references": [ + { "path": "../../models/types" }, + { "path": "../keyman-version/" }, + { "path": "../utils/" } + ], + "include": ["./src/**/*.ts"], + "exclude": ["./src/keyboards/loaders/**/*.ts"] +} diff --git a/common/web/keyboard-processor/tsconfig.json b/common/web/keyboard-processor/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..95071c36a70a844feb41d2e993da6d8f029304e2 --- /dev/null +++ b/common/web/keyboard-processor/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node16", + "declaration": true, + "importHelpers": true, + "inlineSources": true, + "sourceMap": true, + "sourceRoot": "keyman/", + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "experimentalDecorators": true, + "baseUrl": "./", + "outDir": "build/obj/", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" + }, + "references": [ + { "path": "./src/keyboards/loaders/tsconfig.dom.json" }, + { "path": "./src/keyboards/loaders/tsconfig.node.json" } + ], + "include": ["./src/**/*.ts"], + "exclude": ["./src/keyboards/loaders/**/*.ts"] +} diff --git a/common/web/keyman-version/build-bundler.js b/common/web/keyman-version/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..a5f26a44df9368b0fc17e5d83b36f48ff9c29c91 --- /dev/null +++ b/common/web/keyman-version/build-bundler.js @@ -0,0 +1,22 @@ +/* + * Bundles @keymanapp/keyman-version as a single-file CommonJS module for components + * still in need of it. + */ + +import esbuild from 'esbuild'; + +// Bundles to a compact ESModule +esbuild.buildSync({ + entryPoints: ['build/version.inc.js'], + bundle: true, + sourcemap: true, + //minify: true, // No need to minify a module. + //keepNames: true, + format: "cjs", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version import to resolve. + nodePaths: ['..'], + outfile: "build/version.inc.cjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/common/web/keyman-version/build.sh b/common/web/keyman-version/build.sh index 07fb6220601af5e32214e0bc8db292bbee3052f8..939709732c2f27066bfffaa0e3c3dbc865a2e0ba 100755 --- a/common/web/keyman-version/build.sh +++ b/common/web/keyman-version/build.sh @@ -30,7 +30,7 @@ builder_describe "Build the include script for current Keyman version" \ builder_describe_outputs \ configure "/node_modules" \ - build "/common/web/keyman-version/build/keyman-version.mjs" + build "/common/web/keyman-version/build/version.inc.js" builder_parse "$@" @@ -43,53 +43,41 @@ fi if builder_start_action clean; then npm run clean rm -f ./version.inc.ts + rm -f ./keyman-version.mts + rm -rf build builder_finish_action success clean fi if builder_start_action build; then - # Generate keyman-version.mts + # Generate version.inc.ts echo " - // Generated by common/web/keyman-version/build.sh - export default class KEYMAN_VERSION { - static readonly VERSION = \"$VERSION\"; - static readonly VERSION_RELEASE =\"$VERSION_RELEASE\"; - static readonly VERSION_MAJOR = \"$VERSION_MAJOR\"; - static readonly VERSION_MINOR = \"$VERSION_MINOR\"; - static readonly VERSION_PATCH = \"$VERSION_PATCH\"; - static readonly TIER =\"$TIER\"; - static readonly VERSION_TAG = \"$VERSION_TAG\"; - static readonly VERSION_WITH_TAG = \"$VERSION_WITH_TAG\"; - static readonly VERSION_ENVIRONMENT = \"$VERSION_ENVIRONMENT\"; - static readonly SENTRY_RELEASE = \"release-$VERSION_WITH_TAG\"; - } - " > ./keyman-version.mts - - # Generate version.inc.ts -- used by TypeScript code that isn't yet modular - echo " - // Generated by common/web/keyman-version/build.sh - namespace com.keyman { - export class KEYMAN_VERSION { - static readonly VERSION = \"$VERSION\"; - static readonly VERSION_RELEASE =\"$VERSION_RELEASE\"; - static readonly VERSION_MAJOR = \"$VERSION_MAJOR\"; - static readonly VERSION_MINOR = \"$VERSION_MINOR\"; - static readonly VERSION_PATCH = \"$VERSION_PATCH\"; - static readonly TIER =\"$TIER\"; - static readonly VERSION_TAG = \"$VERSION_TAG\"; - static readonly VERSION_WITH_TAG = \"$VERSION_WITH_TAG\"; - static readonly VERSION_ENVIRONMENT = \"$VERSION_ENVIRONMENT\"; - static readonly VERSION_GIT_TAG = \"$VERSION_GIT_TAG\"; - } - } +// Generated by common/web/keyman-version/build.sh +// +// Note: does not use the 'default' keyword so that the export name is +// correct when converted to a CommonJS module with \`esbuild\`. +export class KEYMAN_VERSION { + static readonly VERSION = \"$VERSION\"; + static readonly VERSION_RELEASE =\"$VERSION_RELEASE\"; + static readonly VERSION_MAJOR = \"$VERSION_MAJOR\"; + static readonly VERSION_MINOR = \"$VERSION_MINOR\"; + static readonly VERSION_PATCH = \"$VERSION_PATCH\"; + static readonly TIER =\"$TIER\"; + static readonly VERSION_TAG = \"$VERSION_TAG\"; + static readonly VERSION_WITH_TAG = \"$VERSION_WITH_TAG\"; + static readonly VERSION_ENVIRONMENT = \"$VERSION_ENVIRONMENT\"; + static readonly VERSION_GIT_TAG = \"$VERSION_GIT_TAG\"; +} + +// Also provides it as a 'default' export. +export default KEYMAN_VERSION; " > ./version.inc.ts - # Note: in a dependency build, we'll expect keyman-version to be built by tsc -b - if builder_is_dep_build; then - builder_echo "skipping tsc -b; will be completed by $builder_dep_parent" - else - echo 'Building @keymanapp/keyman-version' - tsc -b $builder_verbose - fi + tsc -b $builder_verbose + # kmlmc (the lexical model compiler) relies on a Node-based import, but after some of the earlier + # ES-modularization work, our main output's an ES module. Fortunately, esbuild can provide an easy stopgap. + + # Generates a CommonJS variant (in case other modules still need it). + node ./build-bundler.js builder_finish_action success build fi diff --git a/common/web/keyman-version/index.ts b/common/web/keyman-version/index.ts deleted file mode 100644 index 64d0ff3da8c4d70998ffcce1aa4bfd60ed53de1f..0000000000000000000000000000000000000000 --- a/common/web/keyman-version/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/// - -// Aliases the namespaced type with the non-namespaced module export. -declare let KEYMAN_VERSION: typeof com.keyman.KEYMAN_VERSION; - -// Add all namespaces defined here to the global scope: -if (typeof module != 'undefined' && typeof module.exports != 'undefined') { - module.exports['KEYMAN_VERSION'] = com.keyman.KEYMAN_VERSION; -} diff --git a/common/web/keyman-version/package.json b/common/web/keyman-version/package.json index ef94e58750ad4187925ff01f19b54b702d56e426..b6f5cfd0cf3a3702feaeac21a8ab717ad6b4653f 100644 --- a/common/web/keyman-version/package.json +++ b/common/web/keyman-version/package.json @@ -1,21 +1,24 @@ { "name": "@keymanapp/keyman-version", "description": "Keyman global version data", - "main": "./build/index.js", + "main": "./build/version.inc.cjs", "exports": { ".": { - "import": "./build/keyman-version.mjs", - "require": "./build/index.js" - } + "import": "./build/version.inc.js", + "require": "./build/version.inc.cjs" + }, + "./build/version.inc.cjs": "./build/version.inc.cjs" }, "files": [ "/build/" ], "scripts": { "build": "echo 'Building @keymanapp/keyman-version' && tsc -b", - "clean": "tsc -b --clean" + "clean": "tsc -b --clean", + "tsc": "tsc" }, "license": "MIT", + "type": "module", "devDependencies": { "@types/node": "^18.7.13", "typescript": "^4.9.5" diff --git a/common/web/keyman-version/tsconfig.esm.json b/common/web/keyman-version/tsconfig.esm.json deleted file mode 100644 index 1e72269873081357583fa68b418e2b4a7f674b3f..0000000000000000000000000000000000000000 --- a/common/web/keyman-version/tsconfig.esm.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "extends": "../../../tsconfig-base.json", - - "compilerOptions": { - "allowJs": false, - "declaration": true, - "module": "es2020", - "outDir": "./build/", - "baseUrl": ".", - "rootDir": ".", - "sourceMap": true, - "lib": ["es6"], - "target": "es6", - }, - "include": [ - "keyman-version.mts" - ] -} diff --git a/common/web/keyman-version/tsconfig.json b/common/web/keyman-version/tsconfig.json index 18ed6ce938f0472eb873df27a472e57a4acaa718..8618c6dd9223f5ae597061a2feff2f81d510bbbc 100644 --- a/common/web/keyman-version/tsconfig.json +++ b/common/web/keyman-version/tsconfig.json @@ -4,15 +4,15 @@ "compilerOptions": { "allowJs": false, "declaration": true, - "module": "none", - "outFile": "./build/index.js", + "module": "es6", "sourceMap": true, + "outDir": "./build", "lib": ["es6"], "target": "es5", "downlevelIteration": true, + "rootDir": "." }, "include": [ - "index.ts", "version.inc.ts" ] } diff --git a/common/web/lm-worker/build-bundler.js b/common/web/lm-worker/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..eb7bb5ea415abc36951f2fcf8ffa96dec06de646 --- /dev/null +++ b/common/web/lm-worker/build-bundler.js @@ -0,0 +1,144 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import fs from 'fs'; +import { determineNeededDowncompileHelpers, buildTslibTreeshaker } from '@keymanapp/tslib/esbuild-tools'; + +/* + * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - + * the `@class` emit comment-annotation is designed to facilitate tree-shaking for ES5-targeted + * down-level emits. `esbuild` doesn't look for it by default... but we can override that with + * this plugin. + */ +let es5ClassAnnotationAsPurePlugin = { + name: '@class -> __PURE__', + setup(build) { + build.onLoad({filter: /\.js$/ }, async (args) => { + let source = await fs.promises.readFile(args.path, 'utf8'); + return { + // Marks any classes compiled by TS (as per the /** @class */ annotation) + // as __PURE__ in order to facilitate tree-shaking. + contents: source.replace('/** @class */', '/* @__PURE__ */ /** @class */'), + loader: 'js' + } + }); + } +} + +let EMIT_FILESIZE_PROFILE = false; + +if(process.argv.length > 2) { + for(let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + + switch(arg) { + case '': + break; + case '--ci': + EMIT_FILESIZE_PROFILE=true + break; + // May add other options if desired in the future. + default: + console.error("Invalid command-line option set for script; only --ci is permitted."); + process.exit(1); + } + } +} + +await esbuild.build({ + bundle: true, + sourcemap: true, + /* + * https://esbuild.github.io/api/#sources-content would theoretically allow us to strip the source + * while still keeping info useful for stack-tracing... but it doesn't pass through the sourcemap + * concatenation setup. + * + * That said, we know how to 'nix it ourselves in post now, so... yeah. + */ + sourcesContent: true, + sourceRoot: "/", + format: "esm", + nodePaths: ['..', '../../models'], + entryPoints: { + 'index': 'build/obj/index.js', + 'worker-main': 'build/obj/worker-main.js' + }, + outdir: 'build/lib', + outExtension: { '.js': '.mjs' }, + plugins: [ es5ClassAnnotationAsPurePlugin ], + tsconfig: 'tsconfig.json', + target: "es5", +}); + +// Bundled CommonJS (classic Node) module version +await esbuild.build({ + bundle: true, + sourcemap: true, + format: "cjs", + nodePaths: ['..'], + entryPoints: { + 'index': 'build/obj/index.js', + 'worker-main': 'build/obj/worker-main.js' + }, + outdir: 'build/lib', + outExtension: { '.js': '.cjs' }, + plugins: [ es5ClassAnnotationAsPurePlugin ], + tsconfig: 'tsconfig.json', + target: "es5" +}); + +// The one that's actually a component of our releases. + +const embeddedWorkerBuildOptions = { + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + format: "iife", + nodePaths: ['..'], + entryPoints: { + 'worker-main': 'build/obj/worker-main.js' + }, + outdir: 'build/lib', + plugins: [ es5ClassAnnotationAsPurePlugin ], + tsconfig: 'tsconfig.json', + target: "es5" +} + +// Prepare the needed setup for `tslib` treeshaking. +const unusedHelpers = await determineNeededDowncompileHelpers(embeddedWorkerBuildOptions); +embeddedWorkerBuildOptions.plugins = [buildTslibTreeshaker(unusedHelpers), ...embeddedWorkerBuildOptions.plugins]; + +// Direct-use version +await esbuild.build(embeddedWorkerBuildOptions); + +// ------------------------ + +// Now to generate a filesize profile for the minified version of the worker. +// We want a specialized bundle build here instead; no output, but minified like +// the actual release worker. +const minifiedProfilingOptions = { + ...embeddedWorkerBuildOptions, + minify: true, + metafile: true, + write: false // don't actually write the file. +} + +let result = await esbuild.build(minifiedProfilingOptions); +let filesizeProfile = await esbuild.analyzeMetafile(result.metafile, { verbose: true }); +fs.writeFileSync('build/filesize-profile.log', ` +// Minified Worker filesize profile, before polyfilling +${filesizeProfile} +`); + +if(EMIT_FILESIZE_PROFILE) { + console.log("Minified, pre-polyfill worker filesize profile:"); + // Profiles the sourcecode! + console.log(filesizeProfile); +} diff --git a/common/web/lm-worker/build-polyfill-concatenator.js b/common/web/lm-worker/build-polyfill-concatenator.js new file mode 100644 index 0000000000000000000000000000000000000000..52b24b910dfbe29aff7c7444e38f835b05b1bd23 --- /dev/null +++ b/common/web/lm-worker/build-polyfill-concatenator.js @@ -0,0 +1,151 @@ +import fs from 'fs'; + +import SourceMapRemapper from '@keymanapp/sourcemap-path-remapper'; + +import SourceMapCombiner from 'combine-source-map'; +import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats. + // Base64, stringified-JSON, end-of-file comment... + +let loadPolyfill = function(scriptFile, sourceMapFile) { + // May want to retool the pathing somewhat! + return { + source: fs.readFileSync(scriptFile).toString(), + sourceFile: sourceMapFile || scriptFile + }; +} + +let loadCompiledModuleFilePair = function(file, mapFilename) { + let module = fs.readFileSync(file); + let moduleSourcemapJSON = JSON.parse(fs.readFileSync(file + '.map').toString()); + + // May want to retool the pathing somewhat! + return { + plainSource: `${module}`, + sourceMapAsJSON: moduleSourcemapJSON, + sourceFile: mapFilename || file, + + /**The source + inlined sourcemap-as-comment used by `combine-source-map`. */ + get source() { + let jsonAsBuffer = Buffer.from(JSON.stringify(this.sourceMapAsJSON)); + return `${this.plainSource}\n${convertSourceMap.fromJSON(jsonAsBuffer).toComment()}`; + } + }; +} + +function concatScriptsAndSourcemaps(files, finalName, separatorFile) { + let combiner = SourceMapCombiner.create(finalName); + + let finalConcatenationArray = []; + let lineCountThusFar = 0; + + for(let filePairing of files) { + let offset = { + line: lineCountThusFar + }; + + console.log(`- ${filePairing.sourceFile}`); + combiner = combiner.addFile(filePairing, offset); + + let rawSourceToConcat = filePairing.plainSource || filePairing.source; + lineCountThusFar += rawSourceToConcat.split('\n').length + 1; // Not sure why it needs the fudge-factor, but it does. + + finalConcatenationArray.push(filePairing); + if(filePairing != files[files.length-1]) { + combiner = combiner.addFile(separatorFile); + finalConcatenationArray.push(separatorFile); + } + } + + let bundledSource = finalConcatenationArray.map((pair => pair.plainSource || pair.source)).join(''); + + return { + script: bundledSource, + sourcemapJSON: JSON.parse(convertSourceMap.fromBase64(combiner.base64()).toJSON()), + scriptFilename: finalName + } +} + +// Centralized? + +console.log("Pass 1: worker + polyfill concatenation"); + +let separatorFile = { + source: ` + + `, + sourceFile: '' +} + +let sourceFileSet = [ + // Needed for Android / Chromium browser pre-41. + loadPolyfill('../../../node_modules/string.prototype.codepointat/codepointat.js', 'polyfills/string.codepointat.js'), + // Needed for Android / Chromium browser pre-45. + loadPolyfill('src/polyfills/array.fill.js', 'polyfills/array.fill.js'), + // Needed for Android / Chromium browser pre-45. + loadPolyfill('src/polyfills/array.findIndex.js', 'polyfills/array.findIndex.js'), + // Needed for Android / Chromium browser pre-45. + loadPolyfill('src/polyfills/array.from.js', 'polyfills/array.from.js'), + // Needed for Android / Chromium browser pre-47. + loadPolyfill('src/polyfills/array.includes.js', 'polyfills/array.includes.js'), + // For Object.values, for iteration over object-based associate arrays. + // Needed for Android / Chromium browser pre-54. + loadPolyfill('src/polyfills/object.values.js', 'polyfills/object.values.js'), + // Needed to support Symbol.iterator, as used by the correction algorithm. + // Needed for Android / Chromium browser pre-43. + loadPolyfill('src/polyfills/symbol-es6.min.js', 'polyfills/symbol-es6.min.js'), + loadCompiledModuleFilePair('build/lib/worker-main.mjs', 'worker-main.mjs'), +]; + +let fullWorkerConcatenation = concatScriptsAndSourcemaps(sourceFileSet, "worker-main.polyfilled.js", separatorFile); + +// New stage: cleaning the sourcemaps +console.log(); + +// Because we're compiling the main project based on its TS build outputs, and our cross-module references +// also link to build outputs, we need to clean up the sourcemap-paths. Also, the individual module sourcemaps' +// paths result in unwanted extra pathing that needs to be cleaned up (models/models, correction/correction, etc) +console.log("Pass 2: cleaning sourcemap source paths"); + +let remappingState = SourceMapRemapper + .fromObject(fullWorkerConcatenation.sourcemapJSON) + .remapPaths([ + {from: /^polyfills\//, to: '/common/web/lm-worker/src/polyfills/'}, + {from: 'models/templates/build/obj/', to :'common/models/templates/src/'}, + {from: 'models/wordbreakers/build/obj/default/default', to: 'common/models/wordbreakers/src/default/'}, + {from: 'models/wordbreakers/build/obj/', to: 'common/models/wordbreakers/src/'}, + {from: 'obj/models/models/', to: 'common/web/lm-worker/src/models/'}, + {from: 'obj/correction/correction/', to: 'common/web/lm-worker/src/correction/'}, + {from: 'obj/', to: 'common/web/lm-worker/src/'}, + {from: /^\/utils\/src\//, to: '/common/web/utils/src/'}, + {from: '/keyman-version/', to: '/common/web/keyman-version/'}, + // {from: /^\//, to: ''} // To avoid the later minification pass mangling the paths. + ], (from, to) => console.log(`- ${from} => ${to}`)); + +if(remappingState.unchangedSourcepaths.length > 0) { + console.log(); + console.log("Not mapped:"); + + for(let path of remappingState.unchangedSourcepaths) { + console.log(`- ${path}`); + } +} + +console.log(); +let sourceRoot = "/@keymanapp/keyman"; +console.log(`Setting sourceRoot: ${sourceRoot}`) +remappingState.sourceRoot = sourceRoot; +fullWorkerConcatenation.sourcemapJSON = remappingState.sourceMap; + +// End "cleaning the sourcemaps" + +console.log(); +console.log("Pass 3: Output intermediate state and perform minification"); + +// IMPORTANT: Remove file-end sourcemap ref comment and replace it! +fullWorkerConcatenation.script = fullWorkerConcatenation.script.substring(0, fullWorkerConcatenation.script.lastIndexOf('//# sourceMappingURL')); +fullWorkerConcatenation.script += `//# sourceMappingURL=${fullWorkerConcatenation.scriptFilename}.map`; + +fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}`, fullWorkerConcatenation.script); +if(fullWorkerConcatenation.sourcemapJSON) { + fs.writeFileSync(`build/lib/${fullWorkerConcatenation.scriptFilename}.map`, convertSourceMap.fromObject(fullWorkerConcatenation.sourcemapJSON).toJSON()); +} \ No newline at end of file diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js new file mode 100644 index 0000000000000000000000000000000000000000..bf50537d9beed44781a6de303d65ea57e9138b71 --- /dev/null +++ b/common/web/lm-worker/build-wrap-and-minify.js @@ -0,0 +1,85 @@ +import fs from 'fs'; +import esbuild from 'esbuild'; + +import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats. + // Base64, stringified-JSON, end-of-file comment... + +let DEBUG = false; +let MINIFY = false; + +if(process.argv.length > 2) { + for(let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + + switch(arg) { + case '--debug': + DEBUG = true; + break; + case '--minify': + MINIFY = true; + break; + // May add other options if desired in the future. + default: + console.error("Invalid command-line option set for script; only --debug and --minify are permitted."); + process.exit(1); + } + } +} + +let sourcemapJSON = ''; + +if(MINIFY) { + await esbuild.build({ + entryPoints: [`build/lib/worker-main.polyfilled.js`], + sourcemap: 'external', + sourcesContent: DEBUG, + minify: MINIFY, + keepNames: true, + outfile: `build/lib/worker-main.polyfilled.min.js` + }); +} + +sourcemapJSON = convertSourcemap.fromJSON(fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js.map`)).toObject(); + +const workerConcatenation = { + script: fs.readFileSync(`build/lib/worker-main.polyfilled${MINIFY ? '.min' : ''}.js`), + sourcemapJSON: sourcemapJSON +} + +// While it IS possible to do partial sourcemaps (without the sources, but with everything else) within the worker... +// the resulting sourcemaps are -surprisingly- large - larger than the code itself! +// +// So... we don't use that strategy here. + +console.log(); +console.log(`Wrapping + generating final output: ${MINIFY ? 'minified' : 'unminified'} + ${DEBUG ? 'full sourcemaps' : 'reduced sourcemaps'}`); + +// Now, to build the wrapper... + +// First, let's build the encoded sourcemap. +const encodedSrcMap = convertSourcemap.fromObject(workerConcatenation.sourcemapJSON).toBase64(); +const srcMapString = `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${encodedSrcMap}`; + +/* + * It'd be nice to do a 'partial' encodeURIComponent that only gets the important bits... + * but my attempts to do so end up triggering errors when loading. + */ + +let rawScript = workerConcatenation.script.toString(); +// Two layers of encoding: one for the raw source (parsed by the JS engine), +// one to 'unwrap' it from a string _within_ that source. +let jsonEncoded = JSON.stringify(rawScript); + +let wrapper = ` +// Autogenerated code. Do not modify! +// --START:LMLayerWorkerCode-- + +export var LMLayerWorkerCode = ${jsonEncoded}; + +${MINIFY && "// Sourcemaps have been omitted for this release build."} +export var LMLayerWorkerSourcemapComment = "${DEBUG ? srcMapString : ''}"; + +// --END:LMLayerWorkerCode +`; + +fs.writeFileSync(`build/lib/worker-main.wrapped${MINIFY ? '.min' : ''}.js`, wrapper); \ No newline at end of file diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 59472b0f93351bd78aed7a3becb320ce359ba61b..82fba9bb39ba3496392d96e6b3fad0d59965e1d8 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -18,72 +18,25 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" # This script runs from its own folder -cd "$(dirname "$THIS_SCRIPT")" - -WORKER_OUTPUT=build -WORKER_OUTPUT_FILENAME=$WORKER_OUTPUT/index.js -WORKER_TEST_BUNDLE_TARGET_FILENAME=$WORKER_OUTPUT/index.wrapped-for-bundle.ts - -# Wraps JavaScript code in a way that can be embedded in a worker. -# To get the inner source code, include the file generated by this function, -# then use name.toString() where `name` is the name passed into this -# function. -wrap-worker-code ( ) { - name="$1" - js="$2" - # Reference: https://stackoverflow.com/a/59046041 - # Use of this annotation allows us to actually build against the file, rather - # than hackily insert it later when/where needed. - echo "// @ts-nocheck" - echo "// Autogenerated code. Do not modify!" - echo "// --START:LMLayerWorkerCode--" - printf "function %s () {\n" "${name}" - - # Since the worker is compiled with "allowJS=false" so that we can make - # declaration files, we have to insert polyfills here. - - ### NOTE ### - # Android API 21 (our current minimum) released with Chrome for Android 37. - # It's also updatable as of this version, but we can't guarantee that the user - # actually updated it, especially on first launch of the Android app/keyboard. - - # This one's a minimal, targeted polyfill. es6-shim could do the same, - # but also adds a lot more code the worker doesn't need to use. - # Recommended by MDN while keeping the worker lean and efficient. - # Needed for Android / Chromium browser pre-41. - cat "../../../node_modules/string.prototype.codepointat/codepointat.js" || builder_die - - # These two are straight from MDN - I didn't find any NPM ones that don't - # use the node `require` statement for the second. They're also relatively - # short and simple, which is good. - cat "src/polyfills/array.fill.js" || builder_die # Needed for Android / Chromium browser pre-45. - cat "src/polyfills/array.findIndex.js" || builder_die # Needed for Android / Chromium browser pre-45. - cat "src/polyfills/array.from.js" || builder_die # Needed for Android / Chromium browser pre-45. - cat "src/polyfills/array.includes.js" || builder_die # Needed for Android / Chromium browser pre-47. - - # For Object.values, for iteration over object-based associate arrays. - cat "src/polyfills/object.values.js" || builder_die # Needed for Android / Chromium browser pre-54. - - # Needed to support Symbol.iterator, as used by the correction algorithm. - cat "src/polyfills/symbol-es6.min.js" || builder_die # Needed for Android / Chromium browser pre-43. - - echo "" - - cat "${js}" || builder_die - printf "\n}\n" - echo "// --END:LMLlayerWorkerCode" -} +cd "$THIS_SCRIPT_PATH" + +WORKER_OUTPUT=build/obj +WORKER_OUTPUT_FILENAME=build/lib/worker-main.js ################################ Main script ################################ builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ "@/common/web/keyman-version" \ - configure clean build test + "@/common/web/tslib" \ + "@/common/models/wordbreakers" \ + "@/common/models/templates" \ + "@/common/tools/sourcemap-path-remapper" \ + configure clean build test --ci builder_describe_outputs \ configure /node_modules \ - build /common/web/lm-worker/build/index.js + build /common/web/lm-worker/build/lib/worker-main.wrapped.min.js builder_parse "$@" @@ -95,38 +48,46 @@ if builder_start_action configure; then builder_finish_action success configure fi -# We always need to clean first because the wrapping function -# breaks the typescript build, as it makes the index.tsbuildinfo file -# inconsistent with the actual index.js and causes the file to be -# corrupted on subsequent builds. In order to use the --build parameter -# of typescript, we need to avoid this! -# TODO: we should try and rework this to avoid the need to manually wrap - if builder_start_action clean; then npm run clean + rm -rf ./build + builder_finish_action success clean fi if builder_start_action build; then - if ! builder_has_action clean; then - npm run clean - fi - # Build worker with tsc first tsc -b $builder_verbose || builder_die "Could not build worker." - # Wrap the worker code and create embedded index.js. Must be run after the - # worker is built - echo "Wrapping worker in function LMLayerWorkerCode ${WORKER_OUTPUT_FILENAME}" - # Note: We use intermediate.js for unit tests in predictive-text - cp "${WORKER_OUTPUT_FILENAME}" "${WORKER_OUTPUT}/intermediate.js" - wrap-worker-code LMLayerWorkerCode "${WORKER_OUTPUT}/intermediate.js" > "${WORKER_OUTPUT_FILENAME}" || builder_die - cp "${WORKER_OUTPUT_FILENAME}" "${WORKER_TEST_BUNDLE_TARGET_FILENAME}" || builder_die + EXT_FLAGS= + if builder_has_option --ci; then + EXT_FLAGS=--ci + fi + + echo "Bundling worker modules" + node build-bundler.js "$EXT_FLAGS" + + # Declaration bundling. + npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/index.d.ts + npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/worker-main.d.ts + + echo "Preparing the polyfills + worker for script-embedding" + node build-polyfill-concatenator.js + + node build-wrap-and-minify.js --debug + node build-wrap-and-minify.js --minify builder_finish_action success build fi if builder_start_action test; then - npm test || builder_die "Tests failed" + MOCHA_FLAGS= + + if builder_has_option --ci; then + MOCHA_FLAGS="$MOCHA_FLAGS --reporter mocha-teamcity-reporter" + fi + + mocha --recursive $MOCHA_FLAGS ./src/test/cases/ + builder_finish_action success test fi diff --git a/common/web/lm-worker/package.json b/common/web/lm-worker/package.json index e93ad163065c0d50555b91891ef8f1fdfb24f7a1..232d03637ac97a560fe9aa7b53fe5c5b05f9319e 100644 --- a/common/web/lm-worker/package.json +++ b/common/web/lm-worker/package.json @@ -1,20 +1,33 @@ { "name": "@keymanapp/lm-worker", "description": "Keyman/Predictive Text worker", - "main": "build/index.js", + "main": "./build/lib/worker-main.wrapped.js", + "exports": { + "./worker-main.wrapped.js": "./build/lib/worker-main.wrapped.js", + "./worker-main.wrapped.min.js": "./build/lib/worker-main.wrapped.min.js" + }, + "imports": { + "#./*.js": "./build/obj/*.js", + "#./correction/*.js": "./build/obj/correction/*.js", + "#./models/*.js": "./build/obj/models/*.js" + }, + "type": "module", "scripts": { "karma": "karma", "mocha": "mocha", "test": "gosh ./test/test.sh", "build": "tsc -b", - "clean": "tsc -b --clean" + "clean": "tsc -b --clean", + "tsc": "tsc" }, "license": "MIT", "devDependencies": { + "@keymanapp/common-test-resources": "*", "@keymanapp/models-types": "*", "@keymanapp/resources-gosh": "*", "@types/node": "^10.17.21", "chai": "^4.3.4", + "combine-source-map": "^0.8.0", "karma": "^6.4.1", "karma-browserstack-launcher": "^1.6.0", "karma-chai": "^0.1.0", @@ -36,9 +49,10 @@ "typescript": "^4.9.5" }, "dependencies": { + "@keymanapp/keyman-version": "*", "@keymanapp/models-templates": "*", "@keymanapp/models-wordbreakers": "*", - "@keymanapp/keyman-version": "*", + "@keymanapp/sourcemap-path-remapper": "*", "@keymanapp/web-utils": "*", "es6-shim": "^0.35.5", "string.prototype.codepointat": "^0.2.1", diff --git a/common/web/lm-worker/src/correction/classical-calculation.ts b/common/web/lm-worker/src/correction/classical-calculation.ts deleted file mode 100644 index 3ac31ecd54b05eb6cf61f010b6d8b93585733848..0000000000000000000000000000000000000000 --- a/common/web/lm-worker/src/correction/classical-calculation.ts +++ /dev/null @@ -1,572 +0,0 @@ -namespace correction { - type EditOperation = 'insert' | 'delete' | 'match' | 'substitute' | 'transpose-start' | 'transpose-end' | 'transpose-insert' | 'transpose-delete'; - - /** - * Represents the lowest-level unit for comparison during edit-distance calculations. - */ - export interface EditToken { - key: TUnit; - } - - // A semi-optimized 'online'/iterative Damerau-Levenshtein calculator with the following features: - // - may add new character to the 'input' string or to the 'match' string, reusing all old calculations efficiently. - // - allows a 'focused' evaluation that seeks if the edit distance is within a specific range. Designed for use in match-searching, - // where we want to find the 'closest' matching strings in a lexicon. - // - towards such a match-searching algorithm/heuristic: should nothing be found within that range, all prior calculations may be reused - // to search across the lexicon with an incremented edit distance. - // - minimized memory footprint: O(m) memory footprint (where m = length of 'input' string), rather than O(mn) (where n = length of 'match' string) - // - guaranteed to use a smaller footprint than DiagonalizedIterativeDamerauLevenshteinCalculation. - // - // In short: Used to optimize calculations for low edit-distance checks, then expanded if/as necessary - // if a greater edit distance is requested. - // - // Reference: https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm#Possible_modifications - // - Motivating statement: "if we are only interested in the distance if it is smaller than a threshold..." - export class ClassicalDistanceCalculation = EditToken, TMatch extends EditToken = EditToken> { - /** - * Stores ONLY the computed diagonal elements, nothing else. - * - * Mapped as seen in the example below (with a diagonal of width 1): - * ``` - * MAX | MAX | MAX | MAX | MAX | ... - * MAX | 0 | 1 | 2 | 3 | ... > - * MAX | 1 | a | b | - | ... ====>> | - | a | b | - * MAX | 2 | c | d | e | ... > | c | d | e | - * MAX | 3 | - | f | g | ... | f | g | ... | - * ... | ... | ... | ... | ... | ... | ... | ... | ... | - * ``` - * - * Any "`-`" entries are undefined, as they lie outside of the diagonal under consideration. - * - * Things of note: - * - The entry where row index = col index will always lie at the center of the row's array. - * - For each +1 increase in row index, the row's entries are (logically) shifted by -1 in order to make this happen. - * - As all of the MAX entries and numerical entries above are fixed, known values, they are not represented here. - */ - resolvedDistances: number[][]; - /** - * Specifies how far off-diagonal calculations should be performed. A value of 0 only evaluates cells with matching - * row and column indicies. - * - * The resulting value from .getFinalCost() is only guaranteed correct if it is less than or equal to this value. - * Otherwise, this object represents a heuristic that _may_ overestimate the true edit distance. Note that it will - * never underestimate. - */ - diagonalWidth: number = 2; // TODO: Ideally, should start at 1... but we'll start at 2 for now - // as a naive workaround for multi-char transform limitations. - - // The sequence of characters input so far. - inputSequence: TInput[] = []; - matchSequence: TMatch[] = []; - - constructor(); - constructor(other: ClassicalDistanceCalculation); - constructor(other?: ClassicalDistanceCalculation) { - if(other) { - // Clone class properties. - let rowCount = other.resolvedDistances.length; - this.resolvedDistances = Array(rowCount); - - for(let r = 0; r < rowCount; r++) { - this.resolvedDistances[r] = Array.from(other.resolvedDistances[r]); - } - - this.inputSequence = Array.from(other.inputSequence); - this.matchSequence = Array.from(other.matchSequence); - this.diagonalWidth = other.diagonalWidth; - } else { - this.resolvedDistances = []; - } - } - - private getTrueIndex(r: number, c: number, width: number): {row: number, col: number, sparse: boolean} { - let retVal = { - row: r, - col: c - r + width, - sparse: false - } - - if(retVal.col < 0 || retVal.col > 2 * width) { - retVal.sparse = true; - } - - return retVal; - } - - private getCostAt(i: number, j: number, width: number = this.diagonalWidth): number { - // Check for and handle the set of fixed-value virtualized indices. - if(i < 0 || j < 0) { - if(i == -1 && j >= -1) { - return j+1; - } else if(j == -1 && i >= -1) { - return i+1; - } - - return Number.MAX_VALUE; - } - - let index = this.getTrueIndex(i, j, width); - return index.sparse ? Number.MAX_VALUE : this.resolvedDistances[index.row][index.col]; - } - - /** - * Noting the above link's statement prefixed "By examining diagonals instead of rows, and by using lazy evaluation...", - * this function will return the actual edit distance between the strings, temporarily increasing the computed - * diagonal's size if necessary. - * - * Does not actually mutate the instance. - */ - getFinalCost(): number { - let buffer = this as ClassicalDistanceCalculation; - let val = buffer.getHeuristicFinalCost(); - - while(val > buffer.diagonalWidth) { - // A consequence of treating this class as immutable. - buffer = buffer.increaseMaxDistance(); - val = buffer.getHeuristicFinalCost(); - } - - return val; - } - - /** - * Returns this instance's computed edit distance. If greater than the diagonal's width value, note that it may be an overestimate. - */ - getHeuristicFinalCost(): number { - return this.getCostAt(this.inputSequence.length-1, this.matchSequence.length-1); - } - - /** - * Returns `true` if the represented edit distance is less than or equal to the specified threshold, minimizing the amount of calculations - * needed to meet the specified limit. - * - * Does not mutate the instance. - * @param threshold - */ - hasFinalCostWithin(threshold: number): boolean { - let buffer = this as ClassicalDistanceCalculation; - let val = buffer.getHeuristicFinalCost(); - let guaranteedBound = this.diagonalWidth; - - do { - // val will never exceed the length of the longer string, no matter how large the threshold. - if(val <= threshold) { - return true; - } else if(guaranteedBound < threshold) { - buffer = buffer.increaseMaxDistance(); - guaranteedBound++; - val = buffer.getHeuristicFinalCost(); - } else { - break; - } - } while(true); - - return false; - } - - /** - * Determines the edit path used to obtain the optimal cost, distinguishing between zero-cost - * substitutions ('match' operations) and actual substitutions. - * @param row - * @param col - */ - public editPath(row: number = this.inputSequence.length - 1, col: number = this.matchSequence.length - 1): EditOperation[] { - let currentCost = this.getCostAt(row, col); - let ops: EditOperation[] = null; - let parent: [number, number] = null; - - let insertParentCost = this.getCostAt(row, col-1); - let deleteParentCost = this.getCostAt(row-1, col); - let substitutionParentCost = this.getCostAt(row-1, col-1); - let [lastInputIndex, lastMatchIndex] = ClassicalDistanceCalculation.getTransposeParent(this, row, col); - if(lastInputIndex >= 0 && lastMatchIndex >= 0) { - // OK, a transposition source is quite possible. Still need to do more vetting, to be sure. - let expectedCost = 1; - - // This transposition includes either 'transpose-insert' or 'transpose-delete' operations. - ops = ['transpose-start']; // always needs a 'start'. - if(lastInputIndex != row-1) { - let count = row - lastInputIndex - 1; - ops = ops.concat( Array(count).fill('transpose-delete') ); - expectedCost += count; - } else { - let count = col - lastMatchIndex - 1; - ops = ops.concat( Array(count).fill('transpose-insert') ); - expectedCost += count; - } - ops.push('transpose-end'); - - // Double-check our expectations. - if(this.getCostAt(lastInputIndex-1, lastMatchIndex-1) != currentCost - expectedCost) { - ops = null; - } - parent = [lastInputIndex-1, lastMatchIndex-1]; - } - - if(ops) { - // bypass the ladder. - } else if(substitutionParentCost == currentCost - 1) { - ops = ['substitute']; - parent = [row-1, col-1]; - } else if(insertParentCost == currentCost - 1) { - ops = ['insert']; - parent = [row, col-1]; - } else if(deleteParentCost == currentCost - 1) { - ops = ['delete']; - parent = [row-1, col]; - } else { //if(substitutionParentCost == currentCost) { - ops = ['match']; - parent = [row-1, col-1]; - } - - // Recursively build the edit path. - if(parent[0] >= 0 && parent[1] >= 0) { - return this.editPath(parent[0], parent[1]).concat(ops); - } else { - if(parent[0] > -1) { - // There are initial deletions. - return Array(parent[0]+1).fill('delete').concat(ops); - } else if(parent[1] > -1) { - // There are initial insertions. - return Array(parent[1]+1).fill('insert').concat(ops); - } else { - return ops; - } - } - } - - private static getTransposeParent, TMatch extends EditToken>( - buffer: ClassicalDistanceCalculation, - r: number, - c: number - ): [number, number] { - // Block any transpositions where the tokens are identical. - // Other operations will be cheaper. Also, block cases where 'parents' are impossible. - if(r < 0 || c < 0 || buffer.inputSequence[r].key == buffer.matchSequence[c].key) { - return [-1, -1]; - } - - // Transposition checks - let lastInputIndex = -1; - for(let i = r-1; i >= 0; i--) { - if(buffer.inputSequence[i].key == buffer.matchSequence[c].key) { - lastInputIndex = i; - break; - } - } - - let lastMatchIndex = -1; - for(let i = c-1; i >= 0; i--) { - if(buffer.matchSequence[i].key == buffer.inputSequence[r].key) { - lastMatchIndex = i; - break; - } - } - - return [lastInputIndex, lastMatchIndex]; - } - - private static initialCostAt, TMatch extends EditToken>( - buffer: ClassicalDistanceCalculation, - r: number, - c: number, - insertCost?: number, - deleteCost?: number) { - var baseSubstitutionCost = buffer.inputSequence[r].key == buffer.matchSequence[c].key ? 0 : 1; - var substitutionCost: number = buffer.getCostAt(r-1, c-1) + baseSubstitutionCost; - var insertionCost: number = insertCost || buffer.getCostAt(r, c-1) + 1; // If set meaningfully, will never equal zero. - var deletionCost: number = deleteCost || buffer.getCostAt(r-1, c) + 1; // If set meaningfully, will never equal zero. - var transpositionCost: number = Number.MAX_VALUE - - if(r > 0 && c > 0) { // bypass when transpositions are known to be impossible. - let [lastInputIndex, lastMatchIndex] = ClassicalDistanceCalculation.getTransposeParent(buffer, r, c); - transpositionCost = buffer.getCostAt(lastInputIndex-1, lastMatchIndex-1) + (r - lastInputIndex - 1) + 1 + (c - lastMatchIndex - 1); - } - - return Math.min(substitutionCost, deletionCost, insertionCost, transpositionCost); - } - - getSubset(inputLength: number, matchLength: number): ClassicalDistanceCalculation { - let trimmedInstance = new ClassicalDistanceCalculation(this); - - if(inputLength > this.inputSequence.length || matchLength > this.matchSequence.length) { - throw "Invalid dimensions specified for trim operation"; - } - // Trim our tracked input & match sequences. - trimmedInstance.inputSequence.splice(inputLength); - trimmedInstance.matchSequence.splice(matchLength); - - // Major index corresponds to input length. - trimmedInstance.resolvedDistances.splice(inputLength); - - // The real fun: trimming off columns. (Minor index, corresponds to match length) - let finalTrueIndex = this.getTrueIndex(inputLength-1, matchLength-1, this.diagonalWidth); - // The diagonal index increases as the row index decreases. - for(let diagonalIndex = finalTrueIndex.col; diagonalIndex <= 2 * this.diagonalWidth; diagonalIndex++) { - let row = finalTrueIndex.row - (diagonalIndex - finalTrueIndex.col); - if(row < 0) { - break; - } - - if(diagonalIndex < 0) { - trimmedInstance.resolvedDistances[row] = Array(2 * trimmedInstance.diagonalWidth + 1).fill(Number.MAX_VALUE); - } else { - let newCount = 2 * this.diagonalWidth - diagonalIndex; - let keptEntries = trimmedInstance.resolvedDistances[row].splice(0, diagonalIndex+1); - let newEntries = Array(newCount).fill(Number.MAX_VALUE); - trimmedInstance.resolvedDistances[row] = keptEntries.concat(newEntries); - } - } - - return trimmedInstance; - } - - private static forDiagonalOfAxis(diagonalWidth: number, centerIndex: number, axisCap: number, closure: (axisIndex: number, diagIndex: number) => void) { - let diagonalCap = axisCap - centerIndex < diagonalWidth ? axisCap - centerIndex + diagonalWidth : 2 * diagonalWidth; - let startOffset = centerIndex - diagonalWidth; // The axis's index for diagonal entry 0. May be negative. - let diagonalStart = startOffset < 0 ? 0 : startOffset; - - for(let diagonalIndex = diagonalStart - startOffset; diagonalIndex <= diagonalCap; diagonalIndex++) { - closure(startOffset + diagonalIndex, diagonalIndex); - } - } - - // Inputs add an extra row / first index entry. - addInputChar(token: TInput): ClassicalDistanceCalculation { - let returnBuffer = new ClassicalDistanceCalculation(this); - - let r = returnBuffer.inputSequence.length; - returnBuffer.inputSequence.push(token); - - // Insert a row, even if we don't actually do anything with it yet. - // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues. - let row = Array(2 * returnBuffer.diagonalWidth + 1).fill(Number.MAX_VALUE); - returnBuffer.resolvedDistances[r] = row; - - // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. - if(returnBuffer.matchSequence.length == 0) { - return returnBuffer; - } - - ClassicalDistanceCalculation.forDiagonalOfAxis(returnBuffer.diagonalWidth, r, returnBuffer.matchSequence.length - 1, function(c, diagIndex) { - row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); - }); - - return returnBuffer; - } - - addMatchChar(token: TMatch): ClassicalDistanceCalculation { - let returnBuffer = new ClassicalDistanceCalculation(this); - - let c = returnBuffer.matchSequence.length; - returnBuffer.matchSequence.push(token); - - // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. - if(returnBuffer.inputSequence.length == 0) { - return returnBuffer; - } - - ClassicalDistanceCalculation.forDiagonalOfAxis(returnBuffer.diagonalWidth, c, returnBuffer.inputSequence.length - 1, function(r, diagIndex) { - var row = returnBuffer.resolvedDistances[r]; - // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column. - row[2 * returnBuffer.diagonalWidth - diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); - }); - - return returnBuffer; - } - - public increaseMaxDistance(): ClassicalDistanceCalculation { - let returnBuffer = new ClassicalDistanceCalculation(this); - returnBuffer.diagonalWidth++; - - if(returnBuffer.inputSequence.length < 1 || returnBuffer.matchSequence.length < 1) { - return returnBuffer; - } - - // An abstraction of the common aspects of transposition handling during diagonal extensions. - function forPossibleTranspositionsInDiagonal(startPos: number, fixedChar: TUnit, lookupString: EditToken[], closure: (axisIndex: number, diagIndex: number) => void) { - let diagonalCap = 2 * (returnBuffer.diagonalWidth - 1); // The maximum diagonal index permitted - let axisCap = lookupString.length - 1; // The maximum index supported by the axis of iteration - - // Ensures that diagonal iteration only occurs within the axis's supported range - diagonalCap = diagonalCap < axisCap - startPos ? diagonalCap : axisCap - startPos; - - // Iterate within the diagonal and call our closure for any potential transpositions. - for(let diagonalIndex = 0; diagonalIndex <= diagonalCap; diagonalIndex++) { - if(fixedChar == lookupString[startPos + diagonalIndex].key) { - closure(startPos + diagonalIndex, diagonalIndex); - } - } - } - - for(let r = 0; r < returnBuffer.inputSequence.length; r++) { - let leftCell = Number.MAX_VALUE; - let c = r - returnBuffer.diagonalWidth // External index of the left-most entry, which we will now calculate. - if(c >= 0) { - // If c == 0, cell is at edge, thus a known value for insertions exists. - // Base cost: r+1, +1 for inserting. - let insertionCost = c == 0 ? r + 2 : Number.MAX_VALUE; - // compute new left cell - leftCell = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c, insertionCost, undefined); - let addedCost = leftCell; - - // daisy-chain possible updates - - // cell (r, c+1): new insertion source - if(c < returnBuffer.matchSequence.length-1) { - // We propagate the new added cost (via insertion) to the old left-most cell, which is one to our right. - ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, r, c+1, addedCost+1, 0); - - // Only possible if insertions are also possible AND more conditions are met. - // cells (r+2, * > c+2): new transposition source - let transposeRow = r+2; - if(r+2 < this.inputSequence.length) { // Row to check for transposes must exist. - let rowChar = returnBuffer.inputSequence[r+1].key; - // First possible match in input could be at index c + 2, which adjusts col c+2's cost. Except that entry in r+2 - // doesn't exist yet - so we start with c+3 instead. - forPossibleTranspositionsInDiagonal(c + 3, rowChar, returnBuffer.matchSequence, function(axisIndex, diagIndex) { - // Because (r+2, c+3) is root, not (r+2, c+2). Min cost of 2. - ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, transposeRow, axisIndex, addedCost + diagIndex + 2, diagIndex); - }); - } - } - } - - let rightCell = Number.MAX_VALUE; - c = r + returnBuffer.diagonalWidth; - if(c < returnBuffer.matchSequence.length) { - // If r == 0, cell is at edge, thus a known value for insertions exists. - // Base cost: c+1, +1 for inserting. - let deletionCost = r == 0 ? c + 2 : Number.MAX_VALUE; - - // the current row wants to use adjusted diagonal width; we must specify use of the old width & its mapping instead. - var insertionCost: number = returnBuffer.getCostAt(r, c-1, this.diagonalWidth) + 1; - - // compute new right cell - rightCell = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c, insertionCost, deletionCost); - let addedCost = rightCell; - - // daisy-chain possible updates - - // cell (r+1, c): new deletion source - if(r < returnBuffer.inputSequence.length - 1) { - // We propagate the new added cost (via deletion) to the old right-most cell, which is one to our right. - ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, r+1, c, addedCost + 1, 2 * this.diagonalWidth); - - // Only possible if deletions are also possible AND more conditions are met. - // cells(* > r+2, c+2): new transposition source - let transposeCol = c+2; - if(c+2 < this.matchSequence.length) { // Row to check for transposes must exist. - let colChar = returnBuffer.matchSequence[r+1].key; - // First possible match in input could be at index r + 2, which adjusts row r+2's cost. Except that entry in c+2 - // doesn't exist yet - so we start with r+3 instead. - forPossibleTranspositionsInDiagonal(r+3, colChar, returnBuffer.inputSequence, function(axisIndex, diagIndex) { - let diagColIndex = 2 * (returnBuffer.diagonalWidth - 1) - diagIndex; - // Because (r+3, c+2) is root, not (r+2, c+2). Min cost of 2. - ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, axisIndex, transposeCol, addedCost + diagIndex + 2, diagColIndex); - }); - } - } - } - - // Constructs the final expanded diagonal for the row. - returnBuffer.resolvedDistances[r] = [leftCell].concat(returnBuffer.resolvedDistances[r], rightCell); - } - - return returnBuffer; - } - - private static propagateUpdateFrom, TMatch extends EditToken>( - buffer: ClassicalDistanceCalculation, - r: number, - c: number, - value: number, - diagonalIndex: number) { - // Note: this function does not actually need the `c` parameter! - // That said, it's very useful when tracing stack traces & debugging. - if(value < buffer.resolvedDistances[r][diagonalIndex]) { - buffer.resolvedDistances[r][diagonalIndex] = value; - } else { - return - } - - let internalRow = r < buffer.inputSequence.length - 1; - let internalCol = c < buffer.matchSequence.length - 1; - - // We have to compensate for the current & following rows not having been expanded yet. - if(diagonalIndex < 2 * (buffer.diagonalWidth - 1) && internalCol) { - // We've inserted to the left of an existing calculation - check for propagation via insertion. - let updateCost = value + 1; - this.propagateUpdateFrom(buffer, r, c+1, updateCost, diagonalIndex+1); - } - - if(diagonalIndex > 0 && internalRow) { - // We've inserted above an existing calculation - check for propagation via deletion - let updateCost = value + 1 - this.propagateUpdateFrom(buffer, r+1, c, updateCost, diagonalIndex-1); - } - - // If both, check for propagation via substitution and possible transpositions - if(internalRow && internalCol) { - let updateCost = value + (buffer.inputSequence[r+1].key == buffer.matchSequence[c+1].key ? 0 : 1); - this.propagateUpdateFrom(buffer, r+1, c+1, updateCost, diagonalIndex); - - // Propagating transpositions (only possible if 'internal'.) - let nextInputIndex = -1; - for(let i = r+2; i < buffer.inputSequence.length; i++) { - if(buffer.inputSequence[i].key == buffer.matchSequence[c+1].key) { - nextInputIndex = i; - break; - } - } - - let nextMatchIndex = -1; - for(let i = c+2; i < buffer.matchSequence.length; i++) { - if(buffer.matchSequence[i].key == buffer.inputSequence[r+1].key) { - nextMatchIndex = i; - break; - } - } - - if(nextInputIndex > 0 && nextMatchIndex > 0) { - let transpositionCost = value + (nextInputIndex - r - 2) + 1 + (nextMatchIndex - c - 2); - this.propagateUpdateFrom(buffer, nextInputIndex, nextMatchIndex, transpositionCost, (buffer.diagonalWidth - 1) + nextMatchIndex - nextInputIndex); - } - } - } - - get mapKey(): string { - let inputString = this.inputSequence.map((value) => value.key).join(''); - let matchString = this.matchSequence.map((value) => value.key).join(''); - return inputString + models.SENTINEL_CODE_UNIT + matchString + models.SENTINEL_CODE_UNIT + this.diagonalWidth; - } - - get lastInputEntry(): TInput { - return this.inputSequence[this.inputSequence.length-1]; - } - - get lastMatchEntry(): TMatch { - return this.matchSequence[this.matchSequence.length-1]; - } - - static computeDistance, TMatch extends EditToken>( - input: TInput[], - match: TMatch[], - bandSize: number = 1) { - // Initialize the calculation buffer, setting the diagonal width (as appropriate) in advance. - let buffer = new ClassicalDistanceCalculation(); - bandSize = bandSize || 1; - buffer.diagonalWidth = bandSize; - - for(let i = 0; i < input.length; i++) { - buffer = buffer.addInputChar(input[i]); - } - - for(let j = 0; j < match.length; j++) { - buffer = buffer.addMatchChar(match[j]); - } - - return buffer; - } - } -} diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts deleted file mode 100644 index e17e201d785441b51fed3e75ea27779a935d987c..0000000000000000000000000000000000000000 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ /dev/null @@ -1,539 +0,0 @@ -/// - -namespace correction { - - function textToCharTransforms(text: string, transformId?: number) { - let perCharTransforms: Transform[] = []; - - for(let i=0; i < text.kmwLength(); i++) { - let char = text.kmwCharAt(i); // is SMP-aware - - let transform: Transform = { - insert: char, - deleteLeft: 0, - id: transformId - }; - - perCharTransforms.push(transform); - } - - return perCharTransforms; - } - export class TrackedContextSuggestion { - suggestion: Suggestion; - tokenWidth: number; - } - - export class TrackedContextToken { - raw: string; - replacementText: string; - - transformDistributions: Distribution[] = []; - replacements: TrackedContextSuggestion[]; - activeReplacementId: number = -1; - - get currentText(): string { - if(this.replacementText === undefined || this.replacementText === null) { - return this.raw; - } else { - return this.replacementText; - } - } - - get replacement(): TrackedContextSuggestion { - let replacementId = this.activeReplacementId; - return this.replacements.find(function(replacement) { - return replacement.suggestion.id == replacementId; - }); - } - - revert() { - delete this.activeReplacementId; - } - } - - export class TrackedContextState { - // Stores the source Context (as a debugging reference). Not currently utilized. - taggedContext: Context; - model: LexicalModel; - - tokens: TrackedContextToken[]; - /** - * How many tokens were removed from the start of the best-matching ancestor. - * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position. - */ - indexOffset: number; - - // Tracks all search spaces starting at the current token. - // In the lm-layer's current form, this should only ever have one entry. - // Leaves 'design space' for if/when we add support for phrase-level corrections/predictions. - searchSpace: SearchSpace[] = []; - - constructor(source: TrackedContextState); - constructor(model: LexicalModel); - constructor(obj: TrackedContextState | LexicalModel) { - if(obj instanceof TrackedContextState) { - let source = obj; - // Be sure to deep-copy the tokens! Pointer-aliasing is bad here. - this.tokens = source.tokens.map(function(token) { - let copy = new TrackedContextToken(); - copy.raw = token.raw; - copy.replacements = token.replacements - copy.activeReplacementId = token.activeReplacementId; - copy.transformDistributions = token.transformDistributions; - - if(token.replacementText) { - copy.replacementText = token.replacementText; - } - - return copy; - }); - this.searchSpace = obj.searchSpace; - this.indexOffset = 0; - this.model = obj.model; - } else { - let lexicalModel = obj; - this.tokens = []; - this.indexOffset = Number.MIN_SAFE_INTEGER; - this.model = lexicalModel; - - if(lexicalModel && lexicalModel.traverseFromRoot) { - this.searchSpace = [new SearchSpace(lexicalModel)]; - } - } - } - - get head(): TrackedContextToken { - return this.tokens[0]; - } - - get tail(): TrackedContextToken { - return this.tokens[this.tokens.length - 1]; - } - - popHead() { - this.tokens.splice(0, 2); - this.indexOffset -= 1; - } - - pushTail(token: TrackedContextToken) { - if(this.model && this.model.traverseFromRoot) { - this.searchSpace = [new SearchSpace(this.model)]; // yeah, need to update SearchSpace for compatibility - } else { - this.searchSpace = []; - } - this.tokens.push(token); - - let state = this; - if(state.searchSpace.length > 0) { - token.transformDistributions.forEach(distrib => state.searchSpace[0].addInput(distrib)); - } - } - - pushWhitespaceToTail(transformDistribution: Distribution = null) { - let whitespaceToken = new TrackedContextToken(); - - // Track the Transform that resulted in the whitespace 'token'. - // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; - - whitespaceToken.raw = null; - this.tokens.push(whitespaceToken); - } - - /** - * Used for 14.0's backspace workaround, which flattens all previous Distribution - * entries because of limitations with direct use of backspace transforms. - * @param tokenText - * @param transformId - */ - replaceTailForBackspace(tokenText: USVString, transformId: number) { - this.tokens.pop(); - - // It's a backspace transform; time for special handling! - // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, - // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. - // - // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... - // not very compatible with that. - let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { - return [{sample: transform, p: 1.0}]; - }); - - let compactedToken = new TrackedContextToken(); - compactedToken.raw = tokenText; - compactedToken.transformDistributions = backspacedTokenContext; - this.pushTail(compactedToken); - } - - updateTail(transformDistribution: Distribution, tokenText?: USVString) { - let editedToken = this.tail; - - // Preserve existing text if new text isn't specified. - tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); - - if(transformDistribution && transformDistribution.length > 0) { - editedToken.transformDistributions.push(transformDistribution); - if(this.searchSpace) { - this.searchSpace.forEach(space => space.addInput(transformDistribution)); - } - } - // Replace old token's raw-text with new token's raw-text. - editedToken.raw = tokenText; - } - - toRawTokenization() { - let sequence: USVString[] = []; - - for(let token of this.tokens) { - // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) - if(token.currentText !== null) { - sequence.push(token.currentText); - } - } - - return sequence; - } - } - - class CircularArray { - static readonly DEFAULT_ARRAY_SIZE = 5; - private circle: Item[]; - private currentHead: number=0; - private currentTail: number=0; - - constructor(size: number = CircularArray.DEFAULT_ARRAY_SIZE) { - this.circle = Array(size); - } - - get count(): number { - let diff = this.currentHead - this.currentTail; - - if(diff < 0) { - diff = diff + this.circle.length; - } - - return diff; - } - - get maxCount(): number { - return this.circle.length; - } - - get oldest(): Item { - if(this.count == 0) { - return undefined; - } - - return this.item(0); - } - - get newest(): Item { - if(this.count == 0) { - return undefined; - } - - return this.item(this.count - 1); - } - - enqueue(item: Item): Item { - var prevItem = null; - let nextHead = (this.currentHead + 1) % this.maxCount; - - if(nextHead == this.currentTail) { - prevItem = this.circle[this.currentTail]; - this.currentTail = (this.currentTail + 1) % this.maxCount; - } - - this.circle[this.currentHead] = item; - this.currentHead = nextHead; - - return prevItem; - } - - dequeue(): Item { - if(this.currentTail == this.currentHead) { - return null; - } else { - let item = this.circle[this.currentTail]; - this.currentTail = (this.currentTail + 1) % this.maxCount; - return item; - } - } - - popNewest(): Item { - if(this.currentTail == this.currentHead) { - return null; - } else { - let item = this.circle[this.currentHead]; - this.currentHead = (this.currentHead - 1 + this.maxCount) % this.maxCount; - return item; - } - } - - /** - * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - - * the same order in which the items will be dequeued. - * @param index - */ - item(index: number) { - if(index >= this.count) { - throw "Invalid array index"; - } - - let mappedIndex = (this.currentTail + index) % this.maxCount; - return this.circle[mappedIndex]; - } - } - - export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], - matchState: TrackedContextState, - transformDistribution?: Distribution,): TrackedContextState { - // Map the previous tokenized state to an edit-distance friendly version. - let matchContext: USVString[] = matchState.toRawTokenization(); - - // Inverted order, since 'match' existed before our new context. - let mapping = ClassicalDistanceCalculation.computeDistance(matchContext.map(value => ({key: value})), - tokenizedContext.map(value => ({key: value})), - 1); - - let editPath = mapping.editPath(); - - let poppedHead = false; - let pushedTail = false; - - // Matters greatly when starting from a nil context. - if(editPath.length > 1) { - // First entry: may not be an 'insert' or a 'transpose' op. - // 'insert' allowed if the next token is 'substitute', as this may occur with an edit path of length 2. - if((editPath[0] == 'insert' && !(editPath[1] == 'substitute' && editPath.length == 2)) || editPath[0].indexOf('transpose') >= 0) { - return null; - } else if(editPath[0] == 'delete') { - poppedHead = true; // a token from the previous state has been wholly removed. - } - } - - // Last entry: may not be a 'delete' or a 'transpose' op. - let tailIndex = editPath.length -1; - let ignorePenultimateMatch = false; - if(editPath[tailIndex] == 'delete' || editPath[0].indexOf('transpose') >= 0) { - return null; - } else if(editPath[tailIndex] == 'insert') { - pushedTail = true; - } else if(tailIndex > 0 && editPath[tailIndex-1] == 'insert' && editPath[tailIndex] == 'substitute') { - // Tends to happen when accepting suggestions. - pushedTail = true; - ignorePenultimateMatch = true; - } - - // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after - // that prior wordbreaking character. - // - // We can't handle it reliably from this match state, but a previous entry (without the empty token) - // should still be in the cache and will be reliable for this example case. - if(tailIndex > 0 && editPath[tailIndex-1] == 'delete' && editPath[tailIndex] == 'substitute') { - return null; - } - - // Now to check everything in-between: should be exclusively 'match'es. - for(let index = 1; index < editPath.length - (ignorePenultimateMatch ? 2 : 1); index++) { - if(editPath[index] != 'match') { - return null; - } - } - - // If we've made it here... success! We have a context match! - let state: TrackedContextState; - - if(pushedTail) { - // On suggestion acceptance, we should update the previous final token. - // We do it first so that the acceptance is replicated in the new TrackedContextState - // as well. - if(ignorePenultimateMatch) { - // For this case, we were likely called by ModelCompositor.acceptSuggestion(), which - // would have marked the accepted suggestion. - matchState.tail.replacementText = tokenizedContext[tokenizedContext.length-2]; - } - - state = new TrackedContextState(matchState); - } else { - // Since we're continuing a previously-cached context, we can reuse the same SearchSpace - // to continue making predictions. - state = matchState; - } - - const hasDistribution = transformDistribution && Array.isArray(transformDistribution); - let primaryInput = hasDistribution ? transformDistribution[0].sample : null; - if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { - primaryInput = null; - } - - const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); - const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); - const finalToken = tokenizedContext[tokenizedContext.length-1]; - - /* Assumption: This is an adequate check for its two sub-branches. - * - * Basis: - * - Assumption: one keystroke may only cause a single token to rotate out of context. - * - That is, no "reasonable" keystroke would emit enough code points to 'bump' two words simultaneously. - * - ... This one may need to be loosened a bit... but it should be enough for initial correction testing as-is. - * - Assumption: one keystroke may only cause a single token to be appended to the context - * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens - * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ - - // If there is/was more than one context token available... - if(editPath.length > 1) { - // We're removing a context token, but at least one remains. - if(poppedHead) { - state.popHead(); - } - - // We're adding an additional context token. - if(pushedTail) { - const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; - /* - * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. - * - * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: - * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! - */ - let pushedToken = new TrackedContextToken(); - pushedToken.raw = tokenizedTail; - - if(isWhitespace || !primaryInput) { - state.pushWhitespaceToTail(transformDistribution ?? []); - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - pushedToken.transformDistributions = []; - } else { - state.pushWhitespaceToTail(); - // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing - // any metadata used to construct the new context state token. - pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; - } - - state.pushTail(pushedToken); - } else { // We're editing the final context token. - // TODO: Assumption: we didn't 'miss' any inputs somehow. - // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. - if(isBackspace) { - state.replaceTailForBackspace(finalToken, primaryInput.id); - } else { - state.updateTail(primaryInput ? transformDistribution : null, finalToken); - } - } - // There is only one word in the context. - } else { - // TODO: Assumption: we didn't 'miss' any inputs somehow. - // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. - - if(editPath[tailIndex] == 'insert') { - // Construct appropriate initial token. - let token = new TrackedContextToken(); - token.raw = tokenizedContext[0]; - token.transformDistributions = [transformDistribution]; - state.pushTail(token); - } else { // Edit the lone context token. - // Consider backspace entry for this case? - if(isBackspace) { - state.replaceTailForBackspace(finalToken, primaryInput.id); - } else { - state.updateTail(primaryInput ? transformDistribution : null, finalToken); - } - } - } - return state; - } - - static modelContextState(tokenizedContext: USVString[], - transformDistribution: Distribution, - lexicalModel: LexicalModel): TrackedContextState { - let baseTokens = tokenizedContext.map(function(entry) { - let token = new TrackedContextToken(); - token.raw = entry; - if(token.raw) { - token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) { - return [{sample: transform, p: 1.0}]; - }); - } else { - // Helps model context-final wordbreaks. - token.transformDistributions = []; - } - return token; - }); - - // And now build the final context state object, which includes whitespace 'tokens'. - let state = new TrackedContextState(lexicalModel); - - if(baseTokens.length > 0) { - state.pushTail(baseTokens.splice(0, 1)[0]); - } - - while(baseTokens.length > 0) { - state.pushWhitespaceToTail(); - state.pushTail(baseTokens.splice(0, 1)[0]); - } - - if(state.tokens.length == 0) { - let token = new TrackedContextToken(); - token.raw = ''; - - state.pushTail(token); - } - - return state; - } - - /** - * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning - * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the - * input information provided to previous prediction calls and persisted correction-search calculations for re-use. - * - * @param model - * @param context - * @param transformDistribution - */ - analyzeState(model: LexicalModel, - context: Context, - transformDistribution?: Distribution): TrackedContextState { - if(!model.traverseFromRoot) { - // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) - // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. - throw "This lexical model does not provide adequate data for correction algorithms and context reuse"; - } - - let tokenizedContext = models.tokenize(model.wordbreaker || wordBreakers.default, context); - - if(tokenizedContext.left.length > 0) { - for(let i = this.count - 1; i >= 0; i--) { - let resultState = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), transformDistribution); - - if(resultState) { - resultState.taggedContext = context; - if(resultState != this.item(i)) { - this.enqueue(resultState); - } - return resultState; - } - } - } - - // Else: either empty OR we've detected a 'new context'. Initialize from scratch; no prior input information is - // available. Only the results of the prior inputs are known. - // - // Assumption: as a caret needs to move to context before any actual transform distributions occur, - // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); - state.taggedContext = context; - this.enqueue(state); - return state; - } - } -} diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts deleted file mode 100644 index 723cbc93ca045f299f1b140ec400a2a8a84f51ad..0000000000000000000000000000000000000000 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ /dev/null @@ -1,805 +0,0 @@ -/// - -namespace correction { - type RealizedInput = ProbabilityMass[]; // NOT Distribution - they're masses from separate distributions. - - export type TraversableToken = { - key: TUnit, - traversal: LexiconTraversal - } - - export const QUEUE_NODE_COMPARATOR: models.Comparator = function(arg1, arg2) { - return arg1.currentCost - arg2.currentCost; - } - - // Represents a processed node for the correction-search's search-space's tree-like graph. May represent - // internal and 'leaf' nodes on said graph, as well as the overall root of the search. Also used to represent - // edges on the graph TO said nodes - there's a bit of overloading here. Either way, it stores the cost of the - // optimum path used to reach the ndoe. - // - // The stored path cost may be an overestimate when the edit distance is greater than the current search threshold. The - // first version of the node to be dequeued from SearchSpace's priority queue hierarchy 'wins' and is taken as the absolute - // minimum; subsequent versions are ignored as suboptimal. - // - // Provides functions usable to enumerate across the node's outward edges to new nodes for continued search. - // Most of the actual calculations occur as part of this process. - // - // For nodes with raw edit-distance cost within the current threshold for correction searches, we do have admissibility. - // If not enough nodes are available within that threshold, however, admissibility may be lost, leaving our search as a - // heuristic. - // - export class SearchNode { - calculation: ClassicalDistanceCalculation, TraversableToken>; - - currentTraversal: LexiconTraversal; - toKey: (wordform: USVString) => USVString = str => str; - priorInput: RealizedInput; - - // Internal lazy-cache for .inputSamplingCost, as it's a bit expensive to re-compute. - private _inputCost?: number; - - constructor(rootTraversal: LexiconTraversal, toKey?: (USVString) => USVString); - constructor(node: SearchNode); - constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (USVString) => USVString) { - toKey = toKey || (x => x); - - if(rootTraversal instanceof SearchNode) { - let priorNode = rootTraversal; - this.calculation = priorNode.calculation; - this.currentTraversal = priorNode.currentTraversal; - this.priorInput = priorNode.priorInput; - this.toKey = priorNode.toKey; - } else { - this.calculation = new ClassicalDistanceCalculation(); - this.currentTraversal = rootTraversal; - this.priorInput = []; - this.toKey = toKey; - } - } - - get knownCost(): number { - return this.calculation.getHeuristicFinalCost(); - } - - get inputSamplingCost(): number { - if(this._inputCost !== undefined) { - return this._inputCost; - } else { - let MIN_P = SearchSpace.MIN_KEYSTROKE_PROBABILITY; - // Should technically re-normalize the sampling distribution. - // -ln(p) is smaller for larger probabilities, as ln(p) is always <= 0. Approaches infinity as p => 0. - - // TODO: probably more efficient to instead use actual 'probability space'... but that'll involve extra changes. - this._inputCost = this.priorInput.map(mass => mass.p > MIN_P ? mass.p : MIN_P).reduce((previous, current) => previous - Math.log(current), 0); - return this._inputCost; - } - } - - // The part used to prioritize our search. - get currentCost(): number { - // - We reintrepret 'known cost' as a psuedo-probability. - // - Noting that 1/e = 0.367879441, an edit-distance cost of 1 may be intepreted as -ln(1/e) - a log-space 'likelihood'. - // - Not exactly normalized, though. - // That's a really, really high likelihood, thoough. - // - // At any rate, we can linearly scale the known-cost to have about whatever probability we want. - // If we can state it as p = 1 / (c * e), note then that ln(c * e) = ln(c) + 1. So, scale * (ln(c) + 1). - // If we can state it as e^x, note that ln(e^x) = x * ln(e) = x - just scale by 'x'! - - // p = 1 / (e^4) = 0.01831563888. This still exceeds many neighboring keys! - // p = 1 / (e^5) = 0.00673794699. Strikes a good balance. - // Should easily give priority to neighboring keys before edit-distance kicks in (when keys are a bit ambiguous) - return SearchSpace.EDIT_DISTANCE_COST_SCALE * this.knownCost + this.inputSamplingCost; - } - - buildInsertionEdges(): SearchNode[] { - let edges: SearchNode[] = []; - - for(let lexicalChild of this.currentTraversal.children()) { - let traversal = lexicalChild.traversal(); - let matchToken = { - key: lexicalChild.char, - traversal: traversal - } - - let childCalc = this.calculation.addMatchChar(matchToken); - - let searchChild = new SearchNode(this); - searchChild.calculation = childCalc; - searchChild.priorInput = this.priorInput; - searchChild.currentTraversal = traversal; - - edges.push(searchChild); - } - - return edges; - } - - buildDeletionEdges(inputDistribution: Distribution): SearchNode[] { - let edges: SearchNode[] = []; - - /* - * If the probability of an input is less than the highest probability * the base edit-distance likelihood, - * don't build an edge for it; just rely on edits from the highest-probability edge. - * - * We may be able to be stricter, but this should be a decent start. - * - * Note: thanks to ModelCompositor.predict, we know the distribution is pre-sorted. - */ - for(let probMass of inputDistribution) { - if(probMass.p < inputDistribution[0].p * Math.exp(-SearchSpace.EDIT_DISTANCE_COST_SCALE)) { - // Again, we're pre-sorted. All further entries will be too low-cost to consider. - break; - } - - let edgeCalc = this.calculation; - let transform = probMass.sample; - if(transform.deleteLeft) { - edgeCalc = edgeCalc.getSubset(edgeCalc.inputSequence.length - transform.deleteLeft, edgeCalc.matchSequence.length); - } - - // TODO: transform.deleteRight currently not supported. - - let inputPath = Array.from(this.priorInput); - inputPath.push(probMass); - // Tokenize and iterate over input chars, adding them into the calc. - for(let i=0; i < transform.insert.length; i++) { - let char = transform.insert[i]; - if(models.isHighSurrogate(char)) { - i++; - char = char + transform.insert[i]; - } - - // In case of NFD input, filter out any empty-strings that may arise - // when 'keying' raw diacritics. - let keyedChar = this.toKey(char); - if(keyedChar) { - edgeCalc = edgeCalc.addInputChar({key: keyedChar}); - } - } - - let childEdge = new SearchNode(this); - childEdge.calculation = edgeCalc; - childEdge.priorInput = inputPath; - - edges.push(childEdge); - } - - return edges; - } - - // While this may SEEM to be unnecessary, note that sometimes substitutions (which are computed - // via insert + delete) may be lower cost than both just-insert and just-delete. - buildSubstitutionEdges(inputDistribution: Distribution): SearchNode[] { - // Handles the 'input' component. - let intermediateEdges = this.buildDeletionEdges(inputDistribution); - let edges: SearchNode[] = []; - - for(let lexicalChild of this.currentTraversal.children()) { - for(let edge of intermediateEdges) { - let traversal = lexicalChild.traversal(); - let matchToken = { - key: lexicalChild.char, - traversal: traversal - } - - let childCalc = edge.calculation.addMatchChar(matchToken); - - let searchChild = new SearchNode(this); - searchChild.calculation = childCalc; - searchChild.priorInput = edge.priorInput; - searchChild.currentTraversal = traversal; - - edges.push(searchChild); - } - } - - return edges; - } - - get mapKey(): string { - let inputString = this.priorInput.map((value) => '+' + value.sample.insert + '-' + value.sample.deleteLeft).join(''); - let matchString = this.calculation.matchSequence.map((value) => value.key).join(''); - - // TODO: might should also track diagonalWidth. - return inputString + models.SENTINEL_CODE_UNIT + matchString; - } - - get isFullReplacement(): boolean { - // If the known edit-distance cost is equal to the input length, this means - // that literally every input has been full-on replaced. Thus, this is - // likely not a good 'root' to use for predictions. - // - // Logic exception: 0 cost, 0 length != a "replacement". - return this.knownCost && this.knownCost == this.priorInput.length; - } - } - - class SearchSpaceTier { - correctionQueue: models.PriorityQueue; - processed: SearchNode[] = []; - index: number; - - constructor(index: number, initialEdges?: SearchNode[]) { - this.index = index; - this.correctionQueue = new models.PriorityQueue(QUEUE_NODE_COMPARATOR, initialEdges); - } - - increaseMaxEditDistance() { - // By extracting the entries from the priority queue and increasing distance outside of it as a batch job, - // we get an O(N) implementation, rather than the O(N log N) that would result from maintaining the original queue. - let entries = this.correctionQueue.toArray(); - - entries.forEach(function(edge) { edge.calculation = edge.calculation.increaseMaxDistance(); }); - - // Since we just modified the stored instances, and the costs may have shifted, we need to re-heapify. - this.correctionQueue = new models.PriorityQueue(QUEUE_NODE_COMPARATOR, entries); - } - } - - export class SearchResult { - private resultNode: SearchNode; - - constructor(node: SearchNode) { - this.resultNode = node; - } - - get inputSequence(): ProbabilityMass[] { - return this.resultNode.priorInput; - } - - get matchSequence(): TraversableToken[] { - return this.resultNode.calculation.matchSequence; - }; - - get matchString(): USVString { - return this.matchSequence.map(value => value.key).join(''); - } - - get knownCost(): number { - return this.resultNode.knownCost; - } - - get inputSamplingCost(): number { - return this.resultNode.inputSamplingCost; - } - - get totalCost(): number { - return this.resultNode.currentCost; - } - - get finalTraversal(): LexiconTraversal { - return this.resultNode.currentTraversal; - } - } - - type NullPath = { - type: 'none' - } - - type IntermediateSearchPath = { - type: 'intermediate', - cost: number - } - - type CompleteSearchPath = { - type: 'complete', - cost: number, - finalNode: SearchNode - } - - type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; - - // The set of search spaces corresponding to the same 'context' for search. - // Whenever a wordbreak boundary is crossed, a new instance should be made. - export class SearchSpace { - private QUEUE_SPACE_COMPARATOR: models.Comparator; - - // p = 1 / (e^4) = 0.01831563888. This still exceeds many neighboring keys! - // p = 1 / (e^5) = 0.00673794699. Strikes a good balance. - // Should easily give priority to neighboring keys before edit-distance kicks in (when keys are a bit ambiguous) - static readonly EDIT_DISTANCE_COST_SCALE = 5; - static readonly MIN_KEYSTROKE_PROBABILITY = 0.0001; - static readonly DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL = 33; // in milliseconds. - - private tierOrdering: SearchSpaceTier[] = []; - private selectionQueue: models.PriorityQueue; - private inputSequence: Distribution[] = []; - private minInputCost: number[] = []; - private rootNode: SearchNode; - - // We use an array and not a PriorityQueue b/c batch-heapifying at a single point in time - // is cheaper than iteratively building a priority queue. - private completedPaths: SearchNode[]; - - // Marks all results that have already been returned since the last input was received. - private returnedValues: {[mapKey: string]: SearchNode} = {}; - - // Signals that the edge has already been processed. - private processedEdgeSet: {[mapKey: string]: boolean} = {}; - - constructor(model: LexicalModel) { - if(!model) { - throw "The LexicalModel parameter must not be null / undefined."; - } else if(!model.traverseFromRoot) { - throw "The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching."; - } - - // Constructs the comparator needed for the following line. - this.buildQueueSpaceComparator(); - this.selectionQueue = new models.PriorityQueue(this.QUEUE_SPACE_COMPARATOR); - this.rootNode = new SearchNode(model.traverseFromRoot(), model.toKey ? model.toKey.bind(model) : null); - - this.completedPaths = [this.rootNode]; - - // Adds a base level queue to handle initial insertions. - // Start with _just_ the root node. Necessary for proper empty-token, empty-input handling! - let baseTier = new SearchSpaceTier(0, [this.rootNode]); - this.tierOrdering.push(baseTier); - this.selectionQueue.enqueue(baseTier); - } - - private buildQueueSpaceComparator() { - let searchSpace = this; - - this.QUEUE_SPACE_COMPARATOR = function(space1, space2) { - let node1 = space1.correctionQueue.peek(); - let node2 = space2.correctionQueue.peek(); - - let index1 = space1.index; - let index2 = space2.index; - - let tierMinCost: number = 0; - let sign = 1; - - if(index2 < index1) { - let temp = index2; - index2 = index1; - index1 = temp; - - sign = -1; - } - - // Boost the cost of the lower tier by the minimum cost possible for the missing inputs between them. - // In essence, compare the nodes as if the lower tier had the most likely input appended for each such - // input missing at the lower tier. - // - // A 100% admissible heuristic to favor a deeper search, since the added cost is guaranteed if the path - // is traversed further. - // - // Remember, tier index i's last used input was from input index i-1. - // As a result, i is the first needed input index, with index2 - 1 the last entry needed to match them. - for(let i=index1; i < index2; i++) { - tierMinCost = tierMinCost + searchSpace.minInputCost[i]; - } - - // Guards, just in case one of the search spaces ever has an empty node. - if(node1 && node2) { - // If node1 is lower-tier, node1 is the one in need of boosted cost. - // `sign` flips it when node2 is lower tier. - return node1.currentCost - node2.currentCost + sign * tierMinCost; - } else if(node2) { - return 1; - } else { - return -1; - } - } - } - - increaseMaxEditDistance() { - this.tierOrdering.forEach(function(tier) { tier.increaseMaxEditDistance() }); - } - - addInput(inputDistribution: Distribution) { - this.inputSequence.push(inputDistribution); - - // Assumes that `inputDistribution` is already sorted. - this.minInputCost.push(-Math.log(inputDistribution[0].p)); - - // With a newly-available input, we can extend new input-dependent paths from - // our previously-reached 'extractedResults' nodes. - let newlyAvailableEdges: SearchNode[] = []; - let batches = this.completedPaths.map(function(node) { - let deletions = node.buildDeletionEdges(inputDistribution); - let substitutions = node.buildSubstitutionEdges(inputDistribution); - - return deletions.concat(substitutions); - }); - - // Don't forget to reset the array; the contained nodes no longer reach the search's end. - this.completedPaths = []; - this.returnedValues = {}; - - batches.forEach(function(batch) { - newlyAvailableEdges = newlyAvailableEdges.concat(batch); - }); - - // Now that we've built the new edges, we can efficiently construct the new search tier. - let tier = new SearchSpaceTier(this.tierOrdering.length, newlyAvailableEdges); - this.tierOrdering.push(tier); - this.selectionQueue.enqueue(tier); - } - - // TODO: will want eventually for reversions and/or backspaces - removeLastInput() { - // 1. truncate all entries from that search tier; we need to 'restore' extractedResults to match - // the state that would have existed without the last search tier. - // 2. remove the last search tier. Which may necessitate reconstructing the tier queue, but oh well. - } - - private hasNextMatchEntry(): boolean { - let topQueue = this.selectionQueue.peek(); - if(topQueue) { - return topQueue.correctionQueue.count > 0; - } else { - return false; - } - } - - private handleNextNode(): PathResult { - if(!this.hasNextMatchEntry()) { - return { type: 'none' }; - } - - let bestTier = this.selectionQueue.dequeue(); - let currentNode = bestTier.correctionQueue.dequeue(); - - let unmatchedResult: IntermediateSearchPath = { - type: 'intermediate', - cost: currentNode.currentCost - } - - // Have we already processed a matching edge? If so, skip it. - // We already know the previous edge is of lower cost. - if(this.processedEdgeSet[currentNode.mapKey]) { - this.selectionQueue.enqueue(bestTier); - return unmatchedResult; - } else { - this.processedEdgeSet[currentNode.mapKey] = true; - } - - // Stage 1: filter out nodes/edges we want to prune - - // Forbid a raw edit-distance of greater than 2. - // Note: .knownCost is not scaled, while its contribution to .currentCost _is_ scaled. - let substitutionsOnly = false; - if(currentNode.knownCost > 2) { - return unmatchedResult; - } else if(currentNode.knownCost == 2) { - // Hard restriction: no further edits will be supported. This helps keep the search - // more narrowly focused. - substitutionsOnly = true; - } - - let tierMinCost = 0; - for(let i = 0; i <= bestTier.index; i++) { - tierMinCost += this.minInputCost[i]; - } - - // Thresholds _any_ path, partially based on currently-traversed distance. - // Allows a little 'wiggle room' + 2 "hard" edits. - // Can be important if needed characters don't actually exist on the keyboard - // ... or even just not the then-current layer of the keyboard. - if(currentNode.currentCost > tierMinCost + 2.5 * SearchSpace.EDIT_DISTANCE_COST_SCALE) { - return unmatchedResult; - } - - // Stage 2: build remaining edges - - // Always possible, as this does not require any new input. - if(!substitutionsOnly) { - let insertionEdges = currentNode.buildInsertionEdges(); - bestTier.correctionQueue.enqueueAll(insertionEdges); - } - - if(bestTier.index == this.tierOrdering.length - 1) { - // It was the final tier - store the node for future reference. - this.completedPaths.push(currentNode); - - // Since we don't modify any other tier, we may simply reinsert the removed tier. - this.selectionQueue.enqueue(bestTier); - - return { - type: 'complete', - cost: currentNode.currentCost, - finalNode: currentNode - }; - } else { - // Time to construct new edges for the next tier! - let nextTier = this.tierOrdering[bestTier.index+1]; - - let inputIndex = nextTier.index; - - let deletionEdges = []; - if(!substitutionsOnly) { - deletionEdges = currentNode.buildDeletionEdges(this.inputSequence[inputIndex-1]); - } - let substitutionEdges = currentNode.buildSubstitutionEdges(this.inputSequence[inputIndex-1]); - - // Note: we're live-modifying the tier's cost here! The priority queue loses its guarantees as a result. - nextTier.correctionQueue.enqueueAll(deletionEdges.concat(substitutionEdges)); - - // So, we simply rebuild the selection queue. - this.selectionQueue = new models.PriorityQueue(this.QUEUE_SPACE_COMPARATOR, this.tierOrdering); - - // We didn't reach an end-node, so we just end the iteration and continue the search. - } - - // If we've somehow fully exhausted all search options, indicate that none remain. - return unmatchedResult; - } - - // Current best guesstimate of how compositor will retrieve ideal corrections. - *getBestMatches(waitMillis?: number): Generator { - // might should also include a 'base cost' parameter of sorts? - let searchSpace = this; - let currentReturns: {[mapKey: string]: SearchNode} = {}; - - let maxTime: number; - if(waitMillis == 0) { - maxTime = Infinity; - } else if(waitMillis == undefined || Number.isNaN(waitMillis)) { // also covers null. - maxTime = SearchSpace.DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL; - } else { - maxTime = waitMillis; - } - - /** - * This inner class is designed to help the algorithm detect its active execution time. - * While there's no official JS way to do this, we can approximate it by polling the - * current system time (in ms) after each iteration of a short-duration loop. Unusual - * spikes in system time for a single iteration is likely to indicate that an OS - * context switch occurred at some point during the iteration's execution. - */ - class ExecutionTimer { - /** - * The system time when this instance was created. - */ - private start: number; - - /** - * Marks the system time at the start of the currently-running loop, as noted - * by a call to the `startLoop` function. - */ - private loopStart: number; - - private maxExecutionTime: number; - private maxTrueTime: number; - - private executionTime: number; - - /** - * Used to track intervals in which potential context swaps by the OS may - * have occurred. Context switches generally seem to pause threads for - * at least 16 ms, while we expect each loop iteration to complete - * within just 1 ms. So, any possible context switch should have the - * longest observed change in system time. - * - * See `updateOutliers` for more details. - */ - private largestIntervals: number[] = [0]; - - constructor(maxExecutionTime: number, maxTrueTime: number) { - // JS measures time by the number of milliseconds since Jan 1, 1970. - this.loopStart = this.start = Date.now(); - this.maxExecutionTime = maxExecutionTime; - this.maxTrueTime = maxTrueTime; - } - - startLoop() { - this.loopStart = Date.now(); - } - - markIteration() { - const now = Date.now(); - const delta = now - this.loopStart; - this.executionTime += delta; - - /** - * Update the list of the three longest system-time intervals observed - * for execution of a single loop iteration. - * - * Ignore any zero-ms length intervals; they'd make the logic much - * messier than necessary otherwise. - */ - if(delta) { - // If the currently-observed interval is longer than the shortest of the 3 - // previously-observed longest intervals, replace it. - if(this.largestIntervals.length > 2 && delta > this.largestIntervals[0]) { - this.largestIntervals[0] = delta; - } else { - this.largestIntervals.push(delta); - } - // Puts the list in ascending order. Shortest of the list becomes the head, - // longest one the tail. - this.largestIntervals.sort(); - - // Then, determine if we need to update our outlier-based tweaks. - this.updateOutliers(); - } - } - - updateOutliers() { - /* Base assumption: since each loop of the search should evaluate within ~1ms, - * notably longer execution times are probably context switches. - * - * Base assumption: OS context switches generally last at least 16ms. (Based on - * a window.setTimeout() usually not evaluating for at least - * that long, even if set to 1ms.) - * - * To mitigate these assumptions: we'll track the execution time of every loop - * iteration. If the longest observation somehow matches or exceeds the length of - * the next two almost-longest observations twice over... we have a very strong - * 'context switch' candidate. - * - * Or, in near-formal math/stats: we expect a very low variance in execution - * time among the iterations of the search's loops. With a very low variance, - * ANY significant proportional spikes in execution time are outliers - outliers - * likely caused by an OS context switch. - * - * Rather than do intensive math, we use a somewhat lazy approach below that - * achieves the same net results given our assumptions, even when relaxed somewhat. - * - * The logic below relaxes the base assumptions a bit to be safe: - * - [2ms, 2ms, 8ms] will cause 8ms to be seen as an outlier. - * - [2ms, 3ms, 10ms] will cause 10ms to be seen as an outlier. - * - * Ideally: - * - [1ms, 1ms, 4ms] will view 4ms as an outlier. - * - * So we can safely handle slightly longer average intervals and slightly shorter - * OS context-switch time intervals. - */ - if(this.largestIntervals.length > 2) { - // Precondition: the `largestIntervals` array is sorted in ascending order. - // Shortest entry is at the head, longest at the tail. - if(this.largestIntervals[2] >= 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { - this.executionTime -= this.largestIntervals[2]; - this.largestIntervals.pop(); - } - } - } - - shouldTimeout(): boolean { - const now = Date.now(); - if(now - this.start > this.maxTrueTime) { - return true; - } - - return this.executionTime > this.maxExecutionTime; - } - - resetOutlierCheck() { - this.largestIntervals = []; - } - } - - class BatchingAssistant { - currentCost = Number.MIN_SAFE_INTEGER; - entries: SearchResult[] = []; - - checkAndAdd(entry: SearchNode): SearchResult[] | null { - var result: SearchResult[] = null; - - if(entry.currentCost > this.currentCost) { - result = this.tryFinalize(); - - this.currentCost = entry.currentCost; - } - - // Filter out any duplicated match sequences. The same match sequence may be reached via - // different input sequences, after all. - let outputMapKey = entry.calculation.matchSequence.map(value => value.key).join(''); - - // First, ensure the edge has an existing 'shared' cache entry. - if(!searchSpace.returnedValues[outputMapKey]) { - searchSpace.returnedValues[outputMapKey] = entry; - } - - // Check the generator's local returned-value cache - this determines whether or not we - // need to add a new 'return' to the batch. - if(!currentReturns[outputMapKey]) { - this.entries.push(new SearchResult(entry)); - currentReturns[outputMapKey] = entry; - } - - return result; - } - - tryFinalize(): SearchResult[] | null { - var result: SearchResult[] = null; - if(this.entries.length > 0) { - result = this.entries; - this.entries = []; - } - - return result; - } - } - - let batcher = new BatchingAssistant(); - - const timer = new ExecutionTimer(maxTime*1.5, maxTime); - - // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. - let returnedValues = Object.values(this.returnedValues); - if(returnedValues.length > 0) { - let preprocessedQueue = new models.PriorityQueue(QUEUE_NODE_COMPARATOR, returnedValues); - - // Build batches of same-cost entries. - timer.startLoop(); - while(preprocessedQueue.count > 0) { - let entry = preprocessedQueue.dequeue(); - - // Is the entry a reasonable result? - if(entry.isFullReplacement) { - // If the entry's 'match' fully replaces the input string, we consider it - // unreasonable and ignore it. - continue; - } - - let batch = batcher.checkAndAdd(entry); - timer.markIteration(); - - if(batch) { - // Do not track yielded time. - yield batch; - } - } - - // As we only return a batch once all entries of the same cost have been processed, we can safely - // finalize the last preprocessed group without issue. - let batch = batcher.tryFinalize(); - if(batch) { - // Do not track yielded time. - yield batch; - } - } - - // Stage 2: the fun part; actually searching! - timer.resetOutlierCheck(); - timer.startLoop(); - let timedOut = false; - do { - let newResult: PathResult; - - // Search for a 'complete' path, skipping all partial paths as long as time remains. - do { - newResult = this.handleNextNode(); - timer.markIteration(); - - if(timer.shouldTimeout()) { - timedOut = true; - } - } while(!timedOut && newResult.type == 'intermediate') - - // TODO: check 'cost' on intermediate, running it through batcher to early-detect cost changes. - let batch: SearchResult[]; - if(newResult.type == 'none') { - break; - } else if(newResult.type == 'complete') { - // Is the entry a reasonable result? - if(newResult.finalNode.isFullReplacement) { - // If the entry's 'match' fully replaces the input string, we consider it - // unreasonable and ignore it. Also, if we've reached this point... - // we can(?) assume that everything thereafter is as well. - break; - } - batch = batcher.checkAndAdd(newResult.finalNode); - } - - if(batch) { - yield batch; - } - } while(!timedOut && this.hasNextMatchEntry()); - - // If we _somehow_ exhaust all search options, make sure to return the final results. - let batch = batcher.tryFinalize(); - if(batch) { - yield batch; - } - - return null; - } - } -} diff --git a/common/web/lm-worker/src/main/correction/classical-calculation.ts b/common/web/lm-worker/src/main/correction/classical-calculation.ts new file mode 100644 index 0000000000000000000000000000000000000000..50ddf935513a654a115bc361d5e72dd874c0e823 --- /dev/null +++ b/common/web/lm-worker/src/main/correction/classical-calculation.ts @@ -0,0 +1,572 @@ +import { SENTINEL_CODE_UNIT } from '@keymanapp/models-templates'; + +type EditOperation = 'insert' | 'delete' | 'match' | 'substitute' | 'transpose-start' | 'transpose-end' | 'transpose-insert' | 'transpose-delete'; + +/** + * Represents the lowest-level unit for comparison during edit-distance calculations. + */ +export interface EditToken { + key: TUnit; +} + +// A semi-optimized 'online'/iterative Damerau-Levenshtein calculator with the following features: +// - may add new character to the 'input' string or to the 'match' string, reusing all old calculations efficiently. +// - allows a 'focused' evaluation that seeks if the edit distance is within a specific range. Designed for use in match-searching, +// where we want to find the 'closest' matching strings in a lexicon. +// - towards such a match-searching algorithm/heuristic: should nothing be found within that range, all prior calculations may be reused +// to search across the lexicon with an incremented edit distance. +// - minimized memory footprint: O(m) memory footprint (where m = length of 'input' string), rather than O(mn) (where n = length of 'match' string) +// - guaranteed to use a smaller footprint than DiagonalizedIterativeDamerauLevenshteinCalculation. +// +// In short: Used to optimize calculations for low edit-distance checks, then expanded if/as necessary +// if a greater edit distance is requested. +// +// Reference: https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm#Possible_modifications +// - Motivating statement: "if we are only interested in the distance if it is smaller than a threshold..." +export class ClassicalDistanceCalculation = EditToken, TMatch extends EditToken = EditToken> { + /** + * Stores ONLY the computed diagonal elements, nothing else. + * + * Mapped as seen in the example below (with a diagonal of width 1): + * ``` + * MAX | MAX | MAX | MAX | MAX | ... + * MAX | 0 | 1 | 2 | 3 | ... > + * MAX | 1 | a | b | - | ... ====>> | - | a | b | + * MAX | 2 | c | d | e | ... > | c | d | e | + * MAX | 3 | - | f | g | ... | f | g | ... | + * ... | ... | ... | ... | ... | ... | ... | ... | ... | + * ``` + * + * Any "`-`" entries are undefined, as they lie outside of the diagonal under consideration. + * + * Things of note: + * - The entry where row index = col index will always lie at the center of the row's array. + * - For each +1 increase in row index, the row's entries are (logically) shifted by -1 in order to make this happen. + * - As all of the MAX entries and numerical entries above are fixed, known values, they are not represented here. + */ + resolvedDistances: number[][]; + /** + * Specifies how far off-diagonal calculations should be performed. A value of 0 only evaluates cells with matching + * row and column indicies. + * + * The resulting value from .getFinalCost() is only guaranteed correct if it is less than or equal to this value. + * Otherwise, this object represents a heuristic that _may_ overestimate the true edit distance. Note that it will + * never underestimate. + */ + diagonalWidth: number = 2; // TODO: Ideally, should start at 1... but we'll start at 2 for now + // as a naive workaround for multi-char transform limitations. + + // The sequence of characters input so far. + inputSequence: TInput[] = []; + matchSequence: TMatch[] = []; + + constructor(); + constructor(other: ClassicalDistanceCalculation); + constructor(other?: ClassicalDistanceCalculation) { + if(other) { + // Clone class properties. + let rowCount = other.resolvedDistances.length; + this.resolvedDistances = Array(rowCount); + + for(let r = 0; r < rowCount; r++) { + this.resolvedDistances[r] = Array.from(other.resolvedDistances[r]); + } + + this.inputSequence = Array.from(other.inputSequence); + this.matchSequence = Array.from(other.matchSequence); + this.diagonalWidth = other.diagonalWidth; + } else { + this.resolvedDistances = []; + } + } + + private getTrueIndex(r: number, c: number, width: number): {row: number, col: number, sparse: boolean} { + let retVal = { + row: r, + col: c - r + width, + sparse: false + } + + if(retVal.col < 0 || retVal.col > 2 * width) { + retVal.sparse = true; + } + + return retVal; + } + + private getCostAt(i: number, j: number, width: number = this.diagonalWidth): number { + // Check for and handle the set of fixed-value virtualized indices. + if(i < 0 || j < 0) { + if(i == -1 && j >= -1) { + return j+1; + } else if(j == -1 && i >= -1) { + return i+1; + } + + return Number.MAX_VALUE; + } + + let index = this.getTrueIndex(i, j, width); + return index.sparse ? Number.MAX_VALUE : this.resolvedDistances[index.row][index.col]; + } + + /** + * Noting the above link's statement prefixed "By examining diagonals instead of rows, and by using lazy evaluation...", + * this function will return the actual edit distance between the strings, temporarily increasing the computed + * diagonal's size if necessary. + * + * Does not actually mutate the instance. + */ + getFinalCost(): number { + let buffer = this as ClassicalDistanceCalculation; + let val = buffer.getHeuristicFinalCost(); + + while(val > buffer.diagonalWidth) { + // A consequence of treating this class as immutable. + buffer = buffer.increaseMaxDistance(); + val = buffer.getHeuristicFinalCost(); + } + + return val; + } + + /** + * Returns this instance's computed edit distance. If greater than the diagonal's width value, note that it may be an overestimate. + */ + getHeuristicFinalCost(): number { + return this.getCostAt(this.inputSequence.length-1, this.matchSequence.length-1); + } + + /** + * Returns `true` if the represented edit distance is less than or equal to the specified threshold, minimizing the amount of calculations + * needed to meet the specified limit. + * + * Does not mutate the instance. + * @param threshold + */ + hasFinalCostWithin(threshold: number): boolean { + let buffer = this as ClassicalDistanceCalculation; + let val = buffer.getHeuristicFinalCost(); + let guaranteedBound = this.diagonalWidth; + + do { + // val will never exceed the length of the longer string, no matter how large the threshold. + if(val <= threshold) { + return true; + } else if(guaranteedBound < threshold) { + buffer = buffer.increaseMaxDistance(); + guaranteedBound++; + val = buffer.getHeuristicFinalCost(); + } else { + break; + } + } while(true); + + return false; + } + + /** + * Determines the edit path used to obtain the optimal cost, distinguishing between zero-cost + * substitutions ('match' operations) and actual substitutions. + * @param row + * @param col + */ + public editPath(row: number = this.inputSequence.length - 1, col: number = this.matchSequence.length - 1): EditOperation[] { + let currentCost = this.getCostAt(row, col); + let ops: EditOperation[] = null; + let parent: [number, number] = null; + + let insertParentCost = this.getCostAt(row, col-1); + let deleteParentCost = this.getCostAt(row-1, col); + let substitutionParentCost = this.getCostAt(row-1, col-1); + let [lastInputIndex, lastMatchIndex] = ClassicalDistanceCalculation.getTransposeParent(this, row, col); + if(lastInputIndex >= 0 && lastMatchIndex >= 0) { + // OK, a transposition source is quite possible. Still need to do more vetting, to be sure. + let expectedCost = 1; + + // This transposition includes either 'transpose-insert' or 'transpose-delete' operations. + ops = ['transpose-start']; // always needs a 'start'. + if(lastInputIndex != row-1) { + let count = row - lastInputIndex - 1; + ops = ops.concat( Array(count).fill('transpose-delete') ); + expectedCost += count; + } else { + let count = col - lastMatchIndex - 1; + ops = ops.concat( Array(count).fill('transpose-insert') ); + expectedCost += count; + } + ops.push('transpose-end'); + + // Double-check our expectations. + if(this.getCostAt(lastInputIndex-1, lastMatchIndex-1) != currentCost - expectedCost) { + ops = null; + } + parent = [lastInputIndex-1, lastMatchIndex-1]; + } + + if(ops) { + // bypass the ladder. + } else if(substitutionParentCost == currentCost - 1) { + ops = ['substitute']; + parent = [row-1, col-1]; + } else if(insertParentCost == currentCost - 1) { + ops = ['insert']; + parent = [row, col-1]; + } else if(deleteParentCost == currentCost - 1) { + ops = ['delete']; + parent = [row-1, col]; + } else { //if(substitutionParentCost == currentCost) { + ops = ['match']; + parent = [row-1, col-1]; + } + + // Recursively build the edit path. + if(parent[0] >= 0 && parent[1] >= 0) { + return this.editPath(parent[0], parent[1]).concat(ops); + } else { + if(parent[0] > -1) { + // There are initial deletions. + return Array(parent[0]+1).fill('delete').concat(ops); + } else if(parent[1] > -1) { + // There are initial insertions. + return Array(parent[1]+1).fill('insert').concat(ops); + } else { + return ops; + } + } + } + + private static getTransposeParent, TMatch extends EditToken>( + buffer: ClassicalDistanceCalculation, + r: number, + c: number + ): [number, number] { + // Block any transpositions where the tokens are identical. + // Other operations will be cheaper. Also, block cases where 'parents' are impossible. + if(r < 0 || c < 0 || buffer.inputSequence[r].key == buffer.matchSequence[c].key) { + return [-1, -1]; + } + + // Transposition checks + let lastInputIndex = -1; + for(let i = r-1; i >= 0; i--) { + if(buffer.inputSequence[i].key == buffer.matchSequence[c].key) { + lastInputIndex = i; + break; + } + } + + let lastMatchIndex = -1; + for(let i = c-1; i >= 0; i--) { + if(buffer.matchSequence[i].key == buffer.inputSequence[r].key) { + lastMatchIndex = i; + break; + } + } + + return [lastInputIndex, lastMatchIndex]; + } + + private static initialCostAt, TMatch extends EditToken>( + buffer: ClassicalDistanceCalculation, + r: number, + c: number, + insertCost?: number, + deleteCost?: number) { + var baseSubstitutionCost = buffer.inputSequence[r].key == buffer.matchSequence[c].key ? 0 : 1; + var substitutionCost: number = buffer.getCostAt(r-1, c-1) + baseSubstitutionCost; + var insertionCost: number = insertCost || buffer.getCostAt(r, c-1) + 1; // If set meaningfully, will never equal zero. + var deletionCost: number = deleteCost || buffer.getCostAt(r-1, c) + 1; // If set meaningfully, will never equal zero. + var transpositionCost: number = Number.MAX_VALUE + + if(r > 0 && c > 0) { // bypass when transpositions are known to be impossible. + let [lastInputIndex, lastMatchIndex] = ClassicalDistanceCalculation.getTransposeParent(buffer, r, c); + transpositionCost = buffer.getCostAt(lastInputIndex-1, lastMatchIndex-1) + (r - lastInputIndex - 1) + 1 + (c - lastMatchIndex - 1); + } + + return Math.min(substitutionCost, deletionCost, insertionCost, transpositionCost); + } + + getSubset(inputLength: number, matchLength: number): ClassicalDistanceCalculation { + let trimmedInstance = new ClassicalDistanceCalculation(this); + + if(inputLength > this.inputSequence.length || matchLength > this.matchSequence.length) { + throw "Invalid dimensions specified for trim operation"; + } + // Trim our tracked input & match sequences. + trimmedInstance.inputSequence.splice(inputLength); + trimmedInstance.matchSequence.splice(matchLength); + + // Major index corresponds to input length. + trimmedInstance.resolvedDistances.splice(inputLength); + + // The real fun: trimming off columns. (Minor index, corresponds to match length) + let finalTrueIndex = this.getTrueIndex(inputLength-1, matchLength-1, this.diagonalWidth); + // The diagonal index increases as the row index decreases. + for(let diagonalIndex = finalTrueIndex.col; diagonalIndex <= 2 * this.diagonalWidth; diagonalIndex++) { + let row = finalTrueIndex.row - (diagonalIndex - finalTrueIndex.col); + if(row < 0) { + break; + } + + if(diagonalIndex < 0) { + trimmedInstance.resolvedDistances[row] = Array(2 * trimmedInstance.diagonalWidth + 1).fill(Number.MAX_VALUE); + } else { + let newCount = 2 * this.diagonalWidth - diagonalIndex; + let keptEntries = trimmedInstance.resolvedDistances[row].splice(0, diagonalIndex+1); + let newEntries = Array(newCount).fill(Number.MAX_VALUE); + trimmedInstance.resolvedDistances[row] = keptEntries.concat(newEntries); + } + } + + return trimmedInstance; + } + + private static forDiagonalOfAxis(diagonalWidth: number, centerIndex: number, axisCap: number, closure: (axisIndex: number, diagIndex: number) => void) { + let diagonalCap = axisCap - centerIndex < diagonalWidth ? axisCap - centerIndex + diagonalWidth : 2 * diagonalWidth; + let startOffset = centerIndex - diagonalWidth; // The axis's index for diagonal entry 0. May be negative. + let diagonalStart = startOffset < 0 ? 0 : startOffset; + + for(let diagonalIndex = diagonalStart - startOffset; diagonalIndex <= diagonalCap; diagonalIndex++) { + closure(startOffset + diagonalIndex, diagonalIndex); + } + } + + // Inputs add an extra row / first index entry. + addInputChar(token: TInput): ClassicalDistanceCalculation { + let returnBuffer = new ClassicalDistanceCalculation(this); + + let r = returnBuffer.inputSequence.length; + returnBuffer.inputSequence.push(token); + + // Insert a row, even if we don't actually do anything with it yet. + // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues. + let row = Array(2 * returnBuffer.diagonalWidth + 1).fill(Number.MAX_VALUE); + returnBuffer.resolvedDistances[r] = row; + + // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. + if(returnBuffer.matchSequence.length == 0) { + return returnBuffer; + } + + ClassicalDistanceCalculation.forDiagonalOfAxis(returnBuffer.diagonalWidth, r, returnBuffer.matchSequence.length - 1, function(c, diagIndex) { + row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); + }); + + return returnBuffer; + } + + addMatchChar(token: TMatch): ClassicalDistanceCalculation { + let returnBuffer = new ClassicalDistanceCalculation(this); + + let c = returnBuffer.matchSequence.length; + returnBuffer.matchSequence.push(token); + + // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. + if(returnBuffer.inputSequence.length == 0) { + return returnBuffer; + } + + ClassicalDistanceCalculation.forDiagonalOfAxis(returnBuffer.diagonalWidth, c, returnBuffer.inputSequence.length - 1, function(r, diagIndex) { + var row = returnBuffer.resolvedDistances[r]; + // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column. + row[2 * returnBuffer.diagonalWidth - diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); + }); + + return returnBuffer; + } + + public increaseMaxDistance(): ClassicalDistanceCalculation { + let returnBuffer = new ClassicalDistanceCalculation(this); + returnBuffer.diagonalWidth++; + + if(returnBuffer.inputSequence.length < 1 || returnBuffer.matchSequence.length < 1) { + return returnBuffer; + } + + // An abstraction of the common aspects of transposition handling during diagonal extensions. + function forPossibleTranspositionsInDiagonal(startPos: number, fixedChar: TUnit, lookupString: EditToken[], closure: (axisIndex: number, diagIndex: number) => void) { + let diagonalCap = 2 * (returnBuffer.diagonalWidth - 1); // The maximum diagonal index permitted + let axisCap = lookupString.length - 1; // The maximum index supported by the axis of iteration + + // Ensures that diagonal iteration only occurs within the axis's supported range + diagonalCap = diagonalCap < axisCap - startPos ? diagonalCap : axisCap - startPos; + + // Iterate within the diagonal and call our closure for any potential transpositions. + for(let diagonalIndex = 0; diagonalIndex <= diagonalCap; diagonalIndex++) { + if(fixedChar == lookupString[startPos + diagonalIndex].key) { + closure(startPos + diagonalIndex, diagonalIndex); + } + } + } + + for(let r = 0; r < returnBuffer.inputSequence.length; r++) { + let leftCell = Number.MAX_VALUE; + let c = r - returnBuffer.diagonalWidth // External index of the left-most entry, which we will now calculate. + if(c >= 0) { + // If c == 0, cell is at edge, thus a known value for insertions exists. + // Base cost: r+1, +1 for inserting. + let insertionCost = c == 0 ? r + 2 : Number.MAX_VALUE; + // compute new left cell + leftCell = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c, insertionCost, undefined); + let addedCost = leftCell; + + // daisy-chain possible updates + + // cell (r, c+1): new insertion source + if(c < returnBuffer.matchSequence.length-1) { + // We propagate the new added cost (via insertion) to the old left-most cell, which is one to our right. + ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, r, c+1, addedCost+1, 0); + + // Only possible if insertions are also possible AND more conditions are met. + // cells (r+2, * > c+2): new transposition source + let transposeRow = r+2; + if(r+2 < this.inputSequence.length) { // Row to check for transposes must exist. + let rowChar = returnBuffer.inputSequence[r+1].key; + // First possible match in input could be at index c + 2, which adjusts col c+2's cost. Except that entry in r+2 + // doesn't exist yet - so we start with c+3 instead. + forPossibleTranspositionsInDiagonal(c + 3, rowChar, returnBuffer.matchSequence, function(axisIndex, diagIndex) { + // Because (r+2, c+3) is root, not (r+2, c+2). Min cost of 2. + ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, transposeRow, axisIndex, addedCost + diagIndex + 2, diagIndex); + }); + } + } + } + + let rightCell = Number.MAX_VALUE; + c = r + returnBuffer.diagonalWidth; + if(c < returnBuffer.matchSequence.length) { + // If r == 0, cell is at edge, thus a known value for insertions exists. + // Base cost: c+1, +1 for inserting. + let deletionCost = r == 0 ? c + 2 : Number.MAX_VALUE; + + // the current row wants to use adjusted diagonal width; we must specify use of the old width & its mapping instead. + var insertionCost: number = returnBuffer.getCostAt(r, c-1, this.diagonalWidth) + 1; + + // compute new right cell + rightCell = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c, insertionCost, deletionCost); + let addedCost = rightCell; + + // daisy-chain possible updates + + // cell (r+1, c): new deletion source + if(r < returnBuffer.inputSequence.length - 1) { + // We propagate the new added cost (via deletion) to the old right-most cell, which is one to our right. + ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, r+1, c, addedCost + 1, 2 * this.diagonalWidth); + + // Only possible if deletions are also possible AND more conditions are met. + // cells(* > r+2, c+2): new transposition source + let transposeCol = c+2; + if(c+2 < this.matchSequence.length) { // Row to check for transposes must exist. + let colChar = returnBuffer.matchSequence[r+1].key; + // First possible match in input could be at index r + 2, which adjusts row r+2's cost. Except that entry in c+2 + // doesn't exist yet - so we start with r+3 instead. + forPossibleTranspositionsInDiagonal(r+3, colChar, returnBuffer.inputSequence, function(axisIndex, diagIndex) { + let diagColIndex = 2 * (returnBuffer.diagonalWidth - 1) - diagIndex; + // Because (r+3, c+2) is root, not (r+2, c+2). Min cost of 2. + ClassicalDistanceCalculation.propagateUpdateFrom(returnBuffer, axisIndex, transposeCol, addedCost + diagIndex + 2, diagColIndex); + }); + } + } + } + + // Constructs the final expanded diagonal for the row. + returnBuffer.resolvedDistances[r] = [leftCell].concat(returnBuffer.resolvedDistances[r], rightCell); + } + + return returnBuffer; + } + + private static propagateUpdateFrom, TMatch extends EditToken>( + buffer: ClassicalDistanceCalculation, + r: number, + c: number, + value: number, + diagonalIndex: number) { + // Note: this function does not actually need the `c` parameter! + // That said, it's very useful when tracing stack traces & debugging. + if(value < buffer.resolvedDistances[r][diagonalIndex]) { + buffer.resolvedDistances[r][diagonalIndex] = value; + } else { + return + } + + let internalRow = r < buffer.inputSequence.length - 1; + let internalCol = c < buffer.matchSequence.length - 1; + + // We have to compensate for the current & following rows not having been expanded yet. + if(diagonalIndex < 2 * (buffer.diagonalWidth - 1) && internalCol) { + // We've inserted to the left of an existing calculation - check for propagation via insertion. + let updateCost = value + 1; + this.propagateUpdateFrom(buffer, r, c+1, updateCost, diagonalIndex+1); + } + + if(diagonalIndex > 0 && internalRow) { + // We've inserted above an existing calculation - check for propagation via deletion + let updateCost = value + 1 + this.propagateUpdateFrom(buffer, r+1, c, updateCost, diagonalIndex-1); + } + + // If both, check for propagation via substitution and possible transpositions + if(internalRow && internalCol) { + let updateCost = value + (buffer.inputSequence[r+1].key == buffer.matchSequence[c+1].key ? 0 : 1); + this.propagateUpdateFrom(buffer, r+1, c+1, updateCost, diagonalIndex); + + // Propagating transpositions (only possible if 'internal'.) + let nextInputIndex = -1; + for(let i = r+2; i < buffer.inputSequence.length; i++) { + if(buffer.inputSequence[i].key == buffer.matchSequence[c+1].key) { + nextInputIndex = i; + break; + } + } + + let nextMatchIndex = -1; + for(let i = c+2; i < buffer.matchSequence.length; i++) { + if(buffer.matchSequence[i].key == buffer.inputSequence[r+1].key) { + nextMatchIndex = i; + break; + } + } + + if(nextInputIndex > 0 && nextMatchIndex > 0) { + let transpositionCost = value + (nextInputIndex - r - 2) + 1 + (nextMatchIndex - c - 2); + this.propagateUpdateFrom(buffer, nextInputIndex, nextMatchIndex, transpositionCost, (buffer.diagonalWidth - 1) + nextMatchIndex - nextInputIndex); + } + } + } + + get mapKey(): string { + let inputString = this.inputSequence.map((value) => value.key).join(''); + let matchString = this.matchSequence.map((value) => value.key).join(''); + return inputString + SENTINEL_CODE_UNIT + matchString + SENTINEL_CODE_UNIT + this.diagonalWidth; + } + + get lastInputEntry(): TInput { + return this.inputSequence[this.inputSequence.length-1]; + } + + get lastMatchEntry(): TMatch { + return this.matchSequence[this.matchSequence.length-1]; + } + + static computeDistance, TMatch extends EditToken>( + input: TInput[], + match: TMatch[], + bandSize: number = 1) { + // Initialize the calculation buffer, setting the diagonal width (as appropriate) in advance. + let buffer = new ClassicalDistanceCalculation(); + bandSize = bandSize || 1; + buffer.diagonalWidth = bandSize; + + for(let i = 0; i < input.length; i++) { + buffer = buffer.addInputChar(input[i]); + } + + for(let j = 0; j < match.length; j++) { + buffer = buffer.addMatchChar(match[j]); + } + + return buffer; + } +} \ No newline at end of file diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts new file mode 100644 index 0000000000000000000000000000000000000000..27783a4852823112886990139d7482f8bd062c54 --- /dev/null +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -0,0 +1,542 @@ +import { tokenize } from '@keymanapp/models-templates'; +import { defaultWordbreaker } from '@keymanapp/models-wordbreakers'; + +import { ClassicalDistanceCalculation } from './classical-calculation.js'; +import { SearchSpace } from './distance-modeler.js'; +import TransformUtils from '../transformUtils.js'; + +function textToCharTransforms(text: string, transformId?: number) { + let perCharTransforms: Transform[] = []; + + for(let i=0; i < text.kmwLength(); i++) { + let char = text.kmwCharAt(i); // is SMP-aware + + let transform: Transform = { + insert: char, + deleteLeft: 0, + id: transformId + }; + + perCharTransforms.push(transform); + } + + return perCharTransforms; +} + +export class TrackedContextSuggestion { + suggestion: Suggestion; + tokenWidth: number; +} + +export class TrackedContextToken { + raw: string; + replacementText: string; + + transformDistributions: Distribution[] = []; + replacements: TrackedContextSuggestion[]; + activeReplacementId: number = -1; + + get currentText(): string { + if(this.replacementText === undefined || this.replacementText === null) { + return this.raw; + } else { + return this.replacementText; + } + } + + get replacement(): TrackedContextSuggestion { + let replacementId = this.activeReplacementId; + return this.replacements.find(function(replacement) { + return replacement.suggestion.id == replacementId; + }); + } + + revert() { + delete this.activeReplacementId; + } +} + +export class TrackedContextState { + // Stores the source Context (as a debugging reference). Not currently utilized. + taggedContext: Context; + model: LexicalModel; + + tokens: TrackedContextToken[]; + /** + * How many tokens were removed from the start of the best-matching ancestor. + * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position. + */ + indexOffset: number; + + // Tracks all search spaces starting at the current token. + // In the lm-layer's current form, this should only ever have one entry. + // Leaves 'design space' for if/when we add support for phrase-level corrections/predictions. + searchSpace: SearchSpace[] = []; + + constructor(source: TrackedContextState); + constructor(model: LexicalModel); + constructor(obj: TrackedContextState | LexicalModel) { + if(obj instanceof TrackedContextState) { + let source = obj; + // Be sure to deep-copy the tokens! Pointer-aliasing is bad here. + this.tokens = source.tokens.map(function(token) { + let copy = new TrackedContextToken(); + copy.raw = token.raw; + copy.replacements = token.replacements + copy.activeReplacementId = token.activeReplacementId; + copy.transformDistributions = token.transformDistributions; + + if(token.replacementText) { + copy.replacementText = token.replacementText; + } + + return copy; + }); + this.searchSpace = obj.searchSpace; + this.indexOffset = 0; + this.model = obj.model; + } else { + let lexicalModel = obj; + this.tokens = []; + this.indexOffset = Number.MIN_SAFE_INTEGER; + this.model = lexicalModel; + + if(lexicalModel && lexicalModel.traverseFromRoot) { + this.searchSpace = [new SearchSpace(lexicalModel)]; + } + } + } + + get head(): TrackedContextToken { + return this.tokens[0]; + } + + get tail(): TrackedContextToken { + return this.tokens[this.tokens.length - 1]; + } + + popHead() { + this.tokens.splice(0, 2); + this.indexOffset -= 1; + } + + pushTail(token: TrackedContextToken) { + if(this.model && this.model.traverseFromRoot) { + this.searchSpace = [new SearchSpace(this.model)]; // yeah, need to update SearchSpace for compatibility + } else { + this.searchSpace = []; + } + this.tokens.push(token); + + let state = this; + if(state.searchSpace.length > 0) { + token.transformDistributions.forEach(distrib => state.searchSpace[0].addInput(distrib)); + } + } + + pushWhitespaceToTail(transformDistribution: Distribution = null) { + let whitespaceToken = new TrackedContextToken(); + + // Track the Transform that resulted in the whitespace 'token'. + // Will be needed for phrase-level correction/prediction. + whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + + whitespaceToken.raw = null; + this.tokens.push(whitespaceToken); + } + + /** + * Used for 14.0's backspace workaround, which flattens all previous Distribution + * entries because of limitations with direct use of backspace transforms. + * @param tokenText + * @param transformId + */ + replaceTailForBackspace(tokenText: USVString, transformId: number) { + this.tokens.pop(); + + // It's a backspace transform; time for special handling! + // + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. + // + // Note that we cannot just use a single, monolithic transform at this point b/c + // of our current edit-distance optimization strategy; diagonalization is currently... + // not very compatible with that. + let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { + return [{sample: transform, p: 1.0}]; + }); + + let compactedToken = new TrackedContextToken(); + compactedToken.raw = tokenText; + compactedToken.transformDistributions = backspacedTokenContext; + this.pushTail(compactedToken); + } + + updateTail(transformDistribution: Distribution, tokenText?: USVString) { + let editedToken = this.tail; + + // Preserve existing text if new text isn't specified. + tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); + + if(transformDistribution && transformDistribution.length > 0) { + editedToken.transformDistributions.push(transformDistribution); + if(this.searchSpace) { + this.searchSpace.forEach(space => space.addInput(transformDistribution)); + } + } + // Replace old token's raw-text with new token's raw-text. + editedToken.raw = tokenText; + } + + toRawTokenization() { + let sequence: USVString[] = []; + + for(let token of this.tokens) { + // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) + if(token.currentText !== null) { + sequence.push(token.currentText); + } + } + + return sequence; + } +} + +class CircularArray { + static readonly DEFAULT_ARRAY_SIZE = 5; + private circle: Item[]; + private currentHead: number=0; + private currentTail: number=0; + + constructor(size: number = CircularArray.DEFAULT_ARRAY_SIZE) { + this.circle = Array(size); + } + + get count(): number { + let diff = this.currentHead - this.currentTail; + + if(diff < 0) { + diff = diff + this.circle.length; + } + + return diff; + } + + get maxCount(): number { + return this.circle.length; + } + + get oldest(): Item { + if(this.count == 0) { + return undefined; + } + + return this.item(0); + } + + get newest(): Item { + if(this.count == 0) { + return undefined; + } + + return this.item(this.count - 1); + } + + enqueue(item: Item): Item { + var prevItem = null; + let nextHead = (this.currentHead + 1) % this.maxCount; + + if(nextHead == this.currentTail) { + prevItem = this.circle[this.currentTail]; + this.currentTail = (this.currentTail + 1) % this.maxCount; + } + + this.circle[this.currentHead] = item; + this.currentHead = nextHead; + + return prevItem; + } + + dequeue(): Item { + if(this.currentTail == this.currentHead) { + return null; + } else { + let item = this.circle[this.currentTail]; + this.currentTail = (this.currentTail + 1) % this.maxCount; + return item; + } + } + + popNewest(): Item { + if(this.currentTail == this.currentHead) { + return null; + } else { + let item = this.circle[this.currentHead]; + this.currentHead = (this.currentHead - 1 + this.maxCount) % this.maxCount; + return item; + } + } + + /** + * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - + * the same order in which the items will be dequeued. + * @param index + */ + item(index: number) { + if(index >= this.count) { + throw "Invalid array index"; + } + + let mappedIndex = (this.currentTail + index) % this.maxCount; + return this.circle[mappedIndex]; + } +} + +export class ContextTracker extends CircularArray { + static attemptMatchContext(tokenizedContext: USVString[], + matchState: TrackedContextState, + transformDistribution?: Distribution,): TrackedContextState { + // Map the previous tokenized state to an edit-distance friendly version. + let matchContext: USVString[] = matchState.toRawTokenization(); + + // Inverted order, since 'match' existed before our new context. + let mapping = ClassicalDistanceCalculation.computeDistance(matchContext.map(value => ({key: value})), + tokenizedContext.map(value => ({key: value})), + 1); + + let editPath = mapping.editPath(); + + let poppedHead = false; + let pushedTail = false; + + // Matters greatly when starting from a nil context. + if(editPath.length > 1) { + // First entry: may not be an 'insert' or a 'transpose' op. + // 'insert' allowed if the next token is 'substitute', as this may occur with an edit path of length 2. + if((editPath[0] == 'insert' && !(editPath[1] == 'substitute' && editPath.length == 2)) || editPath[0].indexOf('transpose') >= 0) { + return null; + } else if(editPath[0] == 'delete') { + poppedHead = true; // a token from the previous state has been wholly removed. + } + } + + // Last entry: may not be a 'delete' or a 'transpose' op. + let tailIndex = editPath.length -1; + let ignorePenultimateMatch = false; + if(editPath[tailIndex] == 'delete' || editPath[0].indexOf('transpose') >= 0) { + return null; + } else if(editPath[tailIndex] == 'insert') { + pushedTail = true; + } else if(tailIndex > 0 && editPath[tailIndex-1] == 'insert' && editPath[tailIndex] == 'substitute') { + // Tends to happen when accepting suggestions. + pushedTail = true; + ignorePenultimateMatch = true; + } + + // Can happen for the first text input after backspace deletes a wordbreaking character, + // thus the new input continues a previous word while dropping the empty word after + // that prior wordbreaking character. + // + // We can't handle it reliably from this match state, but a previous entry (without the empty token) + // should still be in the cache and will be reliable for this example case. + if(tailIndex > 0 && editPath[tailIndex-1] == 'delete' && editPath[tailIndex] == 'substitute') { + return null; + } + + // Now to check everything in-between: should be exclusively 'match'es. + for(let index = 1; index < editPath.length - (ignorePenultimateMatch ? 2 : 1); index++) { + if(editPath[index] != 'match') { + return null; + } + } + + // If we've made it here... success! We have a context match! + let state: TrackedContextState; + + if(pushedTail) { + // On suggestion acceptance, we should update the previous final token. + // We do it first so that the acceptance is replicated in the new TrackedContextState + // as well. + if(ignorePenultimateMatch) { + // For this case, we were likely called by ModelCompositor.acceptSuggestion(), which + // would have marked the accepted suggestion. + matchState.tail.replacementText = tokenizedContext[tokenizedContext.length-2]; + } + + state = new TrackedContextState(matchState); + } else { + // Since we're continuing a previously-cached context, we can reuse the same SearchSpace + // to continue making predictions. + state = matchState; + } + + const hasDistribution = transformDistribution && Array.isArray(transformDistribution); + let primaryInput = hasDistribution ? transformDistribution[0].sample : null; + if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { + primaryInput = null; + } + + const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); + const finalToken = tokenizedContext[tokenizedContext.length-1]; + + /* Assumption: This is an adequate check for its two sub-branches. + * + * Basis: + * - Assumption: one keystroke may only cause a single token to rotate out of context. + * - That is, no "reasonable" keystroke would emit enough code points to 'bump' two words simultaneously. + * - ... This one may need to be loosened a bit... but it should be enough for initial correction testing as-is. + * - Assumption: one keystroke may only cause a single token to be appended to the context + * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens + * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. + */ + + // If there is/was more than one context token available... + if(editPath.length > 1) { + // We're removing a context token, but at least one remains. + if(poppedHead) { + state.popHead(); + } + + // We're adding an additional context token. + if(pushedTail) { + const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; + /* + * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. + * + * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: + * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! + */ + let pushedToken = new TrackedContextToken(); + pushedToken.raw = tokenizedTail; + + if(isWhitespace || !primaryInput) { + state.pushWhitespaceToTail(transformDistribution ?? []); + // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters + // for the new word (token), so the input keystrokes do not correspond to the new text token. + pushedToken.transformDistributions = []; + } else { + state.pushWhitespaceToTail(); + // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing + // any metadata used to construct the new context state token. + pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + } + + state.pushTail(pushedToken); + } else { // We're editing the final context token. + // TODO: Assumption: we didn't 'miss' any inputs somehow. + // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. + if(isBackspace) { + state.replaceTailForBackspace(finalToken, primaryInput.id); + } else { + state.updateTail(primaryInput ? transformDistribution : null, finalToken); + } + } + // There is only one word in the context. + } else { + // TODO: Assumption: we didn't 'miss' any inputs somehow. + // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. + + if(editPath[tailIndex] == 'insert') { + // Construct appropriate initial token. + let token = new TrackedContextToken(); + token.raw = tokenizedContext[0]; + token.transformDistributions = [transformDistribution]; + state.pushTail(token); + } else { // Edit the lone context token. + // Consider backspace entry for this case? + if(isBackspace) { + state.replaceTailForBackspace(finalToken, primaryInput.id); + } else { + state.updateTail(primaryInput ? transformDistribution : null, finalToken); + } + } + } + return state; + } + + static modelContextState(tokenizedContext: USVString[], + transformDistribution: Distribution, + lexicalModel: LexicalModel): TrackedContextState { + let baseTokens = tokenizedContext.map(function(entry) { + let token = new TrackedContextToken(); + token.raw = entry; + if(token.raw) { + token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) { + return [{sample: transform, p: 1.0}]; + }); + } else { + // Helps model context-final wordbreaks. + token.transformDistributions = []; + } + return token; + }); + + // And now build the final context state object, which includes whitespace 'tokens'. + let state = new TrackedContextState(lexicalModel); + + if(baseTokens.length > 0) { + state.pushTail(baseTokens.splice(0, 1)[0]); + } + + while(baseTokens.length > 0) { + state.pushWhitespaceToTail(); + state.pushTail(baseTokens.splice(0, 1)[0]); + } + + if(state.tokens.length == 0) { + let token = new TrackedContextToken(); + token.raw = ''; + + state.pushTail(token); + } + + return state; + } + + /** + * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning + * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the + * input information provided to previous prediction calls and persisted correction-search calculations for re-use. + * + * @param model + * @param context + * @param transformDistribution + */ + analyzeState(model: LexicalModel, + context: Context, + transformDistribution?: Distribution): TrackedContextState { + if(!model.traverseFromRoot) { + // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) + // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. + throw "This lexical model does not provide adequate data for correction algorithms and context reuse"; + } + + let tokenizedContext = tokenize(model.wordbreaker || defaultWordbreaker, context); + + if(tokenizedContext.left.length > 0) { + for(let i = this.count - 1; i >= 0; i--) { + let resultState = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), transformDistribution); + + if(resultState) { + resultState.taggedContext = context; + if(resultState != this.item(i)) { + this.enqueue(resultState); + } + return resultState; + } + } + } + + // Else: either empty OR we've detected a 'new context'. Initialize from scratch; no prior input information is + // available. Only the results of the prior inputs are known. + // + // Assumption: as a caret needs to move to context before any actual transform distributions occur, + // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. + let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); + state.taggedContext = context; + this.enqueue(state); + return state; + } +} diff --git a/common/web/lm-worker/src/main/correction/distance-modeler.ts b/common/web/lm-worker/src/main/correction/distance-modeler.ts new file mode 100644 index 0000000000000000000000000000000000000000..20d72c546890d2d0077065e21e312df1981929c2 --- /dev/null +++ b/common/web/lm-worker/src/main/correction/distance-modeler.ts @@ -0,0 +1,805 @@ +import { Comparator, isHighSurrogate, SENTINEL_CODE_UNIT, PriorityQueue } from '@keymanapp/models-templates'; + +import { ClassicalDistanceCalculation, EditToken } from './classical-calculation.js'; + +type RealizedInput = ProbabilityMass[]; // NOT Distribution - they're masses from separate distributions. + +export type TraversableToken = { + key: TUnit, + traversal: LexiconTraversal +} + +export const QUEUE_NODE_COMPARATOR: Comparator = function(arg1, arg2) { + return arg1.currentCost - arg2.currentCost; +} + +// Represents a processed node for the correction-search's search-space's tree-like graph. May represent +// internal and 'leaf' nodes on said graph, as well as the overall root of the search. Also used to represent +// edges on the graph TO said nodes - there's a bit of overloading here. Either way, it stores the cost of the +// optimum path used to reach the ndoe. +// +// The stored path cost may be an overestimate when the edit distance is greater than the current search threshold. The +// first version of the node to be dequeued from SearchSpace's priority queue hierarchy 'wins' and is taken as the absolute +// minimum; subsequent versions are ignored as suboptimal. +// +// Provides functions usable to enumerate across the node's outward edges to new nodes for continued search. +// Most of the actual calculations occur as part of this process. +// +// For nodes with raw edit-distance cost within the current threshold for correction searches, we do have admissibility. +// If not enough nodes are available within that threshold, however, admissibility may be lost, leaving our search as a +// heuristic. +// +export class SearchNode { + calculation: ClassicalDistanceCalculation, TraversableToken>; + + currentTraversal: LexiconTraversal; + toKey: (wordform: USVString) => USVString = str => str; + priorInput: RealizedInput; + + // Internal lazy-cache for .inputSamplingCost, as it's a bit expensive to re-compute. + private _inputCost?: number; + + constructor(rootTraversal: LexiconTraversal, toKey?: (USVString) => USVString); + constructor(node: SearchNode); + constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (USVString) => USVString) { + toKey = toKey || (x => x); + + if(rootTraversal instanceof SearchNode) { + let priorNode = rootTraversal; + this.calculation = priorNode.calculation; + this.currentTraversal = priorNode.currentTraversal; + this.priorInput = priorNode.priorInput; + this.toKey = priorNode.toKey; + } else { + this.calculation = new ClassicalDistanceCalculation(); + this.currentTraversal = rootTraversal; + this.priorInput = []; + this.toKey = toKey; + } + } + + get knownCost(): number { + return this.calculation.getHeuristicFinalCost(); + } + + get inputSamplingCost(): number { + if(this._inputCost !== undefined) { + return this._inputCost; + } else { + let MIN_P = SearchSpace.MIN_KEYSTROKE_PROBABILITY; + // Should technically re-normalize the sampling distribution. + // -ln(p) is smaller for larger probabilities, as ln(p) is always <= 0. Approaches infinity as p => 0. + + // TODO: probably more efficient to instead use actual 'probability space'... but that'll involve extra changes. + this._inputCost = this.priorInput.map(mass => mass.p > MIN_P ? mass.p : MIN_P).reduce((previous, current) => previous - Math.log(current), 0); + return this._inputCost; + } + } + + // The part used to prioritize our search. + get currentCost(): number { + // - We reintrepret 'known cost' as a psuedo-probability. + // - Noting that 1/e = 0.367879441, an edit-distance cost of 1 may be intepreted as -ln(1/e) - a log-space 'likelihood'. + // - Not exactly normalized, though. + // That's a really, really high likelihood, thoough. + // + // At any rate, we can linearly scale the known-cost to have about whatever probability we want. + // If we can state it as p = 1 / (c * e), note then that ln(c * e) = ln(c) + 1. So, scale * (ln(c) + 1). + // If we can state it as e^x, note that ln(e^x) = x * ln(e) = x - just scale by 'x'! + + // p = 1 / (e^4) = 0.01831563888. This still exceeds many neighboring keys! + // p = 1 / (e^5) = 0.00673794699. Strikes a good balance. + // Should easily give priority to neighboring keys before edit-distance kicks in (when keys are a bit ambiguous) + return SearchSpace.EDIT_DISTANCE_COST_SCALE * this.knownCost + this.inputSamplingCost; + } + + buildInsertionEdges(): SearchNode[] { + let edges: SearchNode[] = []; + + for(let lexicalChild of this.currentTraversal.children()) { + let traversal = lexicalChild.traversal(); + let matchToken = { + key: lexicalChild.char, + traversal: traversal + } + + let childCalc = this.calculation.addMatchChar(matchToken); + + let searchChild = new SearchNode(this); + searchChild.calculation = childCalc; + searchChild.priorInput = this.priorInput; + searchChild.currentTraversal = traversal; + + edges.push(searchChild); + } + + return edges; + } + + buildDeletionEdges(inputDistribution: Distribution): SearchNode[] { + let edges: SearchNode[] = []; + + /* + * If the probability of an input is less than the highest probability * the base edit-distance likelihood, + * don't build an edge for it; just rely on edits from the highest-probability edge. + * + * We may be able to be stricter, but this should be a decent start. + * + * Note: thanks to ModelCompositor.predict, we know the distribution is pre-sorted. + */ + for(let probMass of inputDistribution) { + if(probMass.p < inputDistribution[0].p * Math.exp(-SearchSpace.EDIT_DISTANCE_COST_SCALE)) { + // Again, we're pre-sorted. All further entries will be too low-cost to consider. + break; + } + + let edgeCalc = this.calculation; + let transform = probMass.sample; + if(transform.deleteLeft) { + edgeCalc = edgeCalc.getSubset(edgeCalc.inputSequence.length - transform.deleteLeft, edgeCalc.matchSequence.length); + } + + // TODO: transform.deleteRight currently not supported. + + let inputPath = Array.from(this.priorInput); + inputPath.push(probMass); + // Tokenize and iterate over input chars, adding them into the calc. + for(let i=0; i < transform.insert.length; i++) { + let char = transform.insert[i]; + if(isHighSurrogate(char)) { + i++; + char = char + transform.insert[i]; + } + + // In case of NFD input, filter out any empty-strings that may arise + // when 'keying' raw diacritics. + let keyedChar = this.toKey(char); + if(keyedChar) { + edgeCalc = edgeCalc.addInputChar({key: keyedChar}); + } + } + + let childEdge = new SearchNode(this); + childEdge.calculation = edgeCalc; + childEdge.priorInput = inputPath; + + edges.push(childEdge); + } + + return edges; + } + + // While this may SEEM to be unnecessary, note that sometimes substitutions (which are computed + // via insert + delete) may be lower cost than both just-insert and just-delete. + buildSubstitutionEdges(inputDistribution: Distribution): SearchNode[] { + // Handles the 'input' component. + let intermediateEdges = this.buildDeletionEdges(inputDistribution); + let edges: SearchNode[] = []; + + for(let lexicalChild of this.currentTraversal.children()) { + for(let edge of intermediateEdges) { + let traversal = lexicalChild.traversal(); + let matchToken = { + key: lexicalChild.char, + traversal: traversal + } + + let childCalc = edge.calculation.addMatchChar(matchToken); + + let searchChild = new SearchNode(this); + searchChild.calculation = childCalc; + searchChild.priorInput = edge.priorInput; + searchChild.currentTraversal = traversal; + + edges.push(searchChild); + } + } + + return edges; + } + + get mapKey(): string { + let inputString = this.priorInput.map((value) => '+' + value.sample.insert + '-' + value.sample.deleteLeft).join(''); + let matchString = this.calculation.matchSequence.map((value) => value.key).join(''); + + // TODO: might should also track diagonalWidth. + return inputString + SENTINEL_CODE_UNIT + matchString; + } + + get isFullReplacement(): boolean { + // If the known edit-distance cost is equal to the input length, this means + // that literally every input has been full-on replaced. Thus, this is + // likely not a good 'root' to use for predictions. + // + // Logic exception: 0 cost, 0 length != a "replacement". + return this.knownCost && this.knownCost == this.priorInput.length; + } +} + +class SearchSpaceTier { + correctionQueue: PriorityQueue; + processed: SearchNode[] = []; + index: number; + + constructor(index: number, initialEdges?: SearchNode[]) { + this.index = index; + this.correctionQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR, initialEdges); + } + + increaseMaxEditDistance() { + // By extracting the entries from the priority queue and increasing distance outside of it as a batch job, + // we get an O(N) implementation, rather than the O(N log N) that would result from maintaining the original queue. + let entries = this.correctionQueue.toArray(); + + entries.forEach(function(edge) { edge.calculation = edge.calculation.increaseMaxDistance(); }); + + // Since we just modified the stored instances, and the costs may have shifted, we need to re-heapify. + this.correctionQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR, entries); + } +} + +export class SearchResult { + private resultNode: SearchNode; + + constructor(node: SearchNode) { + this.resultNode = node; + } + + get inputSequence(): ProbabilityMass[] { + return this.resultNode.priorInput; + } + + get matchSequence(): TraversableToken[] { + return this.resultNode.calculation.matchSequence; + }; + + get matchString(): USVString { + return this.matchSequence.map(value => value.key).join(''); + } + + get knownCost(): number { + return this.resultNode.knownCost; + } + + get inputSamplingCost(): number { + return this.resultNode.inputSamplingCost; + } + + get totalCost(): number { + return this.resultNode.currentCost; + } + + get finalTraversal(): LexiconTraversal { + return this.resultNode.currentTraversal; + } +} + +type NullPath = { + type: 'none' +} + +type IntermediateSearchPath = { + type: 'intermediate', + cost: number +} + +type CompleteSearchPath = { + type: 'complete', + cost: number, + finalNode: SearchNode +} + +type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; + +// The set of search spaces corresponding to the same 'context' for search. +// Whenever a wordbreak boundary is crossed, a new instance should be made. +export class SearchSpace { + private QUEUE_SPACE_COMPARATOR: Comparator; + + // p = 1 / (e^4) = 0.01831563888. This still exceeds many neighboring keys! + // p = 1 / (e^5) = 0.00673794699. Strikes a good balance. + // Should easily give priority to neighboring keys before edit-distance kicks in (when keys are a bit ambiguous) + static readonly EDIT_DISTANCE_COST_SCALE = 5; + static readonly MIN_KEYSTROKE_PROBABILITY = 0.0001; + static readonly DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL = 33; // in milliseconds. + + private tierOrdering: SearchSpaceTier[] = []; + private selectionQueue: PriorityQueue; + private inputSequence: Distribution[] = []; + private minInputCost: number[] = []; + private rootNode: SearchNode; + + // We use an array and not a PriorityQueue b/c batch-heapifying at a single point in time + // is cheaper than iteratively building a priority queue. + private completedPaths: SearchNode[]; + + // Marks all results that have already been returned since the last input was received. + private returnedValues: {[mapKey: string]: SearchNode} = {}; + + // Signals that the edge has already been processed. + private processedEdgeSet: {[mapKey: string]: boolean} = {}; + + constructor(model: LexicalModel) { + if(!model) { + throw "The LexicalModel parameter must not be null / undefined."; + } else if(!model.traverseFromRoot) { + throw "The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching."; + } + + // Constructs the comparator needed for the following line. + this.buildQueueSpaceComparator(); + this.selectionQueue = new PriorityQueue(this.QUEUE_SPACE_COMPARATOR); + this.rootNode = new SearchNode(model.traverseFromRoot(), model.toKey ? model.toKey.bind(model) : null); + + this.completedPaths = [this.rootNode]; + + // Adds a base level queue to handle initial insertions. + // Start with _just_ the root node. Necessary for proper empty-token, empty-input handling! + let baseTier = new SearchSpaceTier(0, [this.rootNode]); + this.tierOrdering.push(baseTier); + this.selectionQueue.enqueue(baseTier); + } + + private buildQueueSpaceComparator() { + let searchSpace = this; + + this.QUEUE_SPACE_COMPARATOR = function(space1, space2) { + let node1 = space1.correctionQueue.peek(); + let node2 = space2.correctionQueue.peek(); + + let index1 = space1.index; + let index2 = space2.index; + + let tierMinCost: number = 0; + let sign = 1; + + if(index2 < index1) { + let temp = index2; + index2 = index1; + index1 = temp; + + sign = -1; + } + + // Boost the cost of the lower tier by the minimum cost possible for the missing inputs between them. + // In essence, compare the nodes as if the lower tier had the most likely input appended for each such + // input missing at the lower tier. + // + // A 100% admissible heuristic to favor a deeper search, since the added cost is guaranteed if the path + // is traversed further. + // + // Remember, tier index i's last used input was from input index i-1. + // As a result, i is the first needed input index, with index2 - 1 the last entry needed to match them. + for(let i=index1; i < index2; i++) { + tierMinCost = tierMinCost + searchSpace.minInputCost[i]; + } + + // Guards, just in case one of the search spaces ever has an empty node. + if(node1 && node2) { + // If node1 is lower-tier, node1 is the one in need of boosted cost. + // `sign` flips it when node2 is lower tier. + return node1.currentCost - node2.currentCost + sign * tierMinCost; + } else if(node2) { + return 1; + } else { + return -1; + } + } + } + + increaseMaxEditDistance() { + this.tierOrdering.forEach(function(tier) { tier.increaseMaxEditDistance() }); + } + + addInput(inputDistribution: Distribution) { + this.inputSequence.push(inputDistribution); + + // Assumes that `inputDistribution` is already sorted. + this.minInputCost.push(-Math.log(inputDistribution[0].p)); + + // With a newly-available input, we can extend new input-dependent paths from + // our previously-reached 'extractedResults' nodes. + let newlyAvailableEdges: SearchNode[] = []; + let batches = this.completedPaths.map(function(node) { + let deletions = node.buildDeletionEdges(inputDistribution); + let substitutions = node.buildSubstitutionEdges(inputDistribution); + + return deletions.concat(substitutions); + }); + + // Don't forget to reset the array; the contained nodes no longer reach the search's end. + this.completedPaths = []; + this.returnedValues = {}; + + batches.forEach(function(batch) { + newlyAvailableEdges = newlyAvailableEdges.concat(batch); + }); + + // Now that we've built the new edges, we can efficiently construct the new search tier. + let tier = new SearchSpaceTier(this.tierOrdering.length, newlyAvailableEdges); + this.tierOrdering.push(tier); + this.selectionQueue.enqueue(tier); + } + + // TODO: will want eventually for reversions and/or backspaces + removeLastInput() { + // 1. truncate all entries from that search tier; we need to 'restore' extractedResults to match + // the state that would have existed without the last search tier. + // 2. remove the last search tier. Which may necessitate reconstructing the tier queue, but oh well. + } + + private hasNextMatchEntry(): boolean { + let topQueue = this.selectionQueue.peek(); + if(topQueue) { + return topQueue.correctionQueue.count > 0; + } else { + return false; + } + } + + private handleNextNode(): PathResult { + if(!this.hasNextMatchEntry()) { + return { type: 'none' }; + } + + let bestTier = this.selectionQueue.dequeue(); + let currentNode = bestTier.correctionQueue.dequeue(); + + let unmatchedResult: IntermediateSearchPath = { + type: 'intermediate', + cost: currentNode.currentCost + } + + // Have we already processed a matching edge? If so, skip it. + // We already know the previous edge is of lower cost. + if(this.processedEdgeSet[currentNode.mapKey]) { + this.selectionQueue.enqueue(bestTier); + return unmatchedResult; + } else { + this.processedEdgeSet[currentNode.mapKey] = true; + } + + // Stage 1: filter out nodes/edges we want to prune + + // Forbid a raw edit-distance of greater than 2. + // Note: .knownCost is not scaled, while its contribution to .currentCost _is_ scaled. + let substitutionsOnly = false; + if(currentNode.knownCost > 2) { + return unmatchedResult; + } else if(currentNode.knownCost == 2) { + // Hard restriction: no further edits will be supported. This helps keep the search + // more narrowly focused. + substitutionsOnly = true; + } + + let tierMinCost = 0; + for(let i = 0; i <= bestTier.index; i++) { + tierMinCost += this.minInputCost[i]; + } + + // Thresholds _any_ path, partially based on currently-traversed distance. + // Allows a little 'wiggle room' + 2 "hard" edits. + // Can be important if needed characters don't actually exist on the keyboard + // ... or even just not the then-current layer of the keyboard. + if(currentNode.currentCost > tierMinCost + 2.5 * SearchSpace.EDIT_DISTANCE_COST_SCALE) { + return unmatchedResult; + } + + // Stage 2: build remaining edges + + // Always possible, as this does not require any new input. + if(!substitutionsOnly) { + let insertionEdges = currentNode.buildInsertionEdges(); + bestTier.correctionQueue.enqueueAll(insertionEdges); + } + + if(bestTier.index == this.tierOrdering.length - 1) { + // It was the final tier - store the node for future reference. + this.completedPaths.push(currentNode); + + // Since we don't modify any other tier, we may simply reinsert the removed tier. + this.selectionQueue.enqueue(bestTier); + + return { + type: 'complete', + cost: currentNode.currentCost, + finalNode: currentNode + }; + } else { + // Time to construct new edges for the next tier! + let nextTier = this.tierOrdering[bestTier.index+1]; + + let inputIndex = nextTier.index; + + let deletionEdges = []; + if(!substitutionsOnly) { + deletionEdges = currentNode.buildDeletionEdges(this.inputSequence[inputIndex-1]); + } + let substitutionEdges = currentNode.buildSubstitutionEdges(this.inputSequence[inputIndex-1]); + + // Note: we're live-modifying the tier's cost here! The priority queue loses its guarantees as a result. + nextTier.correctionQueue.enqueueAll(deletionEdges.concat(substitutionEdges)); + + // So, we simply rebuild the selection queue. + this.selectionQueue = new PriorityQueue(this.QUEUE_SPACE_COMPARATOR, this.tierOrdering); + + // We didn't reach an end-node, so we just end the iteration and continue the search. + } + + // If we've somehow fully exhausted all search options, indicate that none remain. + return unmatchedResult; + } + + // Current best guesstimate of how compositor will retrieve ideal corrections. + *getBestMatches(waitMillis?: number): Generator { + // might should also include a 'base cost' parameter of sorts? + let searchSpace = this; + let currentReturns: {[mapKey: string]: SearchNode} = {}; + + let maxTime: number; + if(waitMillis == 0) { + maxTime = Infinity; + } else if(waitMillis == undefined || Number.isNaN(waitMillis)) { // also covers null. + maxTime = SearchSpace.DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL; + } else { + maxTime = waitMillis; + } + + /** + * This inner class is designed to help the algorithm detect its active execution time. + * While there's no official JS way to do this, we can approximate it by polling the + * current system time (in ms) after each iteration of a short-duration loop. Unusual + * spikes in system time for a single iteration is likely to indicate that an OS + * context switch occurred at some point during the iteration's execution. + */ + class ExecutionTimer { + /** + * The system time when this instance was created. + */ + private start: number; + + /** + * Marks the system time at the start of the currently-running loop, as noted + * by a call to the `startLoop` function. + */ + private loopStart: number; + + private maxExecutionTime: number; + private maxTrueTime: number; + + private executionTime: number; + + /** + * Used to track intervals in which potential context swaps by the OS may + * have occurred. Context switches generally seem to pause threads for + * at least 16 ms, while we expect each loop iteration to complete + * within just 1 ms. So, any possible context switch should have the + * longest observed change in system time. + * + * See `updateOutliers` for more details. + */ + private largestIntervals: number[] = [0]; + + constructor(maxExecutionTime: number, maxTrueTime: number) { + // JS measures time by the number of milliseconds since Jan 1, 1970. + this.loopStart = this.start = Date.now(); + this.maxExecutionTime = maxExecutionTime; + this.maxTrueTime = maxTrueTime; + } + + startLoop() { + this.loopStart = Date.now(); + } + + markIteration() { + const now = Date.now(); + const delta = now - this.loopStart; + this.executionTime += delta; + + /** + * Update the list of the three longest system-time intervals observed + * for execution of a single loop iteration. + * + * Ignore any zero-ms length intervals; they'd make the logic much + * messier than necessary otherwise. + */ + if(delta) { + // If the currently-observed interval is longer than the shortest of the 3 + // previously-observed longest intervals, replace it. + if(this.largestIntervals.length > 2 && delta > this.largestIntervals[0]) { + this.largestIntervals[0] = delta; + } else { + this.largestIntervals.push(delta); + } + // Puts the list in ascending order. Shortest of the list becomes the head, + // longest one the tail. + this.largestIntervals.sort(); + + // Then, determine if we need to update our outlier-based tweaks. + this.updateOutliers(); + } + } + + updateOutliers() { + /* Base assumption: since each loop of the search should evaluate within ~1ms, + * notably longer execution times are probably context switches. + * + * Base assumption: OS context switches generally last at least 16ms. (Based on + * a window.setTimeout() usually not evaluating for at least + * that long, even if set to 1ms.) + * + * To mitigate these assumptions: we'll track the execution time of every loop + * iteration. If the longest observation somehow matches or exceeds the length of + * the next two almost-longest observations twice over... we have a very strong + * 'context switch' candidate. + * + * Or, in near-formal math/stats: we expect a very low variance in execution + * time among the iterations of the search's loops. With a very low variance, + * ANY significant proportional spikes in execution time are outliers - outliers + * likely caused by an OS context switch. + * + * Rather than do intensive math, we use a somewhat lazy approach below that + * achieves the same net results given our assumptions, even when relaxed somewhat. + * + * The logic below relaxes the base assumptions a bit to be safe: + * - [2ms, 2ms, 8ms] will cause 8ms to be seen as an outlier. + * - [2ms, 3ms, 10ms] will cause 10ms to be seen as an outlier. + * + * Ideally: + * - [1ms, 1ms, 4ms] will view 4ms as an outlier. + * + * So we can safely handle slightly longer average intervals and slightly shorter + * OS context-switch time intervals. + */ + if(this.largestIntervals.length > 2) { + // Precondition: the `largestIntervals` array is sorted in ascending order. + // Shortest entry is at the head, longest at the tail. + if(this.largestIntervals[2] >= 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { + this.executionTime -= this.largestIntervals[2]; + this.largestIntervals.pop(); + } + } + } + + shouldTimeout(): boolean { + const now = Date.now(); + if(now - this.start > this.maxTrueTime) { + return true; + } + + return this.executionTime > this.maxExecutionTime; + } + + resetOutlierCheck() { + this.largestIntervals = []; + } + } + + class BatchingAssistant { + currentCost = Number.MIN_SAFE_INTEGER; + entries: SearchResult[] = []; + + checkAndAdd(entry: SearchNode): SearchResult[] | null { + var result: SearchResult[] = null; + + if(entry.currentCost > this.currentCost) { + result = this.tryFinalize(); + + this.currentCost = entry.currentCost; + } + + // Filter out any duplicated match sequences. The same match sequence may be reached via + // different input sequences, after all. + let outputMapKey = entry.calculation.matchSequence.map(value => value.key).join(''); + + // First, ensure the edge has an existing 'shared' cache entry. + if(!searchSpace.returnedValues[outputMapKey]) { + searchSpace.returnedValues[outputMapKey] = entry; + } + + // Check the generator's local returned-value cache - this determines whether or not we + // need to add a new 'return' to the batch. + if(!currentReturns[outputMapKey]) { + this.entries.push(new SearchResult(entry)); + currentReturns[outputMapKey] = entry; + } + + return result; + } + + tryFinalize(): SearchResult[] | null { + var result: SearchResult[] = null; + if(this.entries.length > 0) { + result = this.entries; + this.entries = []; + } + + return result; + } + } + + let batcher = new BatchingAssistant(); + + const timer = new ExecutionTimer(maxTime*1.5, maxTime); + + // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. + let returnedValues = Object.values(this.returnedValues); + if(returnedValues.length > 0) { + let preprocessedQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR, returnedValues); + + // Build batches of same-cost entries. + timer.startLoop(); + while(preprocessedQueue.count > 0) { + let entry = preprocessedQueue.dequeue(); + + // Is the entry a reasonable result? + if(entry.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. + continue; + } + + let batch = batcher.checkAndAdd(entry); + timer.markIteration(); + + if(batch) { + // Do not track yielded time. + yield batch; + } + } + + // As we only return a batch once all entries of the same cost have been processed, we can safely + // finalize the last preprocessed group without issue. + let batch = batcher.tryFinalize(); + if(batch) { + // Do not track yielded time. + yield batch; + } + } + + // Stage 2: the fun part; actually searching! + timer.resetOutlierCheck(); + timer.startLoop(); + let timedOut = false; + do { + let newResult: PathResult; + + // Search for a 'complete' path, skipping all partial paths as long as time remains. + do { + newResult = this.handleNextNode(); + timer.markIteration(); + + if(timer.shouldTimeout()) { + timedOut = true; + } + } while(!timedOut && newResult.type == 'intermediate') + + // TODO: check 'cost' on intermediate, running it through batcher to early-detect cost changes. + let batch: SearchResult[]; + if(newResult.type == 'none') { + break; + } else if(newResult.type == 'complete') { + // Is the entry a reasonable result? + if(newResult.finalNode.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + break; + } + batch = batcher.checkAndAdd(newResult.finalNode); + } + + if(batch) { + yield batch; + } + } while(!timedOut && this.hasNextMatchEntry()); + + // If we _somehow_ exhaust all search options, make sure to return the final results. + let batch = batcher.tryFinalize(); + if(batch) { + yield batch; + } + + return null; + } +} diff --git a/common/web/lm-worker/src/main/correction/index.ts b/common/web/lm-worker/src/main/correction/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..890c49ae3305f4c103b3649f45d269e1f1f5615b --- /dev/null +++ b/common/web/lm-worker/src/main/correction/index.ts @@ -0,0 +1,3 @@ +export * from './classical-calculation.js'; +export * from './context-tracker.js'; +export * from './distance-modeler.js'; \ No newline at end of file diff --git a/common/web/lm-worker/src/index.ts b/common/web/lm-worker/src/main/index.ts similarity index 90% rename from common/web/lm-worker/src/index.ts rename to common/web/lm-worker/src/main/index.ts index 6494c3f5d202eaf332d1cf836bee48dadf540599..a44d4e68bbe5a62e6e43465b23905a9953b3aaba 100644 --- a/common/web/lm-worker/src/index.ts +++ b/common/web/lm-worker/src/main/index.ts @@ -30,9 +30,16 @@ */ /// -/// -/// -/// +import { extendString } from "@keymanapp/web-utils"; + +extendString(); + +import * as models from './models/index.js'; +import * as correction from './correction/index.js'; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; + +import ModelCompositor from './model-compositor.js'; +import { ImportScripts, IncomingMessage, LMLayerWorkerState, LoadMessage, ModelEval, ModelFile, ModelSourceSpec, PostMessage } from './worker-interfaces.js'; /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -57,7 +64,7 @@ * The model and the configuration are ONLY relevant in the `ready` state; * as such, they are NOT direct properties of the LMLayerWorker. */ -class LMLayerWorker { +export default class LMLayerWorker { /** * State pattern. This object handles onMessage(). * handleMessage() can transition to a different state, if @@ -288,16 +295,16 @@ class LMLayerWorker { if(payload.source.type == 'file') { _this.loadModelFile(payload.source.file); } else { - // Creates a closure capturing all top-level names that the model must be able to reference. - // `eval` runs by scope rules; our virtualized worker needs a special scope for this to work. - // - // Reference: https://stackoverflow.com/a/40108685 - // Note that we don't need `this`, but we do need the namespaces seen below. let code = payload.source.code; - let evalInContext = function(LMLayerWorker, models, correction, wordBreakers) { - eval(code); - } - evalInContext(_this, models, correction, wordBreakers); + + // Limits the scope accessible by the code we're about to evaluate; the code may only access + // global scope and the arguments specified in the constructor below. + // + // This is far more encapsulated and likely more secure... and the former point means this is + // easier to bundle and more optimizable when bundling than direct eval. + // Reference: https://esbuild.github.io/link/direct-eval + const modelLoader = new Function('LMLayerWorker', 'models', 'correction', 'wordBreakers', code); + modelLoader(_this, models, correction, wordBreakers); } } }; @@ -401,21 +408,4 @@ class LMLayerWorker { return worker; } -} - -// Let LMLayerWorker be available both in the browser and in Node. -if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { - module.exports = LMLayerWorker; - module.exports['correction'] = correction; - module.exports['models'] = models; - module.exports['wordBreakers'] = wordBreakers; - /// XXX: export the ModelCompositor for testing. - module.exports['ModelCompositor'] = ModelCompositor; - module.exports['TransformUtils'] = TransformUtils; -} else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { - // Automatically install if we're in a Web Worker. - LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. -} else { - //@ts-ignore - window.LMLayerWorker = LMLayerWorker; -} +} \ No newline at end of file diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/main/model-compositor.ts similarity index 99% rename from common/web/lm-worker/src/model-compositor.ts rename to common/web/lm-worker/src/main/model-compositor.ts index 2bb4fdbafaa187c84775ea1e11d5da125100e016..561d57a7189e816f4717f9ffcd486b8e6f4dafd8 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/main/model-compositor.ts @@ -1,6 +1,10 @@ -/// +import * as models from '@keymanapp/models-templates'; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import * as correction from './correction/index.js' -class ModelCompositor { +import TransformUtils from './transformUtils.js'; + +export default class ModelCompositor { private lexicalModel: LexicalModel; private contextTracker?: correction.ContextTracker; private static readonly MAX_SUGGESTIONS = 12; diff --git a/common/web/lm-worker/src/main/models/dummy-model.ts b/common/web/lm-worker/src/main/models/dummy-model.ts new file mode 100644 index 0000000000000000000000000000000000000000..5312121fde976f0f684dd3ce16843d8e456b7b00 --- /dev/null +++ b/common/web/lm-worker/src/main/models/dummy-model.ts @@ -0,0 +1,86 @@ +/// + +/* + * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) + * Copyright (c) 2018 SIL International + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + +/** + * @file dummy-model.ts + * + * Defines the Dummy model, which is used for testing the + * prediction API exclusively. + */ + +/** + * The Dummy Model that returns nonsensical, but predictable results. + */ +export default class DummyModel implements LexicalModel { + configuration: Configuration; + punctuation?: LexicalModelPunctuation; + private _futureSuggestions: Suggestion[][]; + + constructor(options?: any) { + options = options || {}; + // Create a shallow copy of the suggestions; + // this class mutates the array. + this._futureSuggestions = options.futureSuggestions + ? options.futureSuggestions.slice() : []; + + if (options.punctuation) { + this.punctuation = options.punctuation; + } + } + + configure(capabilities: Capabilities): Configuration { + this.configuration = { + leftContextCodePoints: capabilities.maxLeftContextCodePoints, + rightContextCodePoints: capabilities.maxRightContextCodePoints + }; + + return this.configuration; + } + + predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Distribution { + let makeUniformDistribution = function(suggestions: Suggestion[]): Distribution { + let distribution: Distribution = []; + let n = suggestions.length; + + for(let s of suggestions) { + distribution.push({sample: s, p: 1}); // For a dummy model, this is sufficient. The uniformness is all that matters. + } + + return distribution; + } + + if (injectedSuggestions) { + return makeUniformDistribution(injectedSuggestions); + } + + let currentSet = this._futureSuggestions.shift(); + + if(!currentSet) { + return []; + } else { + return makeUniformDistribution(currentSet); + } + } +}; \ No newline at end of file diff --git a/common/web/lm-worker/src/main/models/index.ts b/common/web/lm-worker/src/main/models/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..14f77edc493bb0d416779f851aa425c6c9021e5f --- /dev/null +++ b/common/web/lm-worker/src/main/models/index.ts @@ -0,0 +1,2 @@ +export * from '@keymanapp/models-templates'; +export { default as DummyModel } from './dummy-model.js'; diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/main/transformUtils.ts similarity index 94% rename from common/web/lm-worker/src/transformUtils.ts rename to common/web/lm-worker/src/main/transformUtils.ts index cbb2c151fe7557c1a2688d398841de611ccf211b..f9d8d8a63f18324139dd3299b3ea23dc3a3d70f9 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/main/transformUtils.ts @@ -1,4 +1,4 @@ -class TransformUtils { +export default class TransformUtils { static isWhitespace(transform: Transform): boolean { // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; diff --git a/common/web/lm-worker/src/worker-interfaces.ts b/common/web/lm-worker/src/main/worker-interfaces.ts similarity index 82% rename from common/web/lm-worker/src/worker-interfaces.ts rename to common/web/lm-worker/src/main/worker-interfaces.ts index de8f20a1a23db712ad0f4c6f2d85085a4163247c..bcde9b56fd16f67f0d6473dfead7c2e5905ccebe 100644 --- a/common/web/lm-worker/src/worker-interfaces.ts +++ b/common/web/lm-worker/src/main/worker-interfaces.ts @@ -28,24 +28,26 @@ /// +import type ModelCompositor from './model-compositor.js'; + /** * The signature of self.postMessage(), so that unit tests can mock it. */ -type PostMessage = typeof DedicatedWorkerGlobalScope.prototype.postMessage; -type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; +export type PostMessage = typeof DedicatedWorkerGlobalScope.prototype.postMessage; +export type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; /** * The valid incoming message kinds. */ -type IncomingMessageKind = 'config' | 'load' | 'predict' | 'unload' | 'wordbreak' | 'accept' | 'revert' | 'reset-context'; -type IncomingMessage = ConfigMessage | LoadMessage | PredictMessage | UnloadMessage | WordbreakMessage | AcceptMessage | RevertMessage | ResetContextMessage; +export type IncomingMessageKind = 'config' | 'load' | 'predict' | 'unload' | 'wordbreak' | 'accept' | 'revert' | 'reset-context'; +export type IncomingMessage = ConfigMessage | LoadMessage | PredictMessage | UnloadMessage | WordbreakMessage | AcceptMessage | RevertMessage | ResetContextMessage; /** * The structure of a config message. It should include the platform's supported * capabilities. */ -interface ConfigMessage { +export interface ConfigMessage { message: 'config'; /** @@ -61,7 +63,7 @@ interface ConfigMessage { testMode?: boolean } -interface ModelFile { +export interface ModelFile { type: 'file'; /** @@ -70,7 +72,7 @@ interface ModelFile { file: string; } -interface ModelEval { +export interface ModelEval { type: 'raw'; /** @@ -80,13 +82,13 @@ interface ModelEval { code: string; } -type ModelSourceSpec = ModelFile | ModelEval; +export type ModelSourceSpec = ModelFile | ModelEval; /** * The structure of an initialization message. It should include the model (either in * source code or parameter form), as well as the keyboard's capabilities. */ -interface LoadMessage { +export interface LoadMessage { message: 'load'; /** @@ -98,7 +100,7 @@ interface LoadMessage { /** * Message to suggestion text. */ -interface PredictMessage { +export interface PredictMessage { message: 'predict'; /** @@ -124,14 +126,14 @@ interface PredictMessage { context: Context; } -interface UnloadMessage { +export interface UnloadMessage { message: 'unload' } /** * Message used to request the last pre-cursor word in the context. */ -interface WordbreakMessage { +export interface WordbreakMessage { message: 'wordbreak'; /** @@ -146,7 +148,7 @@ interface WordbreakMessage { context: Context; } -interface AcceptMessage { +export interface AcceptMessage { message: 'accept'; /** @@ -177,7 +179,7 @@ interface AcceptMessage { postTransform?: Transform; } -interface RevertMessage { +export interface RevertMessage { message: 'revert'; /** @@ -198,7 +200,7 @@ interface RevertMessage { context: Context; } -interface ResetContextMessage { +export interface ResetContextMessage { message: 'reset-context'; /** @@ -210,12 +212,12 @@ interface ResetContextMessage { /** * The LMLayer can be in one of the following states. The LMLayer can only produce predictions in the 'ready' state. */ -type LMLayerWorkerState = LMLayerWorkerUnconfiguredState | LMLayerWorkerModellessState | LMLayerWorkerReadyState; + export type LMLayerWorkerState = LMLayerWorkerUnconfiguredState | LMLayerWorkerModellessState | LMLayerWorkerReadyState; /** * Represents the unconfigured state of the LMLayer. */ -interface LMLayerWorkerUnconfiguredState { +export interface LMLayerWorkerUnconfiguredState { name: 'unconfigured'; handleMessage(payload: IncomingMessage): void; } @@ -223,7 +225,7 @@ interface LMLayerWorkerUnconfiguredState { /** * Represents the pre-model-load state of the LMLayer. */ -interface LMLayerWorkerModellessState { +export interface LMLayerWorkerModellessState { name: 'modelless'; handleMessage(payload: IncomingMessage): void; } @@ -231,7 +233,7 @@ interface LMLayerWorkerModellessState { /** * Represents the 'ready' state of the LMLayer. */ -interface LMLayerWorkerReadyState { +export interface LMLayerWorkerReadyState { name: 'ready'; handleMessage(payload: IncomingMessage): void; compositor: ModelCompositor; @@ -240,7 +242,7 @@ interface LMLayerWorkerReadyState { /** * Constructors that return worker internal models. */ -interface WorkerInternalModelConstructor { +export interface WorkerInternalModelConstructor { /** * LexicalModel instances are all given the keyboard's * capabilities, plus any parameters they require. diff --git a/common/predictive-text/embedded_worker.d.ts b/common/web/lm-worker/src/main/worker-main.ts similarity index 67% rename from common/predictive-text/embedded_worker.d.ts rename to common/web/lm-worker/src/main/worker-main.ts index 243339f134bdb736fb4420e379c185d49c8d4fac..665943f66fb35953c0717f325cf2f513be65ff3f 100644 --- a/common/predictive-text/embedded_worker.d.ts +++ b/common/web/lm-worker/src/main/worker-main.ts @@ -20,17 +20,13 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -// Include the code intended to run WITHIN the Web Worker. -// The worker code MUST be compiled before this file is compiled. -// If you see a 'File: embedded_worker.js not found.' error message, please -// compile the worker first (stage one). -/*/// */ +import LMLayerWorker from './index.js'; -/** - * The body of this function contains code that should run - * inside a Web Worker context. - * - * Convert this code into a blob URI with LMLayer.asBlobURI(), - * and pass the URI into the new Worker() constructor. - */ - declare function LMLayerWorkerCode(): void; +// Let LMLayerWorker be available both in the browser and in Node. +if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { + // Automatically install if we're in a Web Worker. + LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. +} else { + //@ts-ignore + window.LMLayerWorker = LMLayerWorker; +} diff --git a/common/web/lm-worker/src/models/dummy-model.ts b/common/web/lm-worker/src/models/dummy-model.ts deleted file mode 100644 index e077aff5343a37d1d9292b2ebd40ec1faf706d74..0000000000000000000000000000000000000000 --- a/common/web/lm-worker/src/models/dummy-model.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) - * Copyright (c) 2018 SIL International - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -namespace models { - /** - * @file dummy-model.ts - * - * Defines the Dummy model, which is used for testing the - * prediction API exclusively. - */ - - /** - * The Dummy Model that returns nonsensical, but predictable results. - */ - export class DummyModel implements LexicalModel { - configuration: Configuration; - punctuation?: LexicalModelPunctuation; - private _futureSuggestions: Suggestion[][]; - - constructor(options?: any) { - options = options || {}; - // Create a shallow copy of the suggestions; - // this class mutates the array. - this._futureSuggestions = options.futureSuggestions - ? options.futureSuggestions.slice() : []; - - if (options.punctuation) { - this.punctuation = options.punctuation; - } - } - - configure(capabilities: Capabilities): Configuration { - this.configuration = { - leftContextCodePoints: capabilities.maxLeftContextCodePoints, - rightContextCodePoints: capabilities.maxRightContextCodePoints - }; - - return this.configuration; - } - - predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Distribution { - let makeUniformDistribution = function(suggestions: Suggestion[]): Distribution { - let distribution: Distribution = []; - let n = suggestions.length; - - for(let s of suggestions) { - distribution.push({sample: s, p: 1}); // For a dummy model, this is sufficient. The uniformness is all that matters. - } - - return distribution; - } - - if (injectedSuggestions) { - return makeUniformDistribution(injectedSuggestions); - } - - let currentSet = this._futureSuggestions.shift(); - - if(!currentSet) { - return []; - } else { - return makeUniformDistribution(currentSet); - } - } - }; -} \ No newline at end of file diff --git a/common/predictive-text/unit_tests/headless/edit-distance/classical-calculation.js b/common/web/lm-worker/src/test/cases/edit-distance/classical-calculation.js similarity index 98% rename from common/predictive-text/unit_tests/headless/edit-distance/classical-calculation.js rename to common/web/lm-worker/src/test/cases/edit-distance/classical-calculation.js index 12159d06b0e6d837c8886cb647530bf4c91baf59..1833e9f4d477dad07950a0a7aa7b05e8de2f9f6b 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/classical-calculation.js +++ b/common/web/lm-worker/src/test/cases/edit-distance/classical-calculation.js @@ -1,6 +1,5 @@ -var assert = require('chai').assert; -const LMLayerWorker = require('../../../../web/lm-worker/build/intermediate.js'); -var ClassicalDistanceCalculation = LMLayerWorker.correction.ClassicalDistanceCalculation; +import { assert } from 'chai'; +import { ClassicalDistanceCalculation } from '#./correction/classical-calculation.js'; function prettyPrintMatrix(matrix) { for(let r = 0; r < matrix.length; r++) { @@ -261,7 +260,7 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() { // Two transpositions: da <- abcd, wxyz -> zw and one deletion ('g') it("'daefghiwxyz' -> 'abcdefhizw' (width 2) = 7", function() { - buffer = compute("daefghiwxyz", "abcdefhizw", "InputThenMatch", 2); + let buffer = compute("daefghiwxyz", "abcdefhizw", "InputThenMatch", 2); assert.equal(buffer.getHeuristicFinalCost(), 7); }); }); diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/web/lm-worker/src/test/cases/edit-distance/context-tracker.js similarity index 95% rename from common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js rename to common/web/lm-worker/src/test/cases/edit-distance/context-tracker.js index fa09ca20decd8ec6d3e54dc85159e3b22b258d72..f8b715b43f9d048220b76d30c32bfa631db0f0ed 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/web/lm-worker/src/test/cases/edit-distance/context-tracker.js @@ -1,8 +1,10 @@ -var assert = require('chai').assert; -const LMLayerWorker = require('../../../../web/lm-worker/build/intermediate.js'); -var ContextTracker = LMLayerWorker.correction.ContextTracker; -var ModelCompositor = LMLayerWorker.ModelCompositor; -var models = LMLayerWorker.models; +import { assert } from 'chai'; + +import { ContextTracker } from '#./correction/context-tracker.js'; +import ModelCompositor from '#./model-compositor.js'; +import * as models from '#./models/index.js'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; describe('ContextTracker', function() { function toWrapperDistribution(transform) { @@ -158,7 +160,7 @@ describe('ContextTracker', function() { punctuation: englishPunctuation }; - let model = new models.TrieModel(jsonFixture('tries/english-1000'), options); + let model = new models.TrieModel(jsonFixture('models/tries/english-1000'), options); let compositor = new ModelCompositor(model); let baseContextState = compositor.contextTracker.analyzeState(model, baseContext); diff --git a/common/predictive-text/unit_tests/headless/edit-distance/distance-modeler.js b/common/web/lm-worker/src/test/cases/edit-distance/distance-modeler.js similarity index 97% rename from common/predictive-text/unit_tests/headless/edit-distance/distance-modeler.js rename to common/web/lm-worker/src/test/cases/edit-distance/distance-modeler.js index b893df826cf63bd82a6b1f8a0d44cd5025d95dfc..c2ed18bbc676321cda00e1c6802df5ebb660f19b 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/distance-modeler.js +++ b/common/web/lm-worker/src/test/cases/edit-distance/distance-modeler.js @@ -1,7 +1,8 @@ -var assert = require('chai').assert; -const LMLayerWorker = require('../../../../web/lm-worker/build/intermediate.js'); -let models = LMLayerWorker.models; -let correction = LMLayerWorker.correction; +import { assert } from 'chai'; +import * as models from '#./models/index.js'; +import * as correction from '#./correction/index.js'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; function assertEdgeChars(edge, input, match) { assert.isTrue(edgeHasChars(edge, input, match)); @@ -29,7 +30,7 @@ describe('Correction Distance Modeler', function() { var testModel; before(function() { - testModel = new models.TrieModel(jsonFixture('tries/english-1000')); + testModel = new models.TrieModel(jsonFixture('models/tries/english-1000')); }); it('SearchNode.buildInsertionEdges() - from root', function() { @@ -244,7 +245,7 @@ describe('Correction Distance Modeler', function() { var testModel; before(function() { - testModel = new models.TrieModel(jsonFixture('tries/english-1000')); + testModel = new models.TrieModel(jsonFixture('models/tries/english-1000')); }); let checkResults_teh = function(iter) { diff --git a/common/predictive-text/unit_tests/headless/transform-utils.js b/common/web/lm-worker/src/test/cases/transform-utils.js similarity index 86% rename from common/predictive-text/unit_tests/headless/transform-utils.js rename to common/web/lm-worker/src/test/cases/transform-utils.js index b00a24a1230ebcc66da4f98f1ed63123158cbd1d..d4da4501f43b5c2bae913a9817fd1a77e94955b2 100644 --- a/common/predictive-text/unit_tests/headless/transform-utils.js +++ b/common/web/lm-worker/src/test/cases/transform-utils.js @@ -1,6 +1,6 @@ -var assert = require('chai').assert; +import { assert } from 'chai'; -let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils; +import TransformUtils from '#./transformUtils.js'; describe('TransformUtils', function () { describe('isWhitespace', function () { @@ -20,7 +20,7 @@ describe('TransformUtils', function () { }); it("should match a simple ' ' transform", function() { - transform = { + let transform = { insert: " ", deleteLeft: 0 }; @@ -29,7 +29,7 @@ describe('TransformUtils', function () { }); it("should match a simple ' ' transform with delete-left", function() { - transform = { + let transform = { insert: " ", deleteLeft: 1 }; @@ -38,7 +38,7 @@ describe('TransformUtils', function () { }); it("should match a transform consisting of multiple characters of only whitespace", function() { - transform = { + let transform = { insert: " \n\r\u00a0\t\u2000 ", deleteLeft: 0 }; @@ -47,7 +47,7 @@ describe('TransformUtils', function () { }); it("stress tests", function() { - transform = { + let transform = { insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure. deleteLeft: 0 }; diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/web/lm-worker/src/test/cases/worker-custom-punctuation.js similarity index 92% rename from common/predictive-text/unit_tests/headless/worker-custom-punctuation.js rename to common/web/lm-worker/src/test/cases/worker-custom-punctuation.js index 2e6e477a77ccbd9feb289db88b7d48034e82b34b..e0ff73d1c9e399509294e4b37135b7fef599e49c 100644 --- a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js +++ b/common/web/lm-worker/src/test/cases/worker-custom-punctuation.js @@ -2,10 +2,10 @@ * Unit tests for the Dummy prediction model. */ -var assert = require('chai').assert; -const LMLayerWorker = require('../../../web/lm-worker/build/intermediate.js'); -var DummyModel = LMLayerWorker.models.DummyModel; -var ModelCompositor = LMLayerWorker.ModelCompositor; +import { assert } from 'chai'; + +import DummyModel from '#./models/dummy-model.js'; +import ModelCompositor from '#./model-compositor.js'; describe('Custom Punctuation', function () { it('appears in the keep suggestion', function () { diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/web/lm-worker/src/test/cases/worker-initialization.js similarity index 89% rename from common/predictive-text/unit_tests/headless/worker-initialization.js rename to common/web/lm-worker/src/test/cases/worker-initialization.js index 49316ae273d501d9db227ef96b8157bb2f46ed8d..3c15543bcebc76f7b3e8fdb41954584f4cf10b99 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/web/lm-worker/src/test/cases/worker-initialization.js @@ -1,12 +1,17 @@ -var assert = require('chai').assert; -var sinon = require('sinon'); -var fs = require('fs'); +import { assert } from 'chai'; +import sinon from 'sinon'; +import fs from 'fs'; -let LMLayerWorker = require('../../../web/lm-worker/build/intermediate.js'); +import LMLayerWorker from '#./index.js'; + +import { configWorker, createMessageEventWithData, importScriptsWith } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); // Unit tests for instantiating and initializing the LMLayer Worker in isolation. // -// Although the LMLayerWorker expected to be used inside a DedicatedWorkerGlobalScope, +// Although the LMLayerWorker expects to be used inside a DedicatedWorkerGlobalScope, // these unit tests DO NOT run inside a Worker, and instead use Sinon fakes to assert // behavior. describe('LMLayerWorker', function() { @@ -149,7 +154,7 @@ describe('LMLayerWorker', function() { message: 'load', source: { type: 'file', - file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + file: require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js") } })); @@ -168,7 +173,7 @@ describe('LMLayerWorker', function() { var worker = LMLayerWorker.install(context); configWorker(worker); - let modelCode = fs.readFileSync("./unit_tests/in_browser/resources/models/simple-dummy.js").toString(); + let modelCode = fs.readFileSync(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")).toString(); worker.onMessage(createMessageEventWithData({ message: 'load', @@ -199,7 +204,7 @@ describe('LMLayerWorker', function() { message: 'load', source: { type: 'file', - file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + file: require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js") } })); diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/web/lm-worker/src/test/cases/worker-model-compositor.js similarity index 97% rename from common/predictive-text/unit_tests/headless/worker-model-compositor.js rename to common/web/lm-worker/src/test/cases/worker-model-compositor.js index 8080b48f47369770ff231eb7f3398a0652ca7d0d..c642dfc94cd9083aafb330e228ea519a1b2d87e1 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/web/lm-worker/src/test/cases/worker-model-compositor.js @@ -2,15 +2,20 @@ * Integration tests for the model compositor with the trie model. */ -const { models, wordBreakers, ModelCompositor } = require('../../../web/lm-worker/build/intermediate.js'); +import ModelCompositor from '#./model-compositor.js'; +import * as models from '#./models/index.js'; +import * as wordBreakers from '@keymanapp/models-wordbreakers/lib'; + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -var assert = require('chai').assert; var TrieModel = models.TrieModel; describe('ModelCompositor', function() { describe('Prediction with 14.0+ models', function() { describe('Basic suggestion generation', function() { - var plainModel = new TrieModel(jsonFixture('tries/english-1000'), + var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: wordBreakers.default} ); @@ -169,7 +174,7 @@ describe('ModelCompositor', function() { }; var plainCasedModel = new TrieModel( - jsonFixture('tries/english-1000'), { + jsonFixture('models/tries/english-1000'), { languageUsesCasing: true, applyCasing: plainApplyCasing, wordBreaker: wordBreakers.default, @@ -304,7 +309,7 @@ describe('ModelCompositor', function() { describe('Model uses default-style keying, no casing', function () { var uncasedModel = new TrieModel( - jsonFixture('tries/english-1000'), { + jsonFixture('models/tries/english-1000'), { languageUsesCasing: false, wordBreaker: wordBreakers.default, searchTermToKey: function(text) { @@ -379,7 +384,7 @@ describe('ModelCompositor', function() { }; var casedModel = new TrieModel( - jsonFixture('tries/english-1000'), { + jsonFixture('models/tries/english-1000'), { languageUsesCasing: true, applyCasing: applyCasing, wordBreaker: wordBreakers.default, @@ -505,7 +510,7 @@ describe('ModelCompositor', function() { describe('Prediction with legacy Models (12.0 / 13.0)', function() { it('should compose suggestions from a fat-fingered keypress (no keying needed)', function () { var model = new TrieModel( - jsonFixture('tries/english-1000') + jsonFixture('models/tries/english-1000') ); var composite = new ModelCompositor(model); @@ -535,7 +540,7 @@ describe('ModelCompositor', function() { it('should compose suggestions from a fat-fingered keypress (keying needed)', function () { var model = new TrieModel( - jsonFixture('tries/english-1000') + jsonFixture('models/tries/english-1000') ); var composite = new ModelCompositor(model); @@ -903,7 +908,7 @@ describe('ModelCompositor', function() { id: 13 } - let model = new models.TrieModel(jsonFixture('tries/english-1000'), {punctuation: englishPunctuation}); + let model = new models.TrieModel(jsonFixture('models/tries/english-1000'), {punctuation: englishPunctuation}); let compositor = new ModelCompositor(model); let initialSuggestions = compositor.predict(postTransform, baseContext); @@ -938,7 +943,7 @@ describe('ModelCompositor', function() { id: 13 } - let model = new models.TrieModel(jsonFixture('tries/english-1000'), {punctuation: englishPunctuation}); + let model = new models.TrieModel(jsonFixture('models/tries/english-1000'), {punctuation: englishPunctuation}); let compositor = new ModelCompositor(model); let initialSuggestions = compositor.predict(postTransform, baseContext); diff --git a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js b/common/web/lm-worker/src/test/cases/worker-predict-dummy.js similarity index 93% rename from common/predictive-text/unit_tests/headless/worker-predict-dummy.js rename to common/web/lm-worker/src/test/cases/worker-predict-dummy.js index 7275eeef40d7d433246d715b0dda95ad7be0efed..f552f6e29d7e0671f8cd120a01da3bc02e850ea9 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js +++ b/common/web/lm-worker/src/test/cases/worker-predict-dummy.js @@ -2,9 +2,10 @@ * Unit tests for the Dummy prediction model. */ -var assert = require('chai').assert; -const LMLayerWorker = require('../../../web/lm-worker/build/intermediate.js'); -var DummyModel = LMLayerWorker.models.DummyModel; +import { assert } from 'chai'; +import DummyModel from '#./models/dummy-model.js'; + +import { emptyContext, iGotDistractedByHazel, zeroTransform } from '@keymanapp/common-test-resources/model-helpers.mjs'; describe('LMLayerWorker dummy model', function() { describe('instantiation', function () { diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/web/lm-worker/src/test/cases/worker-predict.js similarity index 79% rename from common/predictive-text/unit_tests/headless/worker-predict.js rename to common/web/lm-worker/src/test/cases/worker-predict.js index 2a2fd40e15ac20a3bcb42a4b6976ad446a72e0d3..f050998e878d627a40f90ec1b509df5efac0f519 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/web/lm-worker/src/test/cases/worker-predict.js @@ -1,8 +1,13 @@ -var assert = require('chai').assert; -var sinon = require('sinon'); +import { assert } from 'chai'; +import sinon from 'sinon'; -const LMLayerWorker = require('../../../web/lm-worker/build/intermediate.js'); +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); +import LMLayerWorker from '#./index.js'; + +import { configWorker, createMessageEventWithData, emptyContext, iGotDistractedByHazel, + importScriptsWith, randomToken, zeroTransform } from '@keymanapp/common-test-resources/model-helpers.mjs'; describe('LMLayerWorker', function () { describe('#predict()', function () { @@ -41,7 +46,7 @@ describe('LMLayerWorker', function () { message: 'load', source: { type: 'file', - file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + file: require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js") } })); sinon.assert.calledWithMatch(fakePostMessage.lastCall, { diff --git a/common/web/lm-worker/tsconfig.json b/common/web/lm-worker/tsconfig.json index 43c77edf0c3e3000805824717b9b1452445e0eae..2fdda0135a121b963f752307aa01074a355a467a 100644 --- a/common/web/lm-worker/tsconfig.json +++ b/common/web/lm-worker/tsconfig.json @@ -4,24 +4,32 @@ "compilerOptions": { "allowJs": false, "declaration": true, - "module": "none", - "outFile": "./build/index.js", - "sourceMap": true, + "module": "es6", + "moduleResolution": "node16", + "importHelpers": true, + "inlineSourceMap": true, + "inlineSources": true, + "sourceRoot": "/common/web/lm-worker/src", "lib": ["webworker", "es6"], "target": "es5", + "types": ["node"], "downlevelIteration": true, + "baseUrl": "./", + "outDir": "build/obj", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src/main" }, "references": [ // types { "path": "../../models/types" }, { "path": "../lm-message-types" }, // modules - { "path": "../keyman-version", "prepend": true }, - { "path": "../utils", "prepend": true }, - { "path": "../../models/templates", "prepend": true }, - { "path": "../../models/wordbreakers", "prepend": true }, + { "path": "../keyman-version" }, + { "path": "../utils" }, + { "path": "../../models/templates" }, + { "path": "../../models/wordbreakers" }, ], "include": [ - "src/**/*.ts" + "src/main/**/*.ts" ] } diff --git a/common/web/recorder/build.sh b/common/web/recorder/build.sh index b9e578e05476a529c290f1d5bb43080b0357d35f..73951544b05d3db09bd6d5053c6f518253963eee 100755 --- a/common/web/recorder/build.sh +++ b/common/web/recorder/build.sh @@ -22,16 +22,11 @@ builder_describe \ "@/common/web/keyman-version" \ configure \ clean \ - build \ - ":module Builds recorder-core module" \ - ":proctor Builds headless-testing, node-oriented 'proctor' component" + build builder_describe_outputs \ - configure "/node_modules" \ - configure:module "/node_modules" \ - configure:proctor "/node_modules" \ - build:module "/common/web/recorder/build/index.js" \ - build:proctor "/common/web/recorder/build/nodeProctor/index.js" + configure "/node_modules" \ + build "/common/web/recorder/build/obj/index.js" builder_parse "$@" @@ -40,22 +35,12 @@ if builder_start_action configure; then builder_finish_action success configure fi -if builder_start_action clean:module; then - tsc -b --clean "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success clean:module +if builder_start_action clean; then + tsc --build --clean "$THIS_SCRIPT_PATH/tsconfig.json" + builder_finish_action success clean fi -if builder_start_action clean:proctor; then - tsc -b --clean "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success clean:proctor -fi - -if builder_start_action build:module; then - tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success build:module -fi - -if builder_start_action build:proctor; then - tsc --build "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success build:proctor +if builder_start_action build; then + tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" + builder_finish_action success build fi \ No newline at end of file diff --git a/common/web/recorder/package.json b/common/web/recorder/package.json index ecc8c438188c73208ebd5e4185bbc3ee48f1a5b6..20ff012f595f15bf378b8f4f7068405ca61cf83e 100644 --- a/common/web/recorder/package.json +++ b/common/web/recorder/package.json @@ -1,7 +1,9 @@ { "name": "@keymanapp/recorder-core", "description": "Core classes used to develop KeymanWeb test cases based on keystrokes", - "main": "index.js", + "main": "build/obj/index.js", + "types": "build/obj/index.d.ts", + "type": "module", "scripts": { "tsc": "tsc", "clean": "tsc -b --clean src/tsconfig.json && tsc -b --clean src/nodeProctor.tsconfig.json" diff --git a/common/web/recorder/src/index.ts b/common/web/recorder/src/index.ts index d01dcad71b1cdb278df2bfae6a5160f60f67d6d7..940f663b684c9d3d44569db22199b7c10b76d784 100644 --- a/common/web/recorder/src/index.ts +++ b/common/web/recorder/src/index.ts @@ -1,774 +1,780 @@ -/// - -namespace KMWRecorder { - //#region Defines the InputEventSpec set, used to reconstruct DOM-based events for browser-based simulation - export abstract class InputEventSpec { - type: "key" | "osk"; - static fromJSONObject(obj: any): InputEventSpec { - if(obj && obj.type) { - if(obj.type == "key") { - return new PhysicalInputEventSpec(obj); - } else if(obj.type == "osk") { - return new OSKInputEventSpec(obj); - } - } else { - throw new SyntaxError("Error in JSON format corresponding to an InputEventSpec!"); - } - } +import { type OutputTarget } from "@keymanapp/keyboard-processor"; +import { KeyDistribution, KeyEvent, Mock } from "@keymanapp/keyboard-processor"; - toPrettyJSON(): string { - // We want the default, non-spaced JSON for this class, even when otherwise adding whitespace. - var str = JSON.stringify(this); - return str; - } - } - - export class PhysicalInputEventSpec extends InputEventSpec { - static readonly modifierCodes: { [mod:string]: number } = { - "Shift":0x0001, - "Control":0x0002, - "Alt":0x0004, - "Meta":0x0008, - "CapsLock":0x0010, - "NumLock":0x0020, - "ScrollLock":0x0040 - }; - - // KeyboardEvent properties - type: "key" = "key"; - key: string; - code: string; - keyCode: number; - modifierSet: number; - location: number; - - constructor(e?: PhysicalInputEventSpec) { // parameter is used to reconstruct from JSON. - super(); - - if(e) { - this.key = e.key; - this.code = e.code; - this.keyCode = e.keyCode; - this.modifierSet = e.modifierSet; - this.location = e.location; - } - } +import Proctor from "./proctor.js"; - getModifierState(key: string): boolean { - return (PhysicalInputEventSpec.modifierCodes[key] & this.modifierSet) != 0; - } +export { default as Proctor } from "./proctor.js"; +export { default as NodeProctor } from "./nodeProctor.js"; - generateModifierString(): string { - var list: string = ""; +import * as utils from "@keymanapp/web-utils"; - for(var key in PhysicalInputEventSpec.modifierCodes) { - if(this.getModifierState(key)) { - list += ((list != "" ? " " : "") + key); - } +//#region Defines the InputEventSpec set, used to reconstruct DOM-based events for browser-based simulation +export abstract class InputEventSpec { + type: "key" | "osk"; + static fromJSONObject(obj: any): InputEventSpec { + if(obj && obj.type) { + if(obj.type == "key") { + return new PhysicalInputEventSpec(obj); + } else if(obj.type == "osk") { + return new OSKInputEventSpec(obj); } - - return list; + } else { + throw new SyntaxError("Error in JSON format corresponding to an InputEventSpec!"); } } - export class OSKInputEventSpec extends InputEventSpec { - type: "osk" = "osk"; - keyID: string; - - // The parameter may be used to reconstruct the item from raw JSON. - constructor(e?: OSKInputEventSpec) { - super(); - if(e) { - this.keyID = e.keyID; - } + toPrettyJSON(): string { + // We want the default, non-spaced JSON for this class, even when otherwise adding whitespace. + var str = JSON.stringify(this); + return str; + } +} + +export class PhysicalInputEventSpec extends InputEventSpec { + static readonly modifierCodes: { [mod:string]: number } = { + "Shift":0x0001, + "Control":0x0002, + "Alt":0x0004, + "Meta":0x0008, + "CapsLock":0x0010, + "NumLock":0x0020, + "ScrollLock":0x0040 + }; + + // KeyboardEvent properties + type: "key" = "key"; + key: string; + code: string; + keyCode: number; + modifierSet: number; + location: number; + + constructor(e?: PhysicalInputEventSpec) { // parameter is used to reconstruct from JSON. + super(); + + if(e) { + this.key = e.key; + this.code = e.code; + this.keyCode = e.keyCode; + this.modifierSet = e.modifierSet; + this.location = e.location; } } - //#endregion - export abstract class RecordedKeystroke { - type: "key" | "osk"; + getModifierState(key: string): boolean { + return (PhysicalInputEventSpec.modifierCodes[key] & this.modifierSet) != 0; + } - static fromJSONObject(obj: any): RecordedKeystroke { - if(obj && obj.type) { - if(obj.type == "key") { - return new RecordedPhysicalKeystroke(obj as RecordedPhysicalKeystroke); - } else if(obj && obj.type) { - return new RecordedSyntheticKeystroke(obj as RecordedSyntheticKeystroke); - } - } else { - throw new SyntaxError("Error in JSON format corresponding to a RecordedKeystroke!"); - } - } + generateModifierString(): string { + var list: string = ""; - toPrettyJSON(): string { - // We want the default, non-spaced JSON for this class, even when otherwise adding whitespace. - var str = JSON.stringify(this); - return str; + for(var key in PhysicalInputEventSpec.modifierCodes) { + if(this.getModifierState(key)) { + list += ((list != "" ? " " : "") + key); + } } - /** - * Returns an InputEventSpec that may be used to simulate the keystroke within a browser-based environment. - */ - abstract get inputEventSpec(): InputEventSpec; + return list; } +} - export class RecordedPhysicalKeystroke extends RecordedKeystroke { - // KeyboardEvent properties - type: "key" = "key"; - - keyCode: number; // may be different from eventSpec's value b/c keymapping - states: number; - modifiers: number; - modifierChanged: boolean; - isVirtualKey: boolean; - vkCode: number; // may be possible to eliminate; differences arise from mnemonics. +export class OSKInputEventSpec extends InputEventSpec { + type: "osk" = "osk"; + keyID: string; - eventSpec: PhysicalInputEventSpec; - - constructor(keystroke: RecordedPhysicalKeystroke) - constructor(keystroke: com.keyman.text.KeyEvent, eventSpec: PhysicalInputEventSpec) - constructor(keystroke: RecordedPhysicalKeystroke|com.keyman.text.KeyEvent, eventSpec?: PhysicalInputEventSpec) { - super(); + // The parameter may be used to reconstruct the item from raw JSON. + constructor(e?: OSKInputEventSpec) { + super(); + if(e) { + this.keyID = e.keyID; + } + } +} +//#endregion - if(keystroke instanceof com.keyman.text.KeyEvent || typeof keystroke.type === 'undefined') { - // Store what is necessary for headless event reconstruction. - keystroke = keystroke as com.keyman.text.KeyEvent; - this.keyCode = keystroke.Lcode; - this.states = keystroke.Lstates; - this.modifiers = keystroke.Lmodifiers; - this.modifierChanged = !!keystroke.LmodifierChange; - this.isVirtualKey = keystroke.LisVirtualKey; - this.vkCode = keystroke.vkCode; +export abstract class RecordedKeystroke { + type: "key" | "osk"; - // Also store the DOM-based event spec for use in integrated testing. - this.eventSpec = eventSpec; - } else { - // It might be a raw object, from JSON. - this.keyCode = keystroke.keyCode; - this.states = keystroke.states; - this.modifiers = keystroke.modifiers; - this.modifierChanged = keystroke.modifierChanged; - this.isVirtualKey = keystroke.isVirtualKey; - this.vkCode = keystroke.vkCode; - - this.eventSpec = new PhysicalInputEventSpec(keystroke.eventSpec); // must also be reconstructed. + static fromJSONObject(obj: any): RecordedKeystroke { + if(obj && obj.type) { + if(obj.type == "key") { + return new RecordedPhysicalKeystroke(obj as RecordedPhysicalKeystroke); + } else if(obj && obj.type) { + return new RecordedSyntheticKeystroke(obj as RecordedSyntheticKeystroke); } + } else { + throw new SyntaxError("Error in JSON format corresponding to a RecordedKeystroke!"); } + } + + toPrettyJSON(): string { + // We want the default, non-spaced JSON for this class, even when otherwise adding whitespace. + var str = JSON.stringify(this); + return str; + } - get inputEventSpec(): InputEventSpec { - return this.eventSpec; + /** + * Returns an InputEventSpec that may be used to simulate the keystroke within a browser-based environment. + */ + abstract get inputEventSpec(): InputEventSpec; +} + +export class RecordedPhysicalKeystroke extends RecordedKeystroke { + // KeyboardEvent properties + type: "key" = "key"; + + keyCode: number; // may be different from eventSpec's value b/c keymapping + states: number; + modifiers: number; + modifierChanged: boolean; + isVirtualKey: boolean; + vkCode: number; // may be possible to eliminate; differences arise from mnemonics. + + eventSpec: PhysicalInputEventSpec; + + constructor(keystroke: RecordedPhysicalKeystroke) + constructor(keystroke: KeyEvent, eventSpec: PhysicalInputEventSpec) + constructor(keystroke: RecordedPhysicalKeystroke|KeyEvent, eventSpec?: PhysicalInputEventSpec) { + super(); + + if(keystroke instanceof KeyEvent || typeof keystroke.type === 'undefined') { + // Store what is necessary for headless event reconstruction. + keystroke = keystroke as KeyEvent; + this.keyCode = keystroke.Lcode; + this.states = keystroke.Lstates; + this.modifiers = keystroke.Lmodifiers; + this.modifierChanged = !!keystroke.LmodifierChange; + this.isVirtualKey = keystroke.LisVirtualKey; + this.vkCode = keystroke.vkCode; + + // Also store the DOM-based event spec for use in integrated testing. + this.eventSpec = eventSpec; + } else { + // It might be a raw object, from JSON. + this.keyCode = keystroke.keyCode; + this.states = keystroke.states; + this.modifiers = keystroke.modifiers; + this.modifierChanged = keystroke.modifierChanged; + this.isVirtualKey = keystroke.isVirtualKey; + this.vkCode = keystroke.vkCode; + + this.eventSpec = new PhysicalInputEventSpec(keystroke.eventSpec); // must also be reconstructed. } } - export class RecordedSyntheticKeystroke extends RecordedKeystroke { - // KeyboardEvent properties - type: "osk" = "osk"; + get inputEventSpec(): InputEventSpec { + return this.eventSpec; + } +} - keyName: string; - layer: string; +export class RecordedSyntheticKeystroke extends RecordedKeystroke { + // KeyboardEvent properties + type: "osk" = "osk"; - keyDistribution?: com.keyman.text.KeyDistribution; + keyName: string; + layer: string; - constructor(keystroke: RecordedSyntheticKeystroke) - constructor(keystroke: com.keyman.text.KeyEvent) - constructor(keystroke: RecordedSyntheticKeystroke|com.keyman.text.KeyEvent) { - super(); + keyDistribution?: KeyDistribution; - if(keystroke instanceof com.keyman.text.KeyEvent || typeof keystroke.type === 'undefined') { - keystroke = keystroke as com.keyman.text.KeyEvent; - // Store what is necessary for headless event reconstruction. + constructor(keystroke: RecordedSyntheticKeystroke) + constructor(keystroke: KeyEvent) + constructor(keystroke: RecordedSyntheticKeystroke|KeyEvent) { + super(); - // Also store the DOM-based event spec for use in integrated testing. - this.layer = keystroke.kbdLayer; - this.keyName = keystroke.kName; - this.keyDistribution = keystroke.keyDistribution; - } else { - // It might be a raw object, from JSON. - this.layer = keystroke.layer; - this.keyName = keystroke.keyName; - this.keyDistribution = keystroke.keyDistribution; - } + if(keystroke instanceof KeyEvent || typeof keystroke.type === 'undefined') { + keystroke = keystroke as KeyEvent; + // Store what is necessary for headless event reconstruction. + + // Also store the DOM-based event spec for use in integrated testing. + this.layer = keystroke.kbdLayer; + this.keyName = keystroke.kName; + this.keyDistribution = keystroke.keyDistribution; + } else { + // It might be a raw object, from JSON. + this.layer = keystroke.layer; + this.keyName = keystroke.keyName; + this.keyDistribution = keystroke.keyDistribution; } + } - get inputEventSpec(): InputEventSpec { - let eventSpec = new OSKInputEventSpec(); - eventSpec.keyID = this.layer + '-' + this.keyName; + get inputEventSpec(): InputEventSpec { + let eventSpec = new OSKInputEventSpec(); + eventSpec.keyID = this.layer + '-' + this.keyName; - return eventSpec; - } + return eventSpec; } +} - export abstract class TestSequence { - inputs: KeyRecord[]; - output: string; - msg?: string; +export abstract class TestSequence { + inputs: KeyRecord[]; + output: string; + msg?: string; - abstract hasOSKInteraction(): boolean; + abstract hasOSKInteraction(): boolean; - test(proctor: Proctor, target?: com.keyman.text.OutputTarget): {success: boolean, result: string} { - // Start with an empty OutputTarget and a fresh KeyboardProcessor. - if(!target) { - target = new com.keyman.text.Mock(); - } + test(proctor: Proctor, target?: OutputTarget): {success: boolean, result: string} { + // Start with an empty OutputTarget and a fresh KeyboardProcessor. + if(!target) { + target = new Mock(); + } - proctor.before(); + proctor.before(); - let result = proctor.simulateSequence(this, target); - proctor.assertEquals(result, this.output, this.msg); + let result = proctor.simulateSequence(this, target); + proctor.assertEquals(result, this.output, this.msg); - return {success: (result == this.output), result: result}; - } + return {success: (result == this.output), result: result}; + } - toPrettyJSON(): string { - var str = "{ "; - if(this.output) { - str += "\"output\": \"" + this.output + "\", "; - } - str += "\"inputs\": [\n"; - for(var i = 0; i < this.inputs.length; i++) { - str += " " + this.inputs[i].toPrettyJSON() + ((i == this.inputs.length-1) ? "\n" : ",\n"); - } - if(this.msg) { - str += "], \"message\": \"" + this.msg + "\" }"; - } else { - str += "]}"; - } - return str; + toPrettyJSON(): string { + var str = "{ "; + if(this.output) { + str += "\"output\": \"" + this.output + "\", "; + } + str += "\"inputs\": [\n"; + for(var i = 0; i < this.inputs.length; i++) { + str += " " + this.inputs[i].toPrettyJSON() + ((i == this.inputs.length-1) ? "\n" : ",\n"); + } + if(this.msg) { + str += "], \"message\": \"" + this.msg + "\" }"; + } else { + str += "]}"; } + return str; } +} - export class InputEventSpecSequence extends TestSequence { - inputs: InputEventSpec[]; - output: string; - msg?: string; +export class InputEventSpecSequence extends TestSequence { + inputs: InputEventSpec[]; + output: string; + msg?: string; - constructor(ins?: InputEventSpec[] | InputEventSpecSequence, outs?: string, msg?: string) { - super(); + constructor(ins?: InputEventSpec[] | InputEventSpecSequence, outs?: string, msg?: string) { + super(); - if(ins) { - if(ins instanceof Array) { - this.inputs = [].concat(ins); - } else { - // We're constructing from existing JSON. - this.inputs = []; - - for(var ie=0; ie < ins.inputs.length; ie++) { - this.inputs.push(InputEventSpec.fromJSONObject(ins.inputs[ie])); - } - - this.output = ins.output; - this.msg = ins.msg; - return; - } + if(ins) { + if(ins instanceof Array) { + this.inputs = [].concat(ins); } else { + // We're constructing from existing JSON. this.inputs = []; - } - if(outs) { - this.output = outs; - } + for(var ie=0; ie < ins.inputs.length; ie++) { + this.inputs.push(InputEventSpec.fromJSONObject(ins.inputs[ie])); + } - if(msg) { - this.msg = msg; + this.output = ins.output; + this.msg = ins.msg; + return; } + } else { + this.inputs = []; } - addInput(event: InputEventSpec, output: string) { - this.inputs.push(event); - this.output = output; + if(outs) { + this.output = outs; } - hasOSKInteraction(): boolean { - for(var i=0; i < this.inputs.length; i++) { - if(this.inputs[i] instanceof OSKInputEventSpec) { - return true; - } - } - - return false; + if(msg) { + this.msg = msg; } } - export class RecordedKeystrokeSequence extends TestSequence { - inputs: RecordedKeystroke[]; - output: string; - msg?: string; + addInput(event: InputEventSpec, output: string) { + this.inputs.push(event); + this.output = output; + } - constructor(ins?: RecordedKeystroke[], outs?: string, msg?: string) - constructor(sequence: RecordedKeystrokeSequence) - constructor(ins?: RecordedKeystroke[] | RecordedKeystrokeSequence, outs?: string, msg?: string) { - super(); + hasOSKInteraction(): boolean { + for(var i=0; i < this.inputs.length; i++) { + if(this.inputs[i] instanceof OSKInputEventSpec) { + return true; + } + } - if(ins) { - if(ins instanceof Array) { - this.inputs = [].concat(ins); - } else { - // We're constructing from existing JSON. - this.inputs = []; + return false; + } +} - for(var ie=0; ie < ins.inputs.length; ie++) { - this.inputs.push(RecordedKeystroke.fromJSONObject(ins.inputs[ie])); - } +export class RecordedKeystrokeSequence extends TestSequence { + inputs: RecordedKeystroke[]; + output: string; + msg?: string; - this.output = ins.output; - this.msg = ins.msg; - return; - } + constructor(ins?: RecordedKeystroke[], outs?: string, msg?: string) + constructor(sequence: RecordedKeystrokeSequence) + constructor(ins?: RecordedKeystroke[] | RecordedKeystrokeSequence, outs?: string, msg?: string) { + super(); + + if(ins) { + if(ins instanceof Array) { + this.inputs = [].concat(ins); } else { + // We're constructing from existing JSON. this.inputs = []; - } - if(outs) { - this.output = outs; - } + for(var ie=0; ie < ins.inputs.length; ie++) { + this.inputs.push(RecordedKeystroke.fromJSONObject(ins.inputs[ie])); + } - if(msg) { - this.msg = msg; + this.output = ins.output; + this.msg = ins.msg; + return; } + } else { + this.inputs = []; } - addInput(event: RecordedKeystroke, output: string) { - this.inputs.push(event); - this.output = output; + if(outs) { + this.output = outs; } - hasOSKInteraction(): boolean { - for(var i=0; i < this.inputs.length; i++) { - if(this.inputs[i] instanceof RecordedSyntheticKeystroke) { - return true; - } - } - - return false; + if(msg) { + this.msg = msg; } } - class FontStubForLanguage { - family: string; - source: string[]; - - constructor(activeStubEntry: any) { - this.family = activeStubEntry.family; - - var src = activeStubEntry.files; - if(!(src instanceof Array)) { - src = [ src ]; - } + addInput(event: RecordedKeystroke, output: string) { + this.inputs.push(event); + this.output = output; + } - this.source = []; - for(var i=0; i < src.length; i++) { - this.source.push(activeStubEntry.path + src[i]); + hasOSKInteraction(): boolean { + for(var i=0; i < this.inputs.length; i++) { + if(this.inputs[i] instanceof RecordedSyntheticKeystroke) { + return true; } } + + return false; } +} - export class LanguageStubForKeyboard { - id: string; - name: string; - region: string; - font?: FontStubForLanguage; - oskFont?: FontStubForLanguage; +class FontStubForLanguage { + family: string; + source: string[]; - constructor(activeStub: any) { - if(activeStub.KLC) { - this.id = activeStub.KLC; - this.name = activeStub.KL; - this.region = activeStub.KR; + constructor(activeStubEntry: any) { + this.family = activeStubEntry.family; - // Fonts. - if(activeStub.KFont) { - this.font = new FontStubForLanguage(activeStub.KFont); - } - if(activeStub.KOskFont) { - this.oskFont = new FontStubForLanguage(activeStub.KOskFont); - } - } else { - this.id = activeStub.id; - this.name = activeStub.name; - this.region = activeStub.region; + var src = activeStubEntry.files; + if(!(src instanceof Array)) { + src = [ src ]; + } - // If we end up adding functionality to FontStubForLanguage, we'll need to properly reconstruct these. - this.font = activeStub.font; - this.oskFont = activeStub.oskFont; - } + this.source = []; + for(var i=0; i < src.length; i++) { + this.source.push(activeStubEntry.path + src[i]); } } +} - export class KeyboardStub { - id: string; - name: string; - filename: string; - languages: LanguageStubForKeyboard | LanguageStubForKeyboard[]; - - // Constructs a stub usable with KeymanWeb's addKeyboards() API function from - // the internally-tracked ActiveStub value for that keyboard. - constructor(json?: KeyboardStub) { - if(json) { - this.id = json.id; - this.name = json.name; - this.filename = json.filename; - - if(!Array.isArray(json.languages)) { - this.languages = new LanguageStubForKeyboard(json.languages); - } else { - this.languages = []; - for(var i=0; i < json.languages.length; i++) { - this.languages.push(new LanguageStubForKeyboard(json.languages[i])); - } - } +export class LanguageStubForKeyboard { + id: string; + name: string; + region: string; + font?: FontStubForLanguage; + oskFont?: FontStubForLanguage; + + constructor(activeStub: any) { + if(activeStub.KLC) { + this.id = activeStub.KLC; + this.name = activeStub.KL; + this.region = activeStub.KR; + + // Fonts. + if(activeStub.KFont) { + this.font = new FontStubForLanguage(activeStub.KFont); } - } + if(activeStub.KOskFont) { + this.oskFont = new FontStubForLanguage(activeStub.KOskFont); + } + } else { + this.id = activeStub.id; + this.name = activeStub.name; + this.region = activeStub.region; - getFirstLanguage() { - if(this.languages instanceof LanguageStubForKeyboard) { - return this.languages.id; + // If we end up adding functionality to FontStubForLanguage, we'll need to properly reconstruct these. + this.font = activeStub.font; + this.oskFont = activeStub.oskFont; + } + } +} + +export class KeyboardStub { + id: string; + name: string; + filename: string; + languages: LanguageStubForKeyboard | LanguageStubForKeyboard[]; + + // Constructs a stub usable with KeymanWeb's addKeyboards() API function from + // the internally-tracked ActiveStub value for that keyboard. + constructor(json?: KeyboardStub) { + if(json) { + this.id = json.id; + this.name = json.name; + this.filename = json.filename; + + if(!Array.isArray(json.languages)) { + this.languages = new LanguageStubForKeyboard(json.languages); } else { - return this.languages[0].id; + this.languages = []; + for(var i=0; i < json.languages.length; i++) { + this.languages.push(new LanguageStubForKeyboard(json.languages[i])); + } } } } - type TARGET = 'hardware'|'desktop'|'phone'|'tablet'; - type OS = 'windows'|'android'|'ios'|'macosx'|'linux'; - type BROWSER = 'chrome'|'firefox'|'safari'|'opera'; // ! no 'edge' detection in KMW! - - export class Constraint { - target: TARGET; - validOSList?: OS[]; - validBrowsers?: BROWSER[]; - - constructor(target: TARGET|Constraint, validOSList?: OS[], validBrowsers?: BROWSER[]) { - if(typeof(target) == 'string') { - this.target = target; - this.validOSList = validOSList; - this.validBrowsers = validBrowsers; - } else { - var json = target; - this.target = json.target; - this.validOSList = json.validOSList; - this.validBrowsers = json.validBrowsers; - } + getFirstLanguage() { + if(this.languages instanceof LanguageStubForKeyboard) { + return this.languages.id; + } else { + return this.languages[0].id; + } + } +} + +type TARGET = 'hardware'|'desktop'|'phone'|'tablet'; +type OS = 'windows'|'android'|'ios'|'macosx'|'linux'; +type BROWSER = 'chrome'|'firefox'|'safari'|'opera'; // ! no 'edge' detection in KMW! + +export class Constraint { + target: TARGET; + validOSList?: OS[]; + validBrowsers?: BROWSER[]; + + constructor(target: TARGET|Constraint, validOSList?: OS[], validBrowsers?: BROWSER[]) { + if(typeof(target) == 'string') { + this.target = target; + this.validOSList = validOSList; + this.validBrowsers = validBrowsers; + } else { + var json = target; + this.target = json.target; + this.validOSList = json.validOSList; + this.validBrowsers = json.validBrowsers; } + } - matchesClient(device: com.keyman.utils.DeviceSpec, usingOSK?: boolean) { - // #1: Platform check. - if(usingOSK === true) { - if(this.target != device.formFactor) { - return false; - } - } else if(usingOSK === false) { - if(this.target != 'hardware') { - return false; - } - } else if(this.target != device.formFactor && this.target != 'hardware') { + matchesClient(device: utils.DeviceSpec, usingOSK?: boolean) { + // #1: Platform check. + if(usingOSK === true) { + if(this.target != device.formFactor) { return false; } - - if(this.validOSList) { - if(this.validOSList.indexOf(device.OS as OS) == -1) { - return false; - } - } - - if(this.validBrowsers) { - if(this.validBrowsers.indexOf(device.browser as BROWSER) == -1) { - return false; - } + } else if(usingOSK === false) { + if(this.target != 'hardware') { + return false; } - - return true; + } else if(this.target != device.formFactor && this.target != 'hardware') { + return false; } - // Checks if another Constraint instance is functionally identical to this one. - equals(other: Constraint) { - if(this.target != other.target) { + if(this.validOSList) { + if(this.validOSList.indexOf(device.OS as OS) == -1) { return false; } + } - var list1 = this.validOSList ? this.validOSList : ['any']; - var list2 = other.validOSList ? other.validOSList : ['any']; - - if(list1.sort().join(',') != list2.sort().join(',')) { + if(this.validBrowsers) { + if(this.validBrowsers.indexOf(device.browser as BROWSER) == -1) { return false; } + } - list1 = this.validBrowsers ? this.validBrowsers : ['web']; - list2 = other.validBrowsers ? other.validBrowsers : ['web']; + return true; + } - if(list1.sort().join(',') != list2.sort().join(',')) { - return false; - } + // Checks if another Constraint instance is functionally identical to this one. + equals(other: Constraint) { + if(this.target != other.target) { + return false; + } - return true; + var list1 = this.validOSList ? this.validOSList : ['any']; + var list2 = other.validOSList ? other.validOSList : ['any']; + + if(list1.sort().join(',') != list2.sort().join(',')) { + return false; } - } - export class TestFailure { - constraint: Constraint; - test: InputEventSpecSequence; - result: string; + list1 = this.validBrowsers ? this.validBrowsers : ['web']; + list2 = other.validBrowsers ? other.validBrowsers : ['web']; - constructor(constraint: Constraint, test: InputEventSpecSequence, output: string) { - this.constraint = constraint; - this.test = test; - this.result = output; + if(list1.sort().join(',') != list2.sort().join(',')) { + return false; } + + return true; } +} - export interface TestSet> { - constraint: Constraint; +export class TestFailure { + constraint: Constraint; + test: InputEventSpecSequence; + result: string; - addTest(seq: Sequence): void; - isValidForDevice(device: com.keyman.utils.DeviceSpec, usingOSK?: boolean): boolean; - test(proctor: Proctor): TestFailure[]; + constructor(constraint: Constraint, test: InputEventSpecSequence, output: string) { + this.constraint = constraint; + this.test = test; + this.result = output; } +} - /** - * The core constraint-specific test set definition used for testing versions 10.0 to 13.0. - */ - export class EventSpecTestSet implements TestSet { - constraint: Constraint; - testSet: InputEventSpecSequence[]; - - constructor(constraint: Constraint|EventSpecTestSet) { - if("target" in constraint) { - this.constraint = constraint as Constraint; - this.testSet = []; - } else { - var json = constraint as EventSpecTestSet; - this.constraint = new Constraint(json.constraint); - this.testSet = []; +export interface TestSet> { + constraint: Constraint; - // Clone each test sequence / reconstruct from methodless JSON object. - for(var i=0; i < json.testSet.length; i++) { - this.testSet.push(new InputEventSpecSequence(json.testSet[i])); - } + addTest(seq: Sequence): void; + isValidForDevice(device: utils.DeviceSpec, usingOSK?: boolean): boolean; + test(proctor: Proctor): TestFailure[]; +} + +/** + * The core constraint-specific test set definition used for testing versions 10.0 to 13.0. + */ +export class EventSpecTestSet implements TestSet { + constraint: Constraint; + testSet: InputEventSpecSequence[]; + + constructor(constraint: Constraint|EventSpecTestSet) { + if("target" in constraint) { + this.constraint = constraint as Constraint; + this.testSet = []; + } else { + var json = constraint as EventSpecTestSet; + this.constraint = new Constraint(json.constraint); + this.testSet = []; + + // Clone each test sequence / reconstruct from methodless JSON object. + for(var i=0; i < json.testSet.length; i++) { + this.testSet.push(new InputEventSpecSequence(json.testSet[i])); } } + } - addTest(seq: InputEventSpecSequence) { - this.testSet.push(seq); - } + addTest(seq: InputEventSpecSequence) { + this.testSet.push(seq); + } - // Used to determine if the current EventSpecTestSet is applicable to be run on a device. - isValidForDevice(device: com.keyman.utils.DeviceSpec, usingOSK?: boolean) { - return this.constraint.matchesClient(device, usingOSK); - } + // Used to determine if the current EventSpecTestSet is applicable to be run on a device. + isValidForDevice(device: utils.DeviceSpec, usingOSK?: boolean) { + return this.constraint.matchesClient(device, usingOSK); + } - // Validity should be checked before calling this method. - test(proctor: Proctor): TestFailure[] { - var failures: TestFailure[] = []; - let testSet = this.testSet; + // Validity should be checked before calling this method. + test(proctor: Proctor): TestFailure[] { + var failures: TestFailure[] = []; + let testSet = this.testSet; - for(var i=0; i < testSet.length; i++) { - var testSeq = this[i]; - var simResult = testSet[i].test(proctor); - if(!simResult.success) { - // Failed test! - failures.push(new TestFailure(this.constraint, testSeq, simResult.result)); - } + for(var i=0; i < testSet.length; i++) { + var testSeq = this[i]; + var simResult = testSet[i].test(proctor); + if(!simResult.success) { + // Failed test! + failures.push(new TestFailure(this.constraint, testSeq, simResult.result)); } - - return failures.length > 0 ? failures : null; } + + return failures.length > 0 ? failures : null; } +} - /** - * The core constraint-specific test set definition used for testing versions 10.0 to 13.0. - */ - export class RecordedSequenceTestSet implements TestSet { - constraint: Constraint; - testSet: RecordedKeystrokeSequence[]; - - constructor(constraint: Constraint|RecordedSequenceTestSet) { - if("target" in constraint) { - this.constraint = constraint as Constraint; - this.testSet = []; - } else { - var json = constraint as RecordedSequenceTestSet; - this.constraint = new Constraint(json.constraint); - this.testSet = []; +/** + * The core constraint-specific test set definition used for testing versions 10.0 to 13.0. + */ +export class RecordedSequenceTestSet implements TestSet { + constraint: Constraint; + testSet: RecordedKeystrokeSequence[]; - // Clone each test sequence / reconstruct from methodless JSON object. - for(var i=0; i < json.testSet.length; i++) { - this.testSet.push(new RecordedKeystrokeSequence(json.testSet[i])); - } + constructor(constraint: Constraint|RecordedSequenceTestSet) { + if("target" in constraint) { + this.constraint = constraint as Constraint; + this.testSet = []; + } else { + var json = constraint as RecordedSequenceTestSet; + this.constraint = new Constraint(json.constraint); + this.testSet = []; + + // Clone each test sequence / reconstruct from methodless JSON object. + for(var i=0; i < json.testSet.length; i++) { + this.testSet.push(new RecordedKeystrokeSequence(json.testSet[i])); } } + } - addTest(seq: RecordedKeystrokeSequence) { - this.testSet.push(seq); - } + addTest(seq: RecordedKeystrokeSequence) { + this.testSet.push(seq); + } - // Used to determine if the current EventSpecTestSet is applicable to be run on a device. - isValidForDevice(device: com.keyman.utils.DeviceSpec, usingOSK?: boolean) { - return this.constraint.matchesClient(device, usingOSK); - } + // Used to determine if the current EventSpecTestSet is applicable to be run on a device. + isValidForDevice(device: utils.DeviceSpec, usingOSK?: boolean) { + return this.constraint.matchesClient(device, usingOSK); + } - // Validity should be checked before calling this method. - test(proctor: Proctor): TestFailure[] { - var failures: TestFailure[] = []; - let testSet = this.testSet; + // Validity should be checked before calling this method. + test(proctor: Proctor): TestFailure[] { + var failures: TestFailure[] = []; + let testSet = this.testSet; - for(var i=0; i < testSet.length; i++) { - var testSeq = this[i]; - var simResult = testSet[i].test(proctor); - if(!simResult.success) { - // Failed test! - failures.push(new TestFailure(this.constraint, testSeq, simResult.result)); - } + for(var i=0; i < testSet.length; i++) { + var testSeq = this[i]; + var simResult = testSet[i].test(proctor); + if(!simResult.success) { + // Failed test! + failures.push(new TestFailure(this.constraint, testSeq, simResult.result)); } + } + + return failures.length > 0 ? failures : null; + } - return failures.length > 0 ? failures : null; + toTestName(): string { + let name = "constraint: for " + this.constraint.target; + + if(this.constraint.target == 'hardware') { + name += " keyboard"; + } else { + name += " OSK"; + } + if(this.constraint.validOSList) { + name += " on OS of " + JSON.stringify(this.constraint.validOSList); + } + if(this.constraint.validBrowsers) { + name += " in browser of " + JSON.stringify(this.constraint.validBrowsers); } - toTestName(): string { - let name = "constraint: for " + this.constraint.target; + return name; + } +} - if(this.constraint.target == 'hardware') { - name += " keyboard"; - } else { - name += " OSK"; - } - if(this.constraint.validOSList) { - name += " on OS of " + JSON.stringify(this.constraint.validOSList); - } - if(this.constraint.validBrowsers) { - name += " in browser of " + JSON.stringify(this.constraint.validBrowsers); - } +export class KeyboardTest { + /** + * Indicates what version of KMW's recorder the spec conforms to. + */ + public specVersion: utils.Version = KeyboardTest.CURRENT_VERSION; - return name; - } - } - - export class KeyboardTest { - /** - * Indicates what version of KMW's recorder the spec conforms to. - */ - public specVersion: com.keyman.utils.Version = KeyboardTest.CURRENT_VERSION; - - /** - * The version of KMW in which the Recorder was first written. Worked from 10.0 to 13.0 with - * only backward-compatible changes and minor tweaks to conform to internal API shifts. - */ - public static readonly FALLBACK_VERSION = new com.keyman.utils.Version("10.0"); - public static readonly CURRENT_VERSION = new com.keyman.utils.Version("14.0"); - - /** - * The stub information to be passed into keyman.addKeyboards() in order to run the test. - */ - keyboard: KeyboardStub; - - /** - * The master array of test sets, each of which specifies constraints a client must fulfill for - * the tests contained therein to be valid. - */ - inputTestSets: TestSet[]; - - /** - * Reconstructs a KeyboardTest object from its JSON representation, restoring its methods. - * @param fromJSON - */ - constructor(fromJSON?: string|KeyboardStub|KeyboardTest) { - if(!fromJSON) { - this.keyboard = null; - this.inputTestSets = []; - return; - } else if(typeof(fromJSON) == 'string') { - fromJSON = JSON.parse(fromJSON) as KeyboardTest; - } else if(fromJSON instanceof KeyboardStub) { - this.keyboard = fromJSON; - this.inputTestSets = []; - return; - } + /** + * The version of KMW in which the Recorder was first written. Worked from 10.0 to 13.0 with + * only backward-compatible changes and minor tweaks to conform to internal API shifts. + */ + public static readonly FALLBACK_VERSION = new utils.Version("10.0"); + public static readonly CURRENT_VERSION = new utils.Version("14.0"); - if(!fromJSON.specVersion) { - fromJSON.specVersion = KeyboardTest.FALLBACK_VERSION; - } else { - // Is serialized to a String when saved. - fromJSON.specVersion = new com.keyman.utils.Version(fromJSON.specVersion as unknown as string); - } + /** + * The stub information to be passed into keyman.addKeyboards() in order to run the test. + */ + keyboard: KeyboardStub; + + /** + * The master array of test sets, each of which specifies constraints a client must fulfill for + * the tests contained therein to be valid. + */ + inputTestSets: TestSet[]; - this.keyboard = new KeyboardStub(fromJSON.keyboard); + /** + * Reconstructs a KeyboardTest object from its JSON representation, restoring its methods. + * @param fromJSON + */ + constructor(fromJSON?: string|KeyboardStub|KeyboardTest) { + if(!fromJSON) { + this.keyboard = null; + this.inputTestSets = []; + return; + } else if(typeof(fromJSON) == 'string') { + fromJSON = JSON.parse(fromJSON) as KeyboardTest; + } else if(fromJSON instanceof KeyboardStub) { + this.keyboard = fromJSON; this.inputTestSets = []; - this.specVersion = fromJSON.specVersion; + return; + } - if(this.specVersion.equals(KeyboardTest.FALLBACK_VERSION)) { - // Top-level test spec: EventSpecTestSet, based entirely on browser events. - for(var i=0; i < fromJSON.inputTestSets.length; i++) { - this.inputTestSets[i] = new EventSpecTestSet(fromJSON.inputTestSets[i] as EventSpecTestSet); - } - } else { - for(var i=0; i < fromJSON.inputTestSets.length; i++) { - this.inputTestSets[i] = new RecordedSequenceTestSet(fromJSON.inputTestSets[i] as RecordedSequenceTestSet); - } - } + if(!fromJSON.specVersion) { + fromJSON.specVersion = KeyboardTest.FALLBACK_VERSION; + } else { + // Is serialized to a String when saved. + fromJSON.specVersion = new utils.Version(fromJSON.specVersion as unknown as string); } - addTest(constraint: Constraint, seq: RecordedKeystrokeSequence) { - if(!this.specVersion.equals(KeyboardTest.CURRENT_VERSION)) { - throw new Error("The currently-loaded test was built to an outdated specification and may not be altered."); - } + this.keyboard = new KeyboardStub(fromJSON.keyboard); + this.inputTestSets = []; + this.specVersion = fromJSON.specVersion; - for(var i=0; i < this.inputTestSets.length; i++) { - if(this.inputTestSets[i].constraint.equals(constraint)) { - this.inputTestSets[i].addTest(seq); - return; - } + if(this.specVersion.equals(KeyboardTest.FALLBACK_VERSION)) { + // Top-level test spec: EventSpecTestSet, based entirely on browser events. + for(var i=0; i < fromJSON.inputTestSets.length; i++) { + this.inputTestSets[i] = new EventSpecTestSet(fromJSON.inputTestSets[i] as EventSpecTestSet); + } + } else { + for(var i=0; i < fromJSON.inputTestSets.length; i++) { + this.inputTestSets[i] = new RecordedSequenceTestSet(fromJSON.inputTestSets[i] as RecordedSequenceTestSet); } + } + } - var newSet = new RecordedSequenceTestSet(new Constraint(constraint)); - this.inputTestSets.push(newSet); - newSet.addTest(seq); + addTest(constraint: Constraint, seq: RecordedKeystrokeSequence) { + if(!this.specVersion.equals(KeyboardTest.CURRENT_VERSION)) { + throw new Error("The currently-loaded test was built to an outdated specification and may not be altered."); } - test(proctor: Proctor) { - var setHasRun = false; - var failures: TestFailure[] = []; + for(var i=0; i < this.inputTestSets.length; i++) { + if(this.inputTestSets[i].constraint.equals(constraint)) { + this.inputTestSets[i].addTest(seq); + return; + } + } - proctor.beforeAll(); + var newSet = new RecordedSequenceTestSet(new Constraint(constraint)); + this.inputTestSets.push(newSet); + newSet.addTest(seq); + } - // The original test spec requires a browser environment and thus requires its own `.run` implementation. - if(!(proctor.compatibleWithSuite(this))) { - throw Error("Cannot perform version " + KeyboardTest.FALLBACK_VERSION + "-based testing outside of browser-based environments."); - } + test(proctor: Proctor) { + var setHasRun = false; + var failures: TestFailure[] = []; - // Otherwise, the test spec instances will know how to run in any currently-supported environment. - for(var i = 0; i < this.inputTestSets.length; i++) { - var testSet = this.inputTestSets[i]; + proctor.beforeAll(); - if(proctor.matchesTestSet(testSet)) { - var testFailures = testSet.test(proctor); - if(testFailures) { - failures = failures.concat(testFailures); - } - setHasRun = true; - } - } + // The original test spec requires a browser environment and thus requires its own `.run` implementation. + if(!(proctor.compatibleWithSuite(this))) { + throw Error("Cannot perform version " + KeyboardTest.FALLBACK_VERSION + "-based testing outside of browser-based environments."); + } - if(!setHasRun) { - // The sets CAN be empty, allowing silent failure if/when we actually want that. - console.warn("No test sets for this keyboard were applicable for this device!"); - } + // Otherwise, the test spec instances will know how to run in any currently-supported environment. + for(var i = 0; i < this.inputTestSets.length; i++) { + var testSet = this.inputTestSets[i]; - // Allow the method's caller to trigger a 'fail'. - if(failures.length > 0) { - return failures; - } else { - return null; + if(proctor.matchesTestSet(testSet)) { + var testFailures = testSet.test(proctor); + if(testFailures) { + failures = failures.concat(testFailures); + } + setHasRun = true; } } - isEmpty() { - return this.inputTestSets.length == 0; + if(!setHasRun) { + // The sets CAN be empty, allowing silent failure if/when we actually want that. + console.warn("No test sets for this keyboard were applicable for this device!"); } - toPrettyJSON() { - return JSON.stringify(this, null, ' '); + // Allow the method's caller to trigger a 'fail'. + if(failures.length > 0) { + return failures; + } else { + return null; } + } - get isLegacy(): boolean { - return !this.specVersion.equals(KeyboardTest.CURRENT_VERSION); - } + isEmpty() { + return this.inputTestSets.length == 0; + } + + toPrettyJSON() { + return JSON.stringify(this, null, ' '); + } + + get isLegacy(): boolean { + return !this.specVersion.equals(KeyboardTest.CURRENT_VERSION); } } \ No newline at end of file diff --git a/common/web/recorder/src/nodeProctor.ts b/common/web/recorder/src/nodeProctor.ts index 0ae4b5adcc515ed7686a6902e1b745eb13fc6ba1..5548905e6e96a48b2e8a9a233086ed3dd87b5a70 100644 --- a/common/web/recorder/src/nodeProctor.ts +++ b/common/web/recorder/src/nodeProctor.ts @@ -1,94 +1,101 @@ +import Proctor, { AssertCallback } from "./proctor.js"; +import { + KeyboardTest, + TestSet, + TestSequence, + RecordedKeystrokeSequence, + RecordedPhysicalKeystroke, + RecordedSyntheticKeystroke +} from "./index.js"; + +import { KeyboardInterface, KeyEvent, KeyEventSpec, KeyboardProcessor, Mock, type OutputTarget, KeyboardHarness } from "@keymanapp/keyboard-processor"; +import { DeviceSpec } from "@keymanapp/web-utils"; + +export default class NodeProctor extends Proctor { + private keyboardWithHarness: KeyboardHarness; + public __debug = false; + + constructor(kbdHarness: KeyboardHarness, device: DeviceSpec, assert: AssertCallback) { + super(device, assert); + + this.keyboardWithHarness = kbdHarness; + } -namespace KMWRecorder { - export class NodeProctor extends Proctor { - private keyboard: com.keyman.keyboards.Keyboard; - public __debug = false; - - constructor(keyboard: com.keyman.keyboards.Keyboard, device: com.keyman.utils.DeviceSpec, assert: AssertCallback) { - super(device, assert); - - this.keyboard = keyboard; - } + beforeAll() { + // + } - beforeAll() { - // - } + before() { + // + } - before() { - // - } + compatibleWithSuite(testSuite: KeyboardTest): boolean { + // Original-version tests did not supply core-compatible KeyEvent data. + return !testSuite.specVersion.equals(KeyboardTest.FALLBACK_VERSION); + } - compatibleWithSuite(testSuite: KeyboardTest): boolean { - // Original-version tests did not supply core-compatible KeyEvent data. - return !testSuite.specVersion.equals(KeyboardTest.FALLBACK_VERSION); - } + get debugMode(): boolean { + return this.__debug; + } - get debugMode(): boolean { - return this.__debug; - } + set debugMode(value: boolean) { + this.__debug = value; + } - set debugMode(value: boolean) { - this.__debug = value; - } + matchesTestSet(testSet: TestSet) { + // KeyboardProcessor is abstract enough to run tests aimed at any platform. + return true; + } - matchesTestSet(testSet: TestSet) { - // KeyboardProcessor is abstract enough to run tests aimed at any platform. - return true; + simulateSequence(sequence: TestSequence, target?: OutputTarget): string { + // Start with an empty OutputTarget and a fresh KeyboardProcessor. + if(!target) { + target = new Mock(); } - simulateSequence(sequence: TestSequence, target?: com.keyman.text.OutputTarget): string { - // Start with an empty OutputTarget and a fresh KeyboardProcessor. - if(!target) { - target = new com.keyman.text.Mock(); - } - - // Establish a fresh processor, setting its keyboard appropriately for the test. - let processor = new com.keyman.text.KeyboardProcessor(this.device); - processor.activeKeyboard = this.keyboard; - - if(sequence instanceof RecordedKeystrokeSequence) { - for(let keystroke of sequence.inputs) { - let keyEvent: com.keyman.text.KeyEvent; - if(keystroke instanceof RecordedPhysicalKeystroke) { - // Use the keystroke's stored data to reconstruct the KeyEvent. - keyEvent = { - Lcode: keystroke.keyCode, - Lmodifiers: keystroke.modifiers, - LmodifierChange: keystroke.modifierChanged, - vkCode: keystroke.vkCode, - Lstates: keystroke.states, - kName: '', - device: this.device, - isSynthetic: false, - LisVirtualKey: this.keyboard.definesPositionalOrMnemonic // Only false for 1.0 keyboards. - } - } else if(keystroke instanceof RecordedSyntheticKeystroke) { - let key = this.keyboard.layout(this.device.formFactor).getLayer(keystroke.layer).getKey(keystroke.keyName); - keyEvent = key.constructKeyEvent(processor, this.device); + // Establish a fresh processor, setting its keyboard appropriately for the test. + let processor = new KeyboardProcessor(this.device); + processor.keyboardInterface = this.keyboardWithHarness as KeyboardInterface; + const keyboard = processor.activeKeyboard; + + if(sequence instanceof RecordedKeystrokeSequence) { + for(let keystroke of sequence.inputs) { + let keyEvent: KeyEventSpec; + if(keystroke instanceof RecordedPhysicalKeystroke) { + // Use the keystroke's stored data to reconstruct the KeyEvent. + keyEvent = { + Lcode: keystroke.keyCode, + Lmodifiers: keystroke.modifiers, + LmodifierChange: keystroke.modifierChanged, + vkCode: keystroke.vkCode, + Lstates: keystroke.states, + kName: '', + device: this.device, + isSynthetic: false, + LisVirtualKey: keyboard.definesPositionalOrMnemonic // Only false for 1.0 keyboards. } + } else if(keystroke instanceof RecordedSyntheticKeystroke) { + let key = keyboard.layout(this.device.formFactor).getLayer(keystroke.layer).getKey(keystroke.keyName); + keyEvent = keyboard.constructKeyEvent(key, this.device, processor.stateKeys); + } - // Fill in the final details of the KeyEvent... - keyEvent.device = this.device; + // Fill in the final details of the KeyEvent... + keyEvent.device = this.device; - // And now, execute the keystroke! - // We don't care too much about particularities of per-keystroke behavior yet. - // ... we _could_ if we wanted to, though. The framework is mostly in place; - // it's a matter of actually adding the feature. - let ruleBehavior = processor.processKeystroke(keyEvent, target); + // And now, execute the keystroke! + // We don't care too much about particularities of per-keystroke behavior yet. + // ... we _could_ if we wanted to, though. The framework is mostly in place; + // it's a matter of actually adding the feature. + let ruleBehavior = processor.processKeystroke(new KeyEvent(keyEvent), target); - if(this.debugMode) { - console.log(JSON.stringify(target, null, ' ')); - console.log(JSON.stringify(ruleBehavior, null, ' ')); - } + if(this.debugMode) { + console.log(JSON.stringify(target, null, ' ')); + console.log(JSON.stringify(ruleBehavior, null, ' ')); } - } else { - throw new Error("NodeProctor only supports RecordedKeystrokeSequences for testing at present."); } - return target.getText(); + } else { + throw new Error("NodeProctor only supports RecordedKeystrokeSequences for testing at present."); } - + return target.getText(); } -} - -// Export the namespace itself, giving access to all contained classes. -module.exports = KMWRecorder; \ No newline at end of file +} \ No newline at end of file diff --git a/common/web/recorder/src/nodeProctor.tsconfig.json b/common/web/recorder/src/nodeProctor.tsconfig.json deleted file mode 100644 index 5b80993dc60b2a000381da8e0a525d8f5d841ad7..0000000000000000000000000000000000000000 --- a/common/web/recorder/src/nodeProctor.tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "extends": "../../../../tsconfig-base.json", - - "compilerOptions": { - "allowJs": true, - "module": "none", - "outDir": "../build/nodeProctor/", - "outFile": "../build/nodeProctor/index.js", - "inlineSources": true, - "inlineSourceMap": true, - "target": "es5", - "types": ["node"], - "lib": ["es6"] - }, - - "files": [ - "index.ts", - "proctor.ts", - "nodeProctor.ts" - ], - - "references": [ - { "path": "../../keyman-version" }, - { "path": "../../utils" }, - { "path": "../../keyboard-processor/src" }, - { "path": "../../lm-message-types" } - ] -} diff --git a/common/web/recorder/src/proctor.ts b/common/web/recorder/src/proctor.ts index 38ffd9997aa295993c7ef79d93af285a97beb1e3..50b07032b6d71a6d9a5b2c7d5d10e49eddd2d92d 100644 --- a/common/web/recorder/src/proctor.ts +++ b/common/web/recorder/src/proctor.ts @@ -1,50 +1,53 @@ -namespace KMWRecorder { - export type AssertCallback = (s1: any, s2: any, msg?: string) => void; +import { type DeviceSpec } from "@keymanapp/web-utils"; +import { type OutputTarget } from "@keymanapp/keyboard-processor"; - /** - * Facilitates running Recorder-generated tests on various platforms. - * - * Note that DOM-aware KeymanWeb will implement a Browser-based version, while - * keyboard-processor and input-processor will use a Node-based version instead. - */ - export abstract class Proctor { - device: com.keyman.utils.DeviceSpec; +import type { KeyboardTest, TestSet, TestSequence } from "./index.js"; - _assert: AssertCallback; +export type AssertCallback = (s1: any, s2: any, msg?: string) => void; - constructor(device: com.keyman.utils.DeviceSpec, assert: AssertCallback) { - this.device = device; +/** + * Facilitates running Recorder-generated tests on various platforms. + * + * Note that DOM-aware KeymanWeb will implement a Browser-based version, while + * keyboard-processor and input-processor will use a Node-based version instead. + */ +export default abstract class Proctor { + device: DeviceSpec; - this._assert = assert; - } + _assert: AssertCallback; + + constructor(device: DeviceSpec, assert: AssertCallback) { + this.device = device; - assertEquals(s1: unknown, s2: unknown, msg?: string) { - if(this._assert) { - this._assert(s1, s2, msg); - } + this._assert = assert; + } + + assertEquals(s1: unknown, s2: unknown, msg?: string) { + if(this._assert) { + this._assert(s1, s2, msg); } + } - // Performs global test prep. - abstract beforeAll(); + // Performs global test prep. + abstract beforeAll(); - // Performs per-test setup - abstract before(); + // Performs per-test setup + abstract before(); - /** - * Allows the proctor to indicate if is capable of executing a suite of tests or not. - * @param testSuite - */ - abstract compatibleWithSuite(testSuite: KeyboardTest): boolean; + /** + * Allows the proctor to indicate if is capable of executing a suite of tests or not. + * @param testSuite + */ + abstract compatibleWithSuite(testSuite: KeyboardTest): boolean; - /** - * Indicates whether or not this Proctor is capable of running the specified set of tests. - */ - abstract matchesTestSet(testSet: TestSet); + /** + * Indicates whether or not this Proctor is capable of running the specified set of tests. + */ + abstract matchesTestSet(testSet: TestSet); - /** - * Simulates the specified test sequence for use in testing. - * @param sequence The recorded sequence, generally provided by a test set. - */ - abstract simulateSequence(sequence: TestSequence, target?: com.keyman.text.OutputTarget); - } + /** + * Simulates the specified test sequence for use in testing. + * @param sequence The recorded sequence, generally provided by a test set. + */ + abstract simulateSequence(sequence: TestSequence, target?: OutputTarget); } \ No newline at end of file diff --git a/common/web/recorder/src/tsconfig.json b/common/web/recorder/src/tsconfig.json deleted file mode 100644 index 00cb00a48221f7340f9afb09874f57574967afe5..0000000000000000000000000000000000000000 --- a/common/web/recorder/src/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "extends": "../../../../tsconfig-base.json", - - "compilerOptions": { - "allowJs": true, - "module": "none", - "outDir": "../build/", - "inlineSources": true, - "inlineSourceMap": true, - "target": "es5", - "types": ["node"], - "lib": ["es6"], - "outFile": "../build/index.js" - }, - - "files": [ - "index.ts", - "proctor.ts" - ], - - "references": [ - { "path": "../../keyman-version" }, - { "path": "../../utils" }, - { "path": "../../keyboard-processor/src" }, - { "path": "../../lm-message-types" } - ] -} diff --git a/common/web/recorder/tsconfig.json b/common/web/recorder/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..3e2b801a8324b37886b9f9ab73e7343d51fdd22e --- /dev/null +++ b/common/web/recorder/tsconfig.json @@ -0,0 +1,31 @@ +{ + "extends": "../../../tsconfig-base.json", + + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "module": "es6", + "moduleResolution": "Node", + "declaration": true, + "inlineSources": true, + "inlineSourceMap": true, + "target": "es5", + "types": ["node"], + "lib": ["es6"], + "baseUrl": "./", + "outDir": "build/obj/", + "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo", + "rootDir": "./src" + }, + + "include": [ + "src/**/*.ts" + ], + + "references": [ + { "path": "../keyman-version" }, + { "path": "../utils/" }, + { "path": "../keyboard-processor/" }, + { "path": "../lm-message-types" } + ], +} diff --git a/common/web/sentry-manager/build-bundler.js b/common/web/sentry-manager/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..8c8b21a5f97e15cf73621fd921ff20d7c09fc32c --- /dev/null +++ b/common/web/sentry-manager/build-bundler.js @@ -0,0 +1,21 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; + +await esbuild.build({ + bundle: true, + sourcemap: true, + format: "iife", + nodePaths: ['../../../node_modules'], + entryPoints: { + 'index': 'build/obj/index.js', + }, + outfile: 'build/lib/index.js', + target: "es5", + tsconfig: './src/tsconfig.json' +}); \ No newline at end of file diff --git a/common/web/sentry-manager/build.sh b/common/web/sentry-manager/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..ce20b57a679bdcc8862559bd20eea1344ef7d25e --- /dev/null +++ b/common/web/sentry-manager/build.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Compiles sentry manager + +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# This script runs from its own folder +cd "$THIS_SCRIPT_PATH" + +builder_describe "Builds the Sentry-reporting module used with Keyman Engine for Web" \ + "@/common/web/keyman-version" \ + clean configure build + +builder_describe_outputs \ + configure /node_modules \ + build /common/web/sentry-manager/build/lib/index.js + +builder_parse "$@" + +### CLEAN ACTIONS + +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +if builder_start_action build; then + tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.json" + node build-bundler.js + builder_finish_action success build +fi diff --git a/common/web/sentry-manager/package.json b/common/web/sentry-manager/package.json index dd3ccdbaba71e97972cee23175dcb9b4d13e84d3..a3e6c142bc20f50dcb6a18a0c0b4de155c9b9df8 100644 --- a/common/web/sentry-manager/package.json +++ b/common/web/sentry-manager/package.json @@ -2,6 +2,7 @@ "name": "@keymanapp/web-sentry-manager", "description": "Common setup for KeymanWeb's Sentry-based error reporting", "main": "dist/index.js", + "type": "module", "scripts": { "tsc": "tsc" }, diff --git a/common/web/sentry-manager/src/build.sh b/common/web/sentry-manager/src/build.sh deleted file mode 100755 index 7f3775702ddf6312bd482c73594cb2501d2f825f..0000000000000000000000000000000000000000 --- a/common/web/sentry-manager/src/build.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# -# Compiles sentry manager - -set -eu - -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../../../resources/build/build-utils.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" - -display_usage ( ) { - echo "build.sh [-skip-package-install]" - echo - echo " -skip-package-install skips the `npm install` dependency check." - echo " (or -S) Intended for use when this script is called by another build script." - echo "" - echo " If more than one target is specified, the last one will take precedence." - exit 1 -} - -# Establish default build parameters -set_default_vars ( ) { - FETCH_DEPS=true -} - -set_default_vars - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -skip-package-install|-S) - set_default_vars - FETCH_DEPS=false - ;; - esac - shift # past argument -done - -if [ $FETCH_DEPS = true ]; then - verify_npm_setup - # We need to build keyman-version with a script for now - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || builder_die "Could not build keyman-version" -fi - -tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" -if [ $? -ne 0 ]; then - builder_die "Compilation of package for Sentry integration with KeymanWeb failed." -fi \ No newline at end of file diff --git a/common/web/sentry-manager/src/index.ts b/common/web/sentry-manager/src/index.ts index 81215a575415ab7f79ff561e5d12854202e9bb63..b5bb89c0ff55eb1074a2f02cf081a73a9045ddd8 100644 --- a/common/web/sentry-manager/src/index.ts +++ b/common/web/sentry-manager/src/index.ts @@ -1,250 +1,260 @@ -/// +import KEYMAN_VERSION from '@keymanapp/keyman-version'; -namespace com.keyman { - /** - * Controls whether or not the generated Sentry event is logged to the console (true) - * or sent to our Sentry server (false). - */ - let DEBUG = false; - - type Options = { - hostPlatform: "native-web" | "ios" | "android" - }; - - export class KeymanSentryManager { - keymanPlatform: string; - _enabled: boolean = true; - - static STANDARD_ALIASABLE_FILES = { - 'keymanweb.js': 'keymanweb.js', - 'keymanandroid.js': 'keyman.js', // Android's embedded name -> embedded compilation - 'keymanios.js': 'keyman.js', // iOS's embedded name -> embedded compilation - 'kmwuibutton.js': 'kmwuibutton.js', - 'kmwuifloat.js': 'kmwuifloat.js', - 'kmwuitoggle.js': 'kmwuitoggle.js', - 'kmwuitoolbar.js': 'kmwuitoolbar.js' - // Also add entries for the naming system used by Android and iOS - and map them to the EMBEDDED upload, not the std 'native' one. - } +// Unfortunately, we can't bundle Sentry via direct import - something in the process breaks +// esbuild due to needing to transform to ES5. +import { default as SentryType } from '@sentry/browser'; - static DEFAULT_OPTIONS: Options = { - hostPlatform: "native-web" - } +// But this will get us sufficient typing to work with as long as we also link (or prepend) +// a pre-bundled Sentry build artifact. +let Sentry: { + init: typeof SentryType.init; +} = window['Sentry']; - constructor(options: Options = KeymanSentryManager.DEFAULT_OPTIONS) { - this.keymanPlatform = options.hostPlatform; - } +/** + * Controls whether or not the generated Sentry event is logged to the console (true) + * or sent to our Sentry server (false). + */ +let DEBUG = false; - // If we've recognized one of our source files, - aliasFilename(filename: string): string|null { - if(!this.mayAlias(filename)) { - return null; - } +type Options = { + hostPlatform: "native-web" | "ios" | "android" +}; - filename = KeymanSentryManager.STANDARD_ALIASABLE_FILES[filename]; +export class KeymanSentryManager { + keymanPlatform: string; + _enabled: boolean = true; - switch(location.protocol) { - case 'http:': - return 'http://' + location.host + '/' + filename; - case 'file:': - return 'file:///' + filename; - default: - return null; - } + static STANDARD_ALIASABLE_FILES = { + 'keymanweb.js': 'keymanweb.js', + 'keymanweb-webview.js': 'keymanweb-webview.js', + 'kmwuibutton.js': 'kmwuibutton.js', + 'kmwuifloat.js': 'kmwuifloat.js', + 'kmwuitoggle.js': 'kmwuitoggle.js', + 'kmwuitoolbar.js': 'kmwuitoolbar.js' + // Also add entries for the naming system used by Android and iOS - and map them to the EMBEDDED upload, not the std 'native' one. + } + + static DEFAULT_OPTIONS: Options = { + hostPlatform: "native-web" + } + + constructor(options: Options = KeymanSentryManager.DEFAULT_OPTIONS) { + this.keymanPlatform = options.hostPlatform; + } + + // If we've recognized one of our source files, + aliasFilename(filename: string): string|null { + if(!this.mayAlias(filename)) { + return null; } - mayAlias(filename: string): boolean { - return !!KeymanSentryManager.STANDARD_ALIASABLE_FILES[filename]; + switch(location.protocol) { + case 'http:': + return 'http://' + location.host + '/' + filename; + case 'file:': + return 'file:///' + filename; + default: + return null; } + } + + mayAlias(filename: string): boolean { + return !!KeymanSentryManager.STANDARD_ALIASABLE_FILES[filename]; + } - // Filters all expected but unnecessary path prefixes, affixes, and suffixes reported by Sentry from our products. - // This allows us to mask all different sorts of installations with a single uploaded path. - // Modifies original object. - pathFilter(event: any) { - // Get the underlying JS error. - let exception = event.exception; - if(!exception) { - // events are not required to have an exception property - // e.g. console.log-type events - return; + // Filters all expected but unnecessary path prefixes, affixes, and suffixes reported by Sentry from our products. + // This allows us to mask all different sorts of installations with a single uploaded path. + // Modifies original object. + pathFilter(event: any) { + // Get the underlying JS error. + let exception = event.exception; + if(!exception) { + // events are not required to have an exception property + // e.g. console.log-type events + return; + } + // Iterate through all wrapped exceptions. + for(let e of exception.values) { + // If Sentry was unable to generate a stacktrace, there's no path filtering to + // perform on its stack frames. + if(!e.stacktrace) { + continue; } - // Iterate through all wrapped exceptions. - for(let e of exception.values) { - // If Sentry was unable to generate a stacktrace, there's no path filtering to - // perform on its stack frames. - if(!e.stacktrace) { - continue; - } - for(let frame of e.stacktrace.frames) { - let URL = frame.filename as string; - let filename: string = ''; - try { - filename = URL.substr(URL.lastIndexOf('/')+1); - if(this.mayAlias(filename)) { - // Make the file look like it's at the host's root so that the Sentry server will recognize it - // regardless of actual filepath position. - frame.filename = this.aliasFilename(filename) || frame.filename; - } - } catch { - // not sure what to do yet, but... don't make more errors in the error handler. That's a bad idea. + for(let frame of e.stacktrace.frames) { + let URL = frame.filename as string; + let filename: string = ''; + try { + filename = URL.substr(URL.lastIndexOf('/')+1); + if(this.mayAlias(filename)) { + // Make the file look like it's at the host's root so that the Sentry server will recognize it + // regardless of actual filepath position. + frame.filename = this.aliasFilename(filename) || frame.filename; } + } catch { + // not sure what to do yet, but... don't make more errors in the error handler. That's a bad idea. } } } + } - // Attaches some useful debugging information to the specified object, pass-by-reference style. - attachEventMetadata(event: any) { - // Ensure that the 'extra' object exists. (May not exist for synthetic/custom Errors.) - event.extra = event.extra || {}; - event.extra.keymanState = window['keyman']['getDebugInfo'](); - event.extra.keymanHostPlatform = this.keymanPlatform; - } + // Attaches some useful debugging information to the specified object, pass-by-reference style. + attachEventMetadata(event: any) { + // Ensure that the 'extra' object exists. (May not exist for synthetic/custom Errors.) + event.extra = event.extra || {}; + event.extra.keymanState = window['keyman']['getDebugInfo'](); + event.extra.keymanHostPlatform = this.keymanPlatform; + } - // Sanitizes the event object (in-place) to remove sensitive information - // from the breadcrumbs and url (for embedded KeymanWeb) - sanitizeEvent(event: any) { - if (event && event.breadcrumbs) { - event.breadcrumbs.forEach((b: any) => { - if (b.category == 'navigation') { - let NAVIGATION_PATTERN = /(.*)?(keyboard\.html#[^-]+)-.*/; - b.data.from = b.data.from.replace(NAVIGATION_PATTERN, '$1$2'); - b.data.to = b.data.to.replace(NAVIGATION_PATTERN, '$1$2'); - } - }); - } + // Sanitizes the event object (in-place) to remove sensitive information + // from the breadcrumbs and url (for embedded KeymanWeb) + sanitizeEvent(event: any) { + if (event && event.breadcrumbs) { + event.breadcrumbs.forEach((b: any) => { + if (b.category == 'navigation') { + let NAVIGATION_PATTERN = /(.*)?(keyboard\.html#[^-]+)-.*/; + b.data.from = b.data.from.replace(NAVIGATION_PATTERN, '$1$2'); + b.data.to = b.data.to.replace(NAVIGATION_PATTERN, '$1$2'); + } + }); + } - if (event && event.request && event.request.url) { - let URL_PATTERN = /#.*$/; - event.request.url = event.request.url.replace(URL_PATTERN, ''); - } + if (event && event.request && event.request.url) { + let URL_PATTERN = /#.*$/; + event.request.url = event.request.url.replace(URL_PATTERN, ''); } + } - /** - * Pre-processes a Sentry event object (in-place) to provide more metadata and enhance - * the Sentry server's ability to match the error against release artifacts. - * Also will sanitize the Sentry event. - * @param event A Sentry-generated event - */ - prepareEvent(event: any): boolean { - this.pathFilter(event); - this.attachEventMetadata(event); - this.sanitizeEvent(event); - - if(DEBUG) { - console.log("DEBUG: event object for Sentry") - console.log(event); - return false; //event - } else if(!this._enabled) { - console.error(event); - return false; - } else { - return true; - } + /** + * Pre-processes a Sentry event object (in-place) to provide more metadata and enhance + * the Sentry server's ability to match the error against release artifacts. + * Also will sanitize the Sentry event. + * @param event A Sentry-generated event + */ + prepareEvent(event: any): boolean { + this.pathFilter(event); + this.attachEventMetadata(event); + this.sanitizeEvent(event); + + if(DEBUG) { + console.log("DEBUG: event object for Sentry") + console.log(event); + return false; //event + } else if(!this._enabled) { + console.error(event); + return false; + } else { + return true; } + } - /** - * Allows debugging our custom event preparation code without bombarding Sentry with errors - * during development. - * - * Note that Sentry expects us either to return the event object to be sent or to return `null` - * if we want to prevent the event from being sent to the server. - * @param event - */ - prepareEventDebugWrapper(event: any) { - if(DEBUG) { - try { - if(this.prepareEvent(event)) { - return event; - } else { - return null; - } - } catch(err) { - console.log(err); - } - } else { - // If not in DEBUG mode, simply forward to the actual preparer; we should be notified - // of any event-prep errors that may occur. + /** + * Allows debugging our custom event preparation code without bombarding Sentry with errors + * during development. + * + * Note that Sentry expects us either to return the event object to be sent or to return `null` + * if we want to prevent the event from being sent to the server. + * @param event + */ + prepareEventDebugWrapper(event: any) { + if(DEBUG) { + try { if(this.prepareEvent(event)) { return event; } else { return null; } + } catch(err) { + console.log(err); + } + } else { + // If not in DEBUG mode, simply forward to the actual preparer; we should be notified + // of any event-prep errors that may occur. + if(this.prepareEvent(event)) { + return event; + } else { + return null; } } + } - /** - * Capture errors and warnings logged to Console in order to get - * stack traces. We can't use CaptureConsole integration until we - * upgrade to a newer version of Sentry, which has a bit of a cascade - * of changes required, in particular a change of module type and - * transpiling down to ES5. - * - * https://stackoverflow.com/a/53214615/1836776 - */ - initConsole() { - // creating function declarations for better stacktraces (otherwise they'd be anonymous function expressions) - let oldConsoleError = console.error; - let _this = this; - // Note that Sentry may have overridden console.error so we are re-overriding it here post-init. - console.error = reportingConsoleError; // defined via function hoisting - function reportingConsoleError() { - let args = Array.prototype.slice.call(arguments); - if(_this._enabled) { - Sentry.captureException(reduceConsoleArgs(args), { level: 'error' }); - } - return oldConsoleError.apply(console, args); - }; - - let oldConsoleWarn = console.warn; - console.warn = reportingConsoleWarn; // defined via function hoisting - function reportingConsoleWarn() { - let args = Array.prototype.slice.call(arguments); - if(_this._enabled) { - Sentry.captureMessage(reduceConsoleArgs(args), { level: 'warning' }); - } - return oldConsoleWarn.apply(console, args); + /** + * Capture errors and warnings logged to Console in order to get + * stack traces. We can't use CaptureConsole integration until we + * upgrade to a newer version of Sentry, which has a bit of a cascade + * of changes required, in particular a change of module type and + * transpiling down to ES5. + * + * https://stackoverflow.com/a/53214615/1836776 + */ + initConsole() { + // creating function declarations for better stacktraces (otherwise they'd be anonymous function expressions) + let oldConsoleError = console.error; + let _this = this; + // Note that Sentry may have overridden console.error so we are re-overriding it here post-init. + console.error = reportingConsoleError; // defined via function hoisting + function reportingConsoleError() { + let args = Array.prototype.slice.call(arguments); + if(_this._enabled) { + // @ts-ignore // should be an enum with the same value... if we could import it. + Sentry.captureException(reduceConsoleArgs(args), { level: 'error' }); } + return oldConsoleError.apply(console, args); + }; - function reduceConsoleArgs(args) { - let errorMsg = args[0]; - // Make sure errorMsg is either an error or string. - // It's therefore best to pass in new Error('msg') instead of just 'msg' since - // that'll give you a stack trace leading up to the creation of that new Error - // whereas if you just pass in a plain string 'msg', the stack trace will include - // reportingConsoleError and reportingConsoleCall - if (!(errorMsg instanceof Error)) { - // stringify all args as a new Error (which creates a stack trace) - errorMsg = new Error( - args.reduce(function(accumulator, currentValue) { - return accumulator.toString() + ' ' + currentValue.toString(); - }, '') - ); - } - return errorMsg; + let oldConsoleWarn = console.warn; + console.warn = reportingConsoleWarn; // defined via function hoisting + function reportingConsoleWarn() { + let args = Array.prototype.slice.call(arguments); + if(_this._enabled) { + // @ts-ignore // should be an enum with the same value... if we could import it. + Sentry.captureMessage(reduceConsoleArgs(args), { level: 'warning' }); } + return oldConsoleWarn.apply(console, args); } - init() { - // Do the actual Sentry initialization. - //@ts-ignore - Sentry.init({ - beforeSend: this.prepareEventDebugWrapper.bind(this), - // Not using beforeBreadcrumb because that caused breadcrumbs to get lost in Sentry - debug: DEBUG, - dsn: 'https://cf96f32d107c4286ab2fd82af49c4d3b@o1005580.ingest.sentry.io/5983524', // keyman-web DSN - release: com.keyman.KEYMAN_VERSION.VERSION_GIT_TAG, - environment: com.keyman.KEYMAN_VERSION.VERSION_ENVIRONMENT, - }); - - this.initConsole(); + function reduceConsoleArgs(args) { + let errorMsg = args[0]; + // Make sure errorMsg is either an error or string. + // It's therefore best to pass in new Error('msg') instead of just 'msg' since + // that'll give you a stack trace leading up to the creation of that new Error + // whereas if you just pass in a plain string 'msg', the stack trace will include + // reportingConsoleError and reportingConsoleCall + if (!(errorMsg instanceof Error)) { + // stringify all args as a new Error (which creates a stack trace) + errorMsg = new Error( + args.reduce(function(accumulator, currentValue) { + return accumulator.toString() + ' ' + currentValue.toString(); + }, '') + ); + } + return errorMsg; } + } - get enabled(): boolean { - return this._enabled; - } + init() { + // Do the actual Sentry initialization. + //@ts-ignore + Sentry.init({ + beforeSend: this.prepareEventDebugWrapper.bind(this), + // Not using beforeBreadcrumb because that caused breadcrumbs to get lost in Sentry + debug: DEBUG, + dsn: 'https://cf96f32d107c4286ab2fd82af49c4d3b@o1005580.ingest.sentry.io/5983524', // keyman-web DSN + release: KEYMAN_VERSION.VERSION_GIT_TAG, + environment: KEYMAN_VERSION.VERSION_ENVIRONMENT, + }); - set enabled(value: boolean) { - this._enabled = value; - } + this.initConsole(); + } + + get enabled(): boolean { + return this._enabled; + } + + set enabled(value: boolean) { + this._enabled = value; } } + +// Publish to the window. +window['KeymanSentryManager'] = KeymanSentryManager; \ No newline at end of file diff --git a/common/web/sentry-manager/src/tsconfig.json b/common/web/sentry-manager/src/tsconfig.json index f72444f4b8d25d4c10ce22b914d52ca26562137c..490c315dc90372064fc14cd063dc1dfc50a1a528 100644 --- a/common/web/sentry-manager/src/tsconfig.json +++ b/common/web/sentry-manager/src/tsconfig.json @@ -2,23 +2,20 @@ "extends": "../../../../tsconfig-base.json", "compilerOptions": { "allowJs": true, - "module": "none", - "outDir": "../build/", + "allowSyntheticDefaultImports": true, + "baseUrl": "./", "inlineSources": true, + "lib": ["es6", "dom"], + "module": "es6", + "moduleResolution": "Node16", + "outDir": "../build/obj", + "rootDir": "./", "sourceMap": true, "target": "es5", - "lib": ["es6", "dom"], - "outFile": "../build/index.js" + "tsBuildInfoFile": "../build/obj/tsconfig.tsbuildinfo", }, - "references": [ - { "path": "../../keyman-version" } - ], "include": [ - "../../../../node_modules/@sentry/browser/build/bundle.min.js", - ], - - "files": [ - "index.ts", + "*.ts" ] } \ No newline at end of file diff --git a/common/web/tslib/README.md b/common/web/tslib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4c87474f627b9528f7f774d7c0b50e2a3175bbf8 --- /dev/null +++ b/common/web/tslib/README.md @@ -0,0 +1,15 @@ +The default import setup for the `tslib` package is unfortunately incompatible with `esbuild` when in ES5 mode. But... +with a little elbow grease, we can fix that with _this_ package by importing its ES5-compatible file and exporting it +as _this_ package's default export for use in anything looking to `"importHelpers"`. + +To utilize this with `esbuild` while enabling the `"importHelpers"` compilation option in your tsconfig.json, you'll want +to set the following in your `esbuild` config: + +```javascript + alias: { + 'tslib': '@keymanapp/tslib' + }, +``` + +Note that esbuild 0.15.16 is the minimum required version to utilize the 'alias' feature necessary to replace `tslib` for +`tsc`-generated `import { /* */ } from 'tslib'` statements that result from enabling `"importHelpers"` in a tsconfig. \ No newline at end of file diff --git a/common/web/tslib/build.sh b/common/web/tslib/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..411e32d8bbb4c3817687363b738f1cc3790e5ca6 --- /dev/null +++ b/common/web/tslib/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +cd "$THIS_SCRIPT_PATH" + +################################ Main script ################################ + +builder_describe \ + "A ES5 + esbuild compatibility wrapper for the 'tslib' package." \ + clean configure build + +builder_describe_outputs \ + configure "/node_modules" \ + build "/common/web/tslib/build/index.js" + +builder_parse "$@" + +builder_run_action configure verify_npm_setup +builder_run_action clean rm -rf build/ +builder_run_action build tsc --build \ No newline at end of file diff --git a/common/web/tslib/package.json b/common/web/tslib/package.json new file mode 100644 index 0000000000000000000000000000000000000000..ceec8c64099e23bd539dc23302fe11b39846a4b0 --- /dev/null +++ b/common/web/tslib/package.json @@ -0,0 +1,28 @@ +{ + "name": "@keymanapp/tslib", + "description": "An ES5 + esbuild-compatible wrapper for the 'tslib' library", + "main": "./build/index.js", + "exports": { + ".": { + "types": "./build/index.d.ts", + "import": "./build/index.js" + }, + "./esbuild-tools": { + "types": "./build/esbuild-tools.d.ts", + "import": "./build/esbuild-tools.js" + } + }, + "scripts": { + "build": "gosh ./build.sh build", + "clean": "gosh ./build.sh clean" + }, + "dependencies": { + "tslib": "^2.5.2", + "typescript": "^4.9.5" + }, + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "esbuild": "^0.15.16" + }, + "type": "module" +} diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts new file mode 100644 index 0000000000000000000000000000000000000000..9e258992535eab18c6f1e4b471b538bd429a4393 --- /dev/null +++ b/common/web/tslib/src/esbuild-tools.ts @@ -0,0 +1,327 @@ +import esbuild from 'esbuild'; +import fs from 'fs'; + +// Note: this package is intended 100% as a dev-tool, hence why esbuild is just a dev-dependency. + +// Component #1: Detect all `tslib` helpers we actually want to use. +export async function determineNeededDowncompileHelpers(config: esbuild.BuildOptions, ignoreFilePattern?: RegExp, log?: boolean) { + const tslibHelperNames = [ + "__extends", + "__assign", + "__rest", + "__decorate", + "__param", + "__metadata", + "__awaiter", + "__generator", + "__exportStar", // uses __createBinding + "__createBinding", + "__values", + "__read", + "__spread", + "__spreadArray", + "__spreadArrays", + "__await", + "__asyncGenerator", + "__asyncDelegator", + "__asyncValues", + "__makeTemplateObject", + "__importStar", + "__setModuleDefault", // only used by the previous entry! + "__importDefault", + "__classPrivateFieldGet", + "__classPrivateFieldSet", + "__classPrivateFieldIn", + "__runInitializers", + "__setFunctionName", + "__propKey", + "__esDecorate" + ]; + + const detectedHelpers: string[] = []; + + let tslibHelperDetectionPlugin = { + name: 'tslib helper use detection', + setup(build) { + build.onLoad({filter: /\.js$/}, async (args) => { + let source = await fs.promises.readFile(args.path, 'utf8'); + + let warnings: esbuild.Message[] = []; + + if(/tslib.js$/.test(args.path)) { + let declarationRegex = /var (__[a-zA-Z0-9]+)/g; + let results = source.match(declarationRegex); + + for(let result of results) { + let capture = result.substring(4); + + if(!tslibHelperNames.find((entry) => entry == capture)) { + // Match: `result` itself. Grab index, build location, build message. + let index = source.indexOf(result); + + warnings.push(buildMessage( + 'tslib helper use detection', + 'Probable `tslib` helper `' + capture + '` has not been validated for tree-shaking potential', + indexToSourceLocation(source, index, result.length, args.path) + )); + } + } + + // Returning `undefined` for `content` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. + return { + warnings: warnings + }; + } + + if(ignoreFilePattern?.test(args.path)) { + return; + } + + for(let helper of tslibHelperNames) { + if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { + detectedHelpers.push(helper); + + if(helper == '__importStar') { + detectedHelpers.push('__setModuleDefault'); + detectedHelpers.push('__createBinding'); + } else if(helper == '__exportStar') { + detectedHelpers.push('__createBinding'); + } + } + } + + // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. + return; + }); + } + } + + // tslib tree-shake phase 1 - detecting which helpers are safe to remove. + await esbuild.build({ + ...config, + // `tslibHelperDetectionPlugin` is pass-through, and so has no net effect on + // the build. We just use this run to scan for any utilized `tslib` helper funcs, + // not to manipulate the actual source in any way. + plugins: [ tslibHelperDetectionPlugin, ...(config.plugins ?? []) ], + write: false + }); + + // At this point, we can determine what's unused. + detectedHelpers.sort(); + + const unusedHelpers: string[] = []; + tslibHelperNames.forEach((entry) => { + if(!detectedHelpers.find((detected) => detected == entry)) { + unusedHelpers.push(entry); + } + }); + + if(log) { + // Logs on the tree-shaking decisions + console.log("Detected helpers from tslib: "); + console.log(); + + console.log(); + console.log("Unused helpers: "); + console.log(unusedHelpers); + } + + return unusedHelpers; +} + +function indexToSourceLocation(source: string, index: number, matchLength: number, file: string): esbuild.Location { + let priorText = source.substring(0, index); + let lineNum = priorText.split('\n').length; + let lastLineBreakIndex = priorText.lastIndexOf('\n'); + let colNum = index - lastLineBreakIndex; + let nextLineBreakIndex = source.indexOf('\n', lastLineBreakIndex+1); + + return { + file: file, + line: lineNum, + column: colNum, + lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex), + namespace: '', + suggestion: '', + length: matchLength + } +} + +function buildMessage(pluginName: string, message: string, location: esbuild.Location): esbuild.Message { + return { + id: '', + notes: [], + detail: '', + pluginName: pluginName, + text: message, + location: location + } +} + +// Ugh. https://github.com/evanw/esbuild/issues/2656#issuecomment-1304013941 +function wrapClassPlugin(instancePlugin: esbuild.Plugin) { + return { + name: instancePlugin.name, + setup: instancePlugin.setup.bind(instancePlugin) + }; +} + +// Component #2: when we've actually determined which ones are safe to remove, this plugin +// can remove their code. +class TslibTreeshaker implements esbuild.Plugin { + public readonly name = 'tslib forced treeshaking'; + + private unusedHelpers: string[]; + + constructor(unusedHelpers: string[]) { + this.unusedHelpers = unusedHelpers; + } + + treeshakeDefinition(source: string, start: number, expectTernary: boolean, name: string, location: esbuild.Location) { + let scopeDepth = 0; + let parenDepth = 0; + let topLevelTernaryActive = 0; + let i = start; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0 || topLevelTernaryActive) { + if(char == '{') { + scopeDepth++; + } else if(char == '(') { + parenDepth++; + } else if(char == ')') { + parenDepth--; + } else if(char == '?' && scopeDepth == 0) { + if(!expectTernary) { + return { + source: source, + error: buildMessage(this.name, `Unexpected conditional for definition of ${name}`, location) + } + } + topLevelTernaryActive++; + } else if(char == ':' && topLevelTernaryActive && scopeDepth == 0) { + topLevelTernaryActive--; + } + i++; + + if(i > source.length) { + return { + source: source, + error: buildMessage(this.name, `Failed to determine end of definition for ${name}`, location) + }; + } + + char = source.charAt(i); + } + i++; // we want to erase the final '}', too. + + // The functions may be wrapped with parens. + if(parenDepth == 1) { + let nextOpen = source.indexOf('(', i); + let nextClosed = source.indexOf(')', i); + + if(nextOpen < nextClosed) { + return { + source: source, + error: buildMessage(this.name, `Failed to determine end of definition for ${name}`, location) + }; + } else { + i = nextClosed + 1; + } + } + + if(source.charAt(i) == ';') { + i++; + } + + return { + source: source.replace(source.substring(start, i), '') + }; + } + + setup(build: esbuild.PluginBuild) { + build.onLoad({filter: /tslib.js$/}, async (args) => { + const trueSource = await fs.promises.readFile(args.path, 'utf8'); + let source = trueSource; + + let warnings: esbuild.Message[] = []; + let errors: esbuild.Message[] = []; + + for(let unusedHelper of this.unusedHelpers) { + // Removes the 'exporter' line used to actually export it from the tslib source. + source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + + // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. + let matchString = `${unusedHelper} = function`; + let expectTernary = false; + + // A special case - it has two different versions depending on if Object.create exists or not. + // This is established via a ternary conditional. Just adds a bit to the parsing. + if(unusedHelper == '__createBinding') { + matchString = `${unusedHelper} = `; + expectTernary = true; + } else if(unusedHelper == '__setModuleDefault') { + matchString = `var ${unusedHelper} = `; // is inlined, not declared at top! + expectTernary = true; + } + let definitionStart = source.indexOf(matchString); + + if(definitionStart > -1) { + const result = this.treeshakeDefinition( + source, + definitionStart, + expectTernary, + unusedHelper, + indexToSourceLocation(trueSource, trueSource.indexOf(matchString), matchString.length, args.path) + ); + + if(result.error) { + errors.push(result.error); + } else { + source = result.source; + } + continue; + } + + // If we reached this point, we couldn't treeshake the unused helper appropriately. + if(definitionStart == -1) { + // Matches the standard definition pattern's left-hand assignment component. + matchString = `${unusedHelper} =` + let bestGuessIndex = trueSource.indexOf(matchString); + + if(bestGuessIndex == -1) { + // Failing the above, we match the declaration higher up within the file. + matchString = `var ${unusedHelper}` + bestGuessIndex = trueSource.indexOf(matchString); + } + + // Failing THAT, we just give up and go start-of-file. + if(bestGuessIndex == -1) { + matchString = ''; + bestGuessIndex = 0; + } + + warnings.push( + buildMessage( + this.name, + "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, + indexToSourceLocation(trueSource, bestGuessIndex, matchString.length, args.path) + ) + ); + } + } + + return { + contents: source, + loader: 'js', + warnings: warnings, + errors: errors + }; + }); + } +}; + +export function buildTslibTreeshaker(unusedHelpers: string[]) { + return wrapClassPlugin(new TslibTreeshaker(unusedHelpers)); +}; \ No newline at end of file diff --git a/common/web/tslib/src/index.ts b/common/web/tslib/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..2f173ad83fa91de2a1a7c3f982e39bae3e5617d9 --- /dev/null +++ b/common/web/tslib/src/index.ts @@ -0,0 +1,3 @@ +export * from '../../../../node_modules/tslib/tslib.js'; + +export * as tslib from '../../../../node_modules/tslib/tslib.js'; \ No newline at end of file diff --git a/common/web/tslib/tsconfig.json b/common/web/tslib/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..c141af3d3c5f0ae12b4bb8c67f824a443b13383a --- /dev/null +++ b/common/web/tslib/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../../tsconfig-base.json", + "compilerOptions": { + "allowJs": true, + "module": "es6", + "moduleResolution": "Node", + "allowSyntheticDefaultImports": true, + "inlineSources": true, + "sourceMap": true, + "declaration": true, + "target": "es5", + "tsBuildInfoFile": "./build/tsconfig.tsbuildinfo", + "types": ["node"], + "lib": ["es6", "DOM"], // The latter is b/c esbuild expects the type. + "baseUrl": "./src", + "outDir": "./build/", + "rootDir": "./src" + }, + "include": [ + "./src/*.ts" + ] +} diff --git a/common/web/types/src/kpj/kpj-file-reader.ts b/common/web/types/src/kpj/kpj-file-reader.ts index 5f768c54f1ea159dccd318e1eb997be1d1d5e5c6..21e92c975b2313a3dd0f1508cbd663ef20654188 100644 --- a/common/web/types/src/kpj/kpj-file-reader.ts +++ b/common/web/types/src/kpj/kpj-file-reader.ts @@ -27,8 +27,8 @@ export class KPJFileReader { return data as KPJFile; } - public validate(source: KPJFile, schemaBuffer: Buffer): void { - const schema = JSON.parse(schemaBuffer.toString('utf8')); + public validate(source: KPJFile, schemaBuffer: Uint8Array): void { + const schema = JSON.parse(new TextDecoder().decode(schemaBuffer)); const ajv = new Ajv(); if(!ajv.validate(schema, source)) { throw new Error(ajv.errorsText()); diff --git a/common/web/types/src/kvk/kvks-file-reader.ts b/common/web/types/src/kvk/kvks-file-reader.ts index c12b7036e4a3828098dd05ae5f99536d9e081783..13c6c8716c11c1b6a02e98f8b63fc18ed12cb6d0 100644 --- a/common/web/types/src/kvk/kvks-file-reader.ts +++ b/common/web/types/src/kvk/kvks-file-reader.ts @@ -7,12 +7,6 @@ import { VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKey import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js'; import { BUILDER_KVK_HEADER_VERSION } from './kvk-file.js'; -export enum KVKSParseErrorType { invalidVkey }; -export class KVKSParseError extends Error { - public type: KVKSParseErrorType; - public vkey: string; -}; - export default class KVKSFileReader { public read(file: Uint8Array): KVKSourceFile { let source: KVKSourceFile; @@ -33,10 +27,12 @@ export default class KVKSFileReader { // rather than using the version tagged on npmjs.com. }); - parser.parseString(file, (e: unknown, r: unknown) => { source = r as KVKSourceFile }); - source = this.boxArrays(source); - this.cleanupFlags(source); - this.cleanupUnderscore('visualkeyboard', source.visualkeyboard); + parser.parseString(file, (e: unknown, r: unknown) => { if(e) { throw e }; source = r as KVKSourceFile }); + if(source) { + source = this.boxArrays(source); + this.cleanupFlags(source); + this.cleanupUnderscore('visualkeyboard', source.visualkeyboard); + } return source; } @@ -72,15 +68,15 @@ export default class KVKSFileReader { } } - public validate(source: KVKSourceFile, schemaBuffer: Buffer): void { - const schema = JSON.parse(schemaBuffer.toString('utf8')); + public validate(source: KVKSourceFile, schemaBuffer: Uint8Array): void { + const schema = JSON.parse(new TextDecoder().decode(schemaBuffer)); const ajv = new Ajv(); if(!ajv.validate(schema, source)) { throw new Error(ajv.errorsText()); } } - public transform(source: KVKSourceFile, errors?: KVKSParseError[]): VisualKeyboard { + public transform(source: KVKSourceFile, invalidVkeys?: string[]): VisualKeyboard { // NOTE: at this point, the xml should have been validated // and matched the schema result so we can assume properties exist let result: VisualKeyboard = { @@ -118,11 +114,8 @@ export default class KVKSFileReader { for(let sourceKey of layer.key) { let vkey = (USVirtualKeyCodes as any)[sourceKey.$?.vkey]; if(!vkey) { - if(errors) { - let e = new KVKSParseError(); - e.type = KVKSParseErrorType.invalidVkey; - e.vkey = sourceKey.$?.vkey; - errors.push(e); + if(typeof invalidVkeys !== 'undefined') { + invalidVkeys.push(sourceKey.$?.vkey); } continue; } diff --git a/common/web/types/src/kvk/kvks-file-writer.ts b/common/web/types/src/kvk/kvks-file-writer.ts index 613c6caa5108289ac9c99901ddc00a68383313dd..1b3a819f580f2d2ef23f1582aade95c98a6b97a7 100644 --- a/common/web/types/src/kvk/kvks-file-writer.ts +++ b/common/web/types/src/kvk/kvks-file-writer.ts @@ -96,7 +96,6 @@ export default class KVKSFileWriter { l.key.push(k); } - // console.dir(kvks, {depth:8}); let result = builder.buildObject(kvks); return result; //Uint8Array.from(result); } diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 35d7f627956417b508e9b358473ce019147afc80..fa74126462451b8a0dd4c70a30532f7f82cc6c53 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -14,16 +14,20 @@ interface NameAndProps { '$$'?: any; // children }; -export default class LDMLKeyboardXMLSourceFileReader { - callbacks: CompilerCallbacks; +export class LDMLKeyboardXMLSourceFileReaderOptions { + importsPath: string; +}; + +export class LDMLKeyboardXMLSourceFileReader { + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + } - constructor(callbacks : CompilerCallbacks) { - this.callbacks = callbacks; + static get defaultImportsURL() { + return new URL(`../import/`, import.meta.url); } - readImportFile(version: string, subpath: string): Buffer { - // TODO-LDML: sanitize input string - let importPath = new URL(`../import/${version}/${subpath}`, import.meta.url); + readImportFile(version: string, subpath: string): Uint8Array { + const importPath = this.callbacks.resolveFilename(this.options.importsPath, `${version}/${subpath}`); return this.callbacks.loadFile(importPath); } @@ -201,8 +205,8 @@ export default class LDMLKeyboardXMLSourceFileReader { /** * @returns true if valid, false if invalid */ - public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile, schemaSource: Buffer): boolean { - const schema = JSON.parse(schemaSource.toString('utf8')); + public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile, schemaSource: Uint8Array): boolean { + const schema = JSON.parse(new TextDecoder().decode(schemaSource)); const ajv = new Ajv(); if(!ajv.validate(schema, source)) { for (let err of ajv.errors) { diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 4e7755038403dc3b57ceecc96d2dbd9470a19aa9..ddaa318d3e073cba9a368cd1f20bcb71eb7c40fc 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -14,7 +14,7 @@ export * as KvksFile from './kvk/kvk-file.js'; export * as LDMLKeyboard from './ldml-keyboard/ldml-keyboard-xml.js'; export { LDMLKeyboardTestDataXMLSourceFile } from './ldml-keyboard/ldml-keyboard-testdata-xml.js'; -export { default as LDMLKeyboardXMLSourceFileReader } from './ldml-keyboard/ldml-keyboard-xml-reader.js'; +export { LDMLKeyboardXMLSourceFileReader, LDMLKeyboardXMLSourceFileReaderOptions } from './ldml-keyboard/ldml-keyboard-xml-reader.js'; export * as Constants from './consts/virtual-key-constants.js'; diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index 849e945321fb5b811f2c81b9b05b7d5d74455381..d3932b7bfa887a997569f686c7309f5421de0152 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -120,13 +120,11 @@ export interface CompilerFileSystemCallbacks { export interface CompilerCallbacks { /** * Attempt to load a file. Return falsy if not found. - * TODO: accept only string * TODO: never return falsy, just throw if not found? - * TODO: Buffer is Node-only. * @param baseFilename * @param filename */ - loadFile(filename: string | URL): Buffer; + loadFile(filename: string): Uint8Array; get path(): CompilerPathCallbacks; get fs(): CompilerFileSystemCallbacks; @@ -138,7 +136,7 @@ export interface CompilerCallbacks { */ resolveFilename(baseFilename: string, filename: string): string; - loadSchema(schema: CompilerSchema): Buffer; + loadSchema(schema: CompilerSchema): Uint8Array; reportMessage(event: CompilerEvent): void; debug(msg: string): void; }; diff --git a/common/web/types/test/helpers/TestCompilerCallbacks.ts b/common/web/types/test/helpers/TestCompilerCallbacks.ts index 596a731d58880a1fbf4d7dab95a81c795bf212e1..8316264a774bf8b85e4adeb94c48937935247fdf 100644 --- a/common/web/types/test/helpers/TestCompilerCallbacks.ts +++ b/common/web/types/test/helpers/TestCompilerCallbacks.ts @@ -42,7 +42,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return resolveFilename(baseFilename, filename); } - loadFile(filename: string | URL): Buffer { + loadFile(filename: string): Uint8Array { // TODO: error management, does it belong here? try { return loadFile(filename); diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/test/helpers/index.ts index 58dc32dc9c1618da13acb027d39ad3a379a53394..06906df148589fb5a10808b92e685888d5fd8a7b 100644 --- a/common/web/types/test/helpers/index.ts +++ b/common/web/types/test/helpers/index.ts @@ -28,7 +28,10 @@ export function loadSchema(schema: CompilerSchema): Buffer { } export function resolveFilename(baseFilename: string, filename: string) { - const basePath = path.dirname(baseFilename); + const basePath = + baseFilename.endsWith('/') || baseFilename.endsWith('\\') ? + baseFilename : + path.dirname(baseFilename); // Transform separators to platform separators -- we are agnostic // in our use here but path prefers files may use // either / or \, although older kps files were always \. diff --git a/common/web/types/test/helpers/reader-callback-test.ts b/common/web/types/test/helpers/reader-callback-test.ts index 8c85a759acf270a8a99e5eb947faf8b51db8bb93..204e0ec5bf4875168e082630354ff2453b756def 100644 --- a/common/web/types/test/helpers/reader-callback-test.ts +++ b/common/web/types/test/helpers/reader-callback-test.ts @@ -1,11 +1,16 @@ import 'mocha'; import {assert} from 'chai'; import { loadFile, makePathToFixture, loadSchema } from '../helpers/index.js'; -import LDMLKeyboardXMLSourceFileReader from '../../src/ldml-keyboard/ldml-keyboard-xml-reader.js'; +import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboardXMLSourceFileReaderOptions } from '../../src/ldml-keyboard/ldml-keyboard-xml-reader.js'; import { CompilerEvent } from '../../src/util/compiler-interfaces.js'; import { LDMLKeyboardXMLSourceFile } from '../../src/ldml-keyboard/ldml-keyboard-xml.js'; import { LDMLKeyboardTestDataXMLSourceFile } from '../ldml-keyboard/ldml-keyboard-testdata-xml.js'; import { TestCompilerCallbacks } from './TestCompilerCallbacks.js'; +import { fileURLToPath } from 'url'; + +const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = { + importsPath: fileURLToPath(LDMLKeyboardXMLSourceFileReader.defaultImportsURL) +}; export interface CompilationCase { /** @@ -71,7 +76,7 @@ export interface TestDataCase { export function testReaderCases(cases : CompilationCase[]) { // we need our own callbacks rather than using the global so messages don't get mixed const callbacks = new TestCompilerCallbacks(); - const reader = new LDMLKeyboardXMLSourceFileReader(callbacks); + const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); for (let testcase of cases) { const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`: @@ -122,7 +127,7 @@ export function testReaderCases(cases : CompilationCase[]) { export function testTestdataReaderCases(cases : TestDataCase[]) { // we need our own callbacks rather than using the global so messages don't get mixed const callbacks = new TestCompilerCallbacks(); - const reader = new LDMLKeyboardXMLSourceFileReader(callbacks); + const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); for (let testcase of cases) { const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`: diff --git a/common/web/types/test/kpj/test-kpj-file-reader.ts b/common/web/types/test/kpj/test-kpj-file-reader.ts index b7c2f710131db0d9bdf8be3f9190a267ee4a2ad5..95b5f1d01ae86030139e1ff594eed17fb5796713 100644 --- a/common/web/types/test/kpj/test-kpj-file-reader.ts +++ b/common/web/types/test/kpj/test-kpj-file-reader.ts @@ -15,7 +15,6 @@ describe('kpj-file-reader', function () { const input = fs.readFileSync(path); const reader = new KPJFileReader(callbacks); const kpj = reader.read(input); - console.dir(kpj); assert.doesNotThrow(() => { reader.validate(kpj, loadSchema('kpj')); }); @@ -79,7 +78,6 @@ describe('kpj-file-reader', function () { assert.lengthOf(project.files, 2); let f: KeymanDeveloperProjectFile10 = project.files[0]; - console.dir(f); assert.equal(f.id, 'id_f347675c33d2e6b1c705c787fad4941a'); assert.equal(f.filename, 'khmer_angkor.kmn'); assert.equal(f.filePath, 'source/khmer_angkor.kmn'); diff --git a/common/web/types/test/kvk/test-kvk-round-trip.ts b/common/web/types/test/kvk/test-kvk-round-trip.ts index 41dbe2dbc0cb25b553c790172998f006269c416b..031ad60b9e6e1c2fda395c2dda7caf8614794a90 100644 --- a/common/web/types/test/kvk/test-kvk-round-trip.ts +++ b/common/web/types/test/kvk/test-kvk-round-trip.ts @@ -5,7 +5,7 @@ import Hexy from 'hexy'; import gitDiff from 'git-diff'; const { hexy } = Hexy; import { loadSchema, makePathToFixture } from '../helpers/index.js'; -import KvksFileReader, { KVKSParseError } from "../../src/kvk/kvks-file-reader.js"; +import KvksFileReader from "../../src/kvk/kvks-file-reader.js"; import KvkFileReader from "../../src/kvk/kvk-file-reader.js"; import KvkFileWriter from "../../src/kvk/kvk-file-writer.js"; import KvksFileWriter from "../../src/kvk/kvks-file-writer.js"; @@ -53,9 +53,9 @@ describe('kvks-file-reader', function () { assert.doesNotThrow(() => { reader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); const writer = new KvkFileWriter(); const output = writer.write(vk); assertBufferMatch(Buffer.from(output), compiled); @@ -76,9 +76,9 @@ describe('kvks-file-reader', function () { assert.doesNotThrow(() => { kvksReader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk2 = kvksReader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk2 = kvksReader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); // make sure the binary is the same assert.deepEqual(vk2, vk); diff --git a/common/web/types/test/kvk/test-kvks-file.ts b/common/web/types/test/kvk/test-kvks-file.ts index 523bfaf54d618ce21da9f9137c67d9b04c39bd6a..4cbb299a0085b188d405d5a46aa23e964ad1a344 100644 --- a/common/web/types/test/kvk/test-kvks-file.ts +++ b/common/web/types/test/kvk/test-kvks-file.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import 'mocha'; import { loadSchema, makePathToFixture } from '../helpers/index.js'; -import KvksFileReader, { KVKSParseError } from "../../src/kvk/kvks-file-reader.js"; +import KvksFileReader from "../../src/kvk/kvks-file-reader.js"; import KvksFileWriter from "../../src/kvk/kvks-file-writer.js"; import { verify_khmer_angkor } from './test-kvk-utils.js'; import { assert } from 'chai'; @@ -16,9 +16,9 @@ describe('kvks-file-reader', function() { assert.doesNotThrow(() => { reader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); verify_khmer_angkor(vk); }); }); @@ -30,9 +30,9 @@ describe('kvks-file-writer', function() { const reader = new KvksFileReader(); const kvksExpected = reader.read(input); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvksExpected, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvksExpected, invalidVkeys); + assert.isEmpty(invalidVkeys); const writer = new KvksFileWriter(); const output = writer.write(vk); diff --git a/common/web/types/test/tsconfig.json b/common/web/types/test/tsconfig.json index 9c97b463afd01e2855a701a35ed22430b978ac90..c522129c99a432b0bd3a7c7b0639488578e44d4b 100644 --- a/common/web/types/test/tsconfig.json +++ b/common/web/types/test/tsconfig.json @@ -14,7 +14,7 @@ "./helpers/*.ts", ], "references": [ - { "path": "../../keyman-version/tsconfig.esm.json" }, + { "path": "../../keyman-version" }, { "path": "../../../../core/include/ldml/"}, { "path": "../" }, ] diff --git a/common/web/types/tsconfig.json b/common/web/types/tsconfig.json index a32b8e1ed7f7fd098b0cdc87b64517623f6503ad..983a09fda6721a964099200885c0af74a68024c9 100644 --- a/common/web/types/tsconfig.json +++ b/common/web/types/tsconfig.json @@ -11,7 +11,7 @@ "src/**/*.ts" ], "references": [ - { "path": "../keyman-version/tsconfig.esm.json" }, + { "path": "../keyman-version" }, { "path": "../../../core/include/ldml/tsconfig.json"}, ] } diff --git a/common/web/utils/build-bundler.js b/common/web/utils/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..cda215474f8ecc4eae4c086c72028cf3bdfffcb2 --- /dev/null +++ b/common/web/utils/build-bundler.js @@ -0,0 +1,44 @@ +/* + * Bundles @keymanapp/web-utils as single-file modules based upon the export list in src/index.ts. + */ + +import esbuild from 'esbuild'; +import { spawn } from 'child_process'; + +// Bundles to a compact ESModule +esbuild.buildSync({ + entryPoints: ['build/obj/index.js'], + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + //minify: true, // No need to minify a module. + //keepNames: true, + format: "esm", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version import to resolve. + nodePaths: ['..'], + outfile: "build/lib/index.mjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); + +// Bundles to a compact CommonJS (classic Node) module +esbuild.buildSync({ + entryPoints: ['build/obj/index.js'], + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + //minify: true, // No need to minify a module. + //keepNames: true, + format: "cjs", + // Sets 'common/web' as a root folder for module resolution; + // this allows the keyman-version import to resolve. + nodePaths: ['..'], + outfile: "build/lib/index.cjs", + tsconfig: 'tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/common/web/utils/build.sh b/common/web/utils/build.sh index 012f6ad41e9ace13da34a665ec2048e52ff2d724..d4ff9ef8f31dde20531e589695cae39ffc2dd1ac 100755 --- a/common/web/utils/build.sh +++ b/common/web/utils/build.sh @@ -19,11 +19,13 @@ cd "$THIS_SCRIPT_PATH" builder_describe \ "Compiles the web-oriented utility function module." \ "@/common/web/keyman-version" \ - clean configure build test + "@/common/web/tslib" \ + clean configure build test \ + "--ci For use with action ${BUILDER_TERM_START}test${BUILDER_TERM_END} - emits CI-friendly test reports" builder_describe_outputs \ configure "/node_modules" \ - build "/common/web/utils/build/index.js" + build "/common/web/utils/build/lib/index.mjs" builder_parse "$@" @@ -38,11 +40,25 @@ if builder_start_action clean; then fi if builder_start_action build; then - # Note: in a dependency build, we'll expect utils to be built by tsc -b - if builder_is_dep_build; then - builder_echo "skipping tsc -b; will be completed by $builder_dep_parent" - else - tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" - fi + tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" + node build-bundler.js + + # So... tsc does declaration-bundling on its own pretty well, at least for local development. + tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts + builder_finish_action success build +fi + +if builder_start_action test; then + builder_heading "Running web-utils test suite" + + FLAGS= + if builder_has_option --ci; then + echo "Replacing user-friendly test reports with CI-friendly versions." + FLAGS="$FLAGS --reporter mocha-teamcity-reporter" + fi + + mocha --recursive $FLAGS ./src/test/ + + builder_finish_action success test fi \ No newline at end of file diff --git a/common/web/utils/package.json b/common/web/utils/package.json index fbe518e2b8ae0a82bcc0df6e69311453cb18af9b..31e8388bb764929f8aa07d850a45344465dfb9b2 100644 --- a/common/web/utils/package.json +++ b/common/web/utils/package.json @@ -1,11 +1,26 @@ { "name": "@keymanapp/web-utils", "description": "Common utility functions used throughout other Keyman packages", - "main": "./build/index.js", - "types": "./build/index.d.ts", + "main": "./build/obj/index.js", + "exports": { + ".": { + "types": "./build/obj/index.d.ts", + "import": "./build/obj/index.js" + }, + "./lib": { + "types": "./build/lib/index.d.ts", + "import": "./build/lib/index.mjs", + "require": "./build/lib/index.cjs" + }, + "./obj/*.js": { + "types": "./build/obj/*.js.map", + "import": "./build/obj/*.js" + } + }, "scripts": { "build": "gosh ./build.sh", "clean": "tsc -b --clean", + "mocha": "mocha", "tsc": "tsc" }, "repository": { @@ -21,7 +36,14 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "@keymanapp/keyman-version": "*", + "chai": "^4.3.4", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "@types/node": "^14.0.5", "typescript": "^4.9.5" + }, + "type": "module", + "paths": { + "@keymanapp/keyman-version": "*" } } diff --git a/common/web/utils/src/deepCopy.ts b/common/web/utils/src/deepCopy.ts index ce344391c4b4c37df77e7ad88e2fefccf971adcd..ce05270365ccad985dfae891f2cfd60e92005d31 100644 --- a/common/web/utils/src/deepCopy.ts +++ b/common/web/utils/src/deepCopy.ts @@ -1,25 +1,23 @@ -namespace com.keyman.utils { - /** - * Function deepCopy - * Scope Private - * @param {Object} p object to copy - * @param {Array=} c0 array member being copied - * @return {Object} clone ('deep copy') of object - * Description Makes an actual copy (not a reference) of an object, copying simple members, - * arrays and member objects but not functions, so use with care! - */ - export function deepCopy(p:T, c0?): T { - var c = c0 || {}; - for (var i in p) { - if(typeof p[i] === 'object' && p[i] != null) { - c[i] = (p[i].constructor === Array ) ? [] : {}; - deepCopy(p[i],c[i]); - } - else { - c[i] = p[i]; - } +/** + * Function deepCopy + * Scope Private + * @param {Object} p object to copy + * @param {Array=} c0 array member being copied + * @return {Object} clone ('deep copy') of object + * Description Makes an actual copy (not a reference) of an object, copying simple members, + * arrays and member objects but not functions, so use with care! + */ +export default function deepCopy(p:T, c0?): T { + var c = c0 || {}; + for (var i in p) { + if(typeof p[i] === 'object' && p[i] != null) { + c[i] = (p[i].constructor === Array ) ? [] : {}; + deepCopy(p[i],c[i]); + } + else { + c[i] = p[i]; } - - return c; } + + return c; } \ No newline at end of file diff --git a/common/web/utils/src/deviceSpec.ts b/common/web/utils/src/deviceSpec.ts index 7e2932e471232b9882a6a919e88eade829b5eb01..3ab0352fc468ecb6e51e7a8ff79ac28116db06cd 100644 --- a/common/web/utils/src/deviceSpec.ts +++ b/common/web/utils/src/deviceSpec.ts @@ -1,4 +1,56 @@ -namespace com.keyman.utils { +/** + * This class provides an abstract version of com.keyman.Device that is core-friendly, + * containing only the information needed by web-core for text processing use, devoid + * of any direct references to the DOM. + */ +export class DeviceSpec { + readonly browser: DeviceSpec.Browser; + readonly formFactor: DeviceSpec.FormFactor; + readonly OS: DeviceSpec.OperatingSystem; + readonly touchable: boolean; + + constructor(browser: string, formFactor: string, OS: string, touchable: boolean) { + switch(browser.toLowerCase() as DeviceSpec.Browser) { + case DeviceSpec.Browser.Chrome: + case DeviceSpec.Browser.Edge: + case DeviceSpec.Browser.Firefox: + case DeviceSpec.Browser.Native: + case DeviceSpec.Browser.Opera: + case DeviceSpec.Browser.Safari: + this.browser = browser.toLowerCase() as DeviceSpec.Browser; + break; + default: + this.browser = DeviceSpec.Browser.Other; + } + + switch(formFactor.toLowerCase() as DeviceSpec.FormFactor) { + case DeviceSpec.FormFactor.Desktop: + case DeviceSpec.FormFactor.Phone: + case DeviceSpec.FormFactor.Tablet: + this.formFactor = formFactor.toLowerCase() as DeviceSpec.FormFactor; + break; + default: + throw ("Invalid form factor specified for device: " + formFactor); + } + + switch(OS.toLowerCase() as DeviceSpec.OperatingSystem) { + case DeviceSpec.OperatingSystem.Windows.toLowerCase(): + case DeviceSpec.OperatingSystem.macOS.toLowerCase(): + case DeviceSpec.OperatingSystem.Linux.toLowerCase(): + case DeviceSpec.OperatingSystem.Android.toLowerCase(): + case DeviceSpec.OperatingSystem.iOS.toLowerCase(): + this.OS = OS.toLowerCase() as DeviceSpec.OperatingSystem; + break; + default: + this.OS = DeviceSpec.OperatingSystem.Other; + } + + this.touchable = touchable; + } +} + +// Namespaces these under DeviceSpec, as each is primarily used with it. +export namespace DeviceSpec { export enum Browser { Chrome = 'chrome', Edge = 'edge', @@ -23,55 +75,10 @@ namespace com.keyman.utils { Phone = 'phone', Tablet = 'tablet' } +} - /** - * This class provides an abstract version of com.keyman.Device that is core-friendly, - * containing only the information needed by web-core for text processing use, devoid - * of any direct references to the DOM. - */ - export class DeviceSpec { - readonly browser: Browser; - readonly formFactor: FormFactor; - readonly OS: OperatingSystem; - readonly touchable: boolean; - - constructor(browser: string, formFactor: string, OS: string, touchable: boolean) { - switch(browser.toLowerCase() as Browser) { - case Browser.Chrome: - case Browser.Edge: - case Browser.Firefox: - case Browser.Native: - case Browser.Opera: - case Browser.Safari: - this.browser = browser.toLowerCase() as Browser; - break; - default: - this.browser = Browser.Other; - } +export function physicalKeyDeviceAlias(device: DeviceSpec) { + return new DeviceSpec(device.browser, DeviceSpec.FormFactor.Desktop, device.OS, false); +} - switch(formFactor.toLowerCase() as FormFactor) { - case FormFactor.Desktop: - case FormFactor.Phone: - case FormFactor.Tablet: - this.formFactor = formFactor.toLowerCase() as FormFactor; - break; - default: - throw ("Invalid form factor specified for device: " + formFactor); - } - - switch(OS.toLowerCase() as OperatingSystem) { - case OperatingSystem.Windows.toLowerCase(): - case OperatingSystem.macOS.toLowerCase(): - case OperatingSystem.Linux.toLowerCase(): - case OperatingSystem.Android.toLowerCase(): - case OperatingSystem.iOS.toLowerCase(): - this.OS = OS.toLowerCase() as OperatingSystem; - break; - default: - this.OS = OperatingSystem.Other; - } - - this.touchable = touchable; - } - } -} \ No newline at end of file +export default DeviceSpec; \ No newline at end of file diff --git a/common/web/utils/src/globalObject.ts b/common/web/utils/src/globalObject.ts index 08eb08df6069d04bc77c4608bcd610724eecc8aa..b26868721696d332bf720df6dc2fcc80bde1aaa4 100644 --- a/common/web/utils/src/globalObject.ts +++ b/common/web/utils/src/globalObject.ts @@ -1,32 +1,30 @@ -namespace com.keyman.utils { - /** - * Returns the base global object available to the current JS platform. - * - In browsers, returns `window`. - * - In WebWorkers, returns `self`. - * - In Node, returns `global`. - */ - export function getGlobalObject(): typeof globalThis { - // Evergreen browsers have started defining 'globalThis'. - // Refer to https://devblogs.microsoft.com/typescript/announcing-typescript-3-4/#type-checking-for-globalthis - // and its referenced polyfill. Said polyfill is very complex, so we opt for this far leaner variant. - if(typeof globalThis != 'undefined') { - return globalThis; // Not available in IE or older Edge versions - // @ts-ignore (TS will throw errors for whatever platform we're not compiling for.) - } else if(typeof window != 'undefined') { - // @ts-ignore - return window; // The browser-based classic - // @ts-ignore - } else if(typeof self != 'undefined') { - // @ts-ignore - return self; // WebWorker global - } else { - // Assumption - if neither of the above exist, we're in Node, for unit-testing. - // Node doesn't have as many methods and properties as the other two, but what - // matters for us is that it's the base global. - // - // Some other headless JS solutions use 'this' instead, but Node's enough for our needs. - // @ts-ignore - return (global as any) as typeof globalThis; - } +/** + * Returns the base global object available to the current JS platform. + * - In browsers, returns `window`. + * - In WebWorkers, returns `self`. + * - In Node, returns `global`. + */ +export default function getGlobalObject(): typeof globalThis { + // Evergreen browsers have started defining 'globalThis'. + // Refer to https://devblogs.microsoft.com/typescript/announcing-typescript-3-4/#type-checking-for-globalthis + // and its referenced polyfill. Said polyfill is very complex, so we opt for this far leaner variant. + if(typeof globalThis != 'undefined') { + return globalThis; // Not available in IE or older Edge versions + // @ts-ignore (TS will throw errors for whatever platform we're not compiling for.) + } else if(typeof window != 'undefined') { + // @ts-ignore + return window; // The browser-based classic + // @ts-ignore + } else if(typeof self != 'undefined') { + // @ts-ignore + return self; // WebWorker global + } else { + // Assumption - if neither of the above exist, we're in Node, for unit-testing. + // Node doesn't have as many methods and properties as the other two, but what + // matters for us is that it's the base global. + // + // Some other headless JS solutions use 'this' instead, but Node's enough for our needs. + // @ts-ignore + return (global as any) as typeof globalThis; } } \ No newline at end of file diff --git a/common/web/utils/src/index.ts b/common/web/utils/src/index.ts index 8d7a2e7773ed44839e31701c4856b33171cb3ba9..dc3f868973aa089ddd51bccad60d31fd7233b6a8 100644 --- a/common/web/utils/src/index.ts +++ b/common/web/utils/src/index.ts @@ -1,6 +1,27 @@ // References all utility includes from a single file, making import/export simple. -/// -/// -/// -/// -/// \ No newline at end of file + +export { default as deepCopy } from "./deepCopy.js"; + +export { default as DeviceSpec, physicalKeyDeviceAlias } from "./deviceSpec.js"; + +/* + // An example valid use, post-import: + let testSpec = new DeviceSpec(DeviceSpec.Browser.Chrome, + DeviceSpec.FormFactor.Tablet, + DeviceSpec.OperatingSystem.Android, + true); + */ + +export { default as Version } from "./version.js"; + +export { default as globalObject } from "./globalObject.js"; + +export { default as extendString } from "./kmwstring.js"; + + + +export { default as ManagedPromise } from "./managedPromise.js"; + +// // Uncomment the following line and run the bundled output to verify successful +// // esbuild bundling of this submodule: +// console.log(Version.CURRENT.toString()); \ No newline at end of file diff --git a/common/web/utils/src/kmwstring.ts b/common/web/utils/src/kmwstring.ts index 397b9041a5434f8ba1cdd01989231f8b75f32e18..21d165e8ff778c2044f9349aa8877527c06aeb0c 100644 --- a/common/web/utils/src/kmwstring.ts +++ b/common/web/utils/src/kmwstring.ts @@ -3,468 +3,479 @@ Copyright 2020 SIL International ***/ + /* * TODO: Remove this file as part of addressing https://github.com/keymanapp/keyman/issues/2492. */ -interface StringConstructor { - kmwFromCharCode(cp0: number): string, - _kmwFromCharCode(cp0: number): string, - kmwEnableSupplementaryPlane(bEnable: boolean) -} +declare global { + interface StringConstructor { + kmwFromCharCode(cp0: number): string, + _kmwFromCharCode(cp0: number): string, + kmwEnableSupplementaryPlane(bEnable: boolean) + } -interface String { - kmwCharCodeAt(codePointIndex: number): number, - kmwCharAt(codePointIndex: number) : string, - kmwIndexOf(searchValue: string, fromIndex?: number) : number, - kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, - kmwSlice(beginSlice: number, endSlice: number) : string, - kmwSubstring(start: number, length: number) : string, - kmwSubstr(start: number, length?: number) : string, - kmwBMPSubstr(start: number, length?: number) : string, - kmwLength(): number, - kmwBMPLength(): number, - kmwNextChar(codeUnitIndex: number): number, - kmwBMPNextChar(codeUnitIndex: number): number, - kmwPrevChar(codeUnitIndex: number): number, - kmwBMPPrevChar(codeUnitIndex: number): number, - kmwCodePointToCodeUnit(codePointIndex: number) : number, - kmwBMPCodePointToCodeUnit(codePointIndex: number) : number, - kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, - kmwBMPCodeUnitToCodePoint(codeUnitIndex: number) : number, - _kmwCharCodeAt(codePointIndex: number): number, - _kmwCharAt(codePointIndex: number) : string, - _kmwIndexOf(searchValue: string, fromIndex?: number) : number, - _kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, - _kmwSlice(beginSlice: number, endSlice: number) : string, - _kmwSubstring(start: number, length?: number) : string, - _kmwSubstr(start: number, length?: number) : string, - _kmwLength(): number, - _kmwNextChar(codeUnitIndex: number): number, - _kmwPrevChar(codeUnitIndex: number): number, - _kmwCodePointToCodeUnit(codePointIndex: number) : number, - _kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, + interface String { + kmwCharCodeAt(codePointIndex: number): number, + kmwCharAt(codePointIndex: number) : string, + kmwIndexOf(searchValue: string, fromIndex?: number) : number, + kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, + kmwSlice(beginSlice: number, endSlice: number) : string, + kmwSubstring(start: number, length: number) : string, + kmwSubstr(start: number, length?: number) : string, + kmwBMPSubstr(start: number, length?: number) : string, + kmwLength(): number, + kmwBMPLength(): number, + kmwNextChar(codeUnitIndex: number): number, + kmwBMPNextChar(codeUnitIndex: number): number, + kmwPrevChar(codeUnitIndex: number): number, + kmwBMPPrevChar(codeUnitIndex: number): number, + kmwCodePointToCodeUnit(codePointIndex: number) : number, + kmwBMPCodePointToCodeUnit(codePointIndex: number) : number, + kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, + kmwBMPCodeUnitToCodePoint(codeUnitIndex: number) : number, + _kmwCharCodeAt(codePointIndex: number): number, + _kmwCharAt(codePointIndex: number) : string, + _kmwIndexOf(searchValue: string, fromIndex?: number) : number, + _kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, + _kmwSlice(beginSlice: number, endSlice: number) : string, + _kmwSubstring(start: number, length?: number) : string, + _kmwSubstr(start: number, length?: number) : string, + _kmwLength(): number, + _kmwNextChar(codeUnitIndex: number): number, + _kmwPrevChar(codeUnitIndex: number): number, + _kmwCodePointToCodeUnit(codePointIndex: number) : number, + _kmwCodeUnitToCodePoint(codeUnitIndex: number) : number + } } -/** - * Constructs a string from one or more Unicode character codepoint values - * passed as integer parameters. - * - * @param {number} cp0,... 1 or more Unicode codepoints, e.g. 0x0065, 0x10000 - * @return {string|null} The new String object. - */ -String.kmwFromCharCode = function(cp0) { - var chars = [], i; - for (i = 0; i < arguments.length; i++) { - var c = Number(arguments[i]); - if (!isFinite(c) || c < 0 || c > 0x10FFFF || Math.floor(c) !== c) { - throw new RangeError("Invalid code point " + c); - } - if (c < 0x10000) { - chars.push(c); - } else { - c -= 0x10000; - chars.push((c >> 10) + 0xD800); - chars.push((c % 0x400) + 0xDC00); +export default function extendString() { + /** + * Constructs a string from one or more Unicode character codepoint values + * passed as integer parameters. + * + * @param {number} cp0,... 1 or more Unicode codepoints, e.g. 0x0065, 0x10000 + * @return {string|null} The new String object. + */ + String.kmwFromCharCode = function(cp0) { + var chars = [], i; + for (i = 0; i < arguments.length; i++) { + var c = Number(arguments[i]); + if (!isFinite(c) || c < 0 || c > 0x10FFFF || Math.floor(c) !== c) { + throw new RangeError("Invalid code point " + c); + } + if (c < 0x10000) { + chars.push(c); + } else { + c -= 0x10000; + chars.push((c >> 10) + 0xD800); + chars.push((c % 0x400) + 0xDC00); + } } + return String.fromCharCode.apply(undefined, chars); } - return String.fromCharCode.apply(undefined, chars); -} -/** - * Returns a number indicating the Unicode value of the character at the given - * code point index, with support for supplementary plane characters. - * - * @param {number} codePointIndex The code point index into the string (not - the code unit index) to return - * @return {number} The Unicode character value - */ -String.prototype.kmwCharCodeAt = function(codePointIndex) { - var str = String(this); - var codeUnitIndex = 0; - - if (codePointIndex < 0 || codePointIndex >= str.length) { - return NaN; - } + /** + * Returns a number indicating the Unicode value of the character at the given + * code point index, with support for supplementary plane characters. + * + * @param {number} codePointIndex The code point index into the string (not + the code unit index) to return + * @return {number} The Unicode character value + */ + String.prototype.kmwCharCodeAt = function(codePointIndex) { + var str = String(this); + var codeUnitIndex = 0; - for(var i = 0; i < codePointIndex; i++) { - codeUnitIndex = str.kmwNextChar(codeUnitIndex); - if(codeUnitIndex === null) return NaN; - } - - var first = str.charCodeAt(codeUnitIndex); - if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - var second = str.charCodeAt(codeUnitIndex + 1); - if (second >= 0xDC00 && second <= 0xDFFF) { - return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; + if (codePointIndex < 0 || codePointIndex >= str.length) { + return NaN; } + + for(var i = 0; i < codePointIndex; i++) { + codeUnitIndex = str.kmwNextChar(codeUnitIndex); + if(codeUnitIndex === null) return NaN; + } + + var first = str.charCodeAt(codeUnitIndex); + if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + var second = str.charCodeAt(codeUnitIndex + 1); + if (second >= 0xDC00 && second <= 0xDFFF) { + return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; + } + } + return first; } - return first; -} -/** - * Returns the code point index within the calling String object of the first occurrence - * of the specified value, or -1 if not found. - * - * @param {string} searchValue The value to search for - * @param {number} [fromIndex] Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ -String.prototype.kmwIndexOf = function(searchValue, fromIndex) { - var str = String(this); - var codeUnitIndex = str.indexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; + /** + * Returns the code point index within the calling String object of the first occurrence + * of the specified value, or -1 if not found. + * + * @param {string} searchValue The value to search for + * @param {number} [fromIndex] Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ + String.prototype.kmwIndexOf = function(searchValue, fromIndex) { + var str = String(this); + var codeUnitIndex = str.indexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + var codePointIndex = 0; + for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; + return codePointIndex; } - - var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; - return codePointIndex; -} -/** - * Returns the code point index within the calling String object of the last occurrence - * of the specified value, or -1 if not found. - * - * @param {string} searchValue The value to search for - * @param {number} fromIndex Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ -String.prototype.kmwLastIndexOf = function(searchValue, fromIndex) -{ - var str = String(this); - var codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; + /** + * Returns the code point index within the calling String object of the last occurrence + * of the specified value, or -1 if not found. + * + * @param {string} searchValue The value to search for + * @param {number} fromIndex Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ + String.prototype.kmwLastIndexOf = function(searchValue, fromIndex) + { + var str = String(this); + var codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + var codePointIndex = 0; + for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; + return codePointIndex; } - - var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; - return codePointIndex; -} -/** - * Returns the length of the string in code points, as opposed to code units. - * - * @return {number} The length of the string in code points - */ -String.prototype.kmwLength = function() { - var str = String(this); - - if(str.length == 0) return 0; - - for(var i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) - codeUnitIndex = str.kmwNextChar(codeUnitIndex); - return i; -} + /** + * Returns the length of the string in code points, as opposed to code units. + * + * @return {number} The length of the string in code points + */ + String.prototype.kmwLength = function() { + var str = String(this); -/** - * Extracts a section of a string and returns a new string. - * - * @param {number} beginSlice The start code point index in the string to - * extract from - * @param {number} endSlice Optional end code point index in the string - * to extract to - * @return {string} The substring as selected by beginSlice and - * endSlice - */ -String.prototype.kmwSlice = function(beginSlice, endSlice) { - var str = String(this); - var beginSliceCodeUnit = str.kmwCodePointToCodeUnit(beginSlice); - var endSliceCodeUnit = str.kmwCodePointToCodeUnit(endSlice); - if(beginSliceCodeUnit === null || endSliceCodeUnit === null) - return ''; - else - return str.slice(beginSliceCodeUnit, endSliceCodeUnit); -} + if(str.length == 0) return 0; -/** - * Returns the characters in a string beginning at the specified location through - * the specified number of characters. - * - * @param {number} start The start code point index in the string to - * extract from - * @param {number=} length Optional length to extract - * @return {string} The substring as selected by start and length - */ -String.prototype.kmwSubstr = function(start, length?) -{ - var str = String(this); - if(start < 0) - { - start = str.kmwLength() + start; + for(var i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) + codeUnitIndex = str.kmwNextChar(codeUnitIndex); + return i; } - if(start < 0) start = 0; - var startCodeUnit = str.kmwCodePointToCodeUnit(start); - var endCodeUnit = startCodeUnit; - - if(startCodeUnit === null) return ''; - - if(arguments.length < 2) { - endCodeUnit = str.length; - } else { - for(var i = 0; i < length; i++) endCodeUnit = str.kmwNextChar(endCodeUnit); + + /** + * Extracts a section of a string and returns a new string. + * + * @param {number} beginSlice The start code point index in the string to + * extract from + * @param {number} endSlice Optional end code point index in the string + * to extract to + * @return {string} The substring as selected by beginSlice and + * endSlice + */ + String.prototype.kmwSlice = function(beginSlice, endSlice) { + var str = String(this); + var beginSliceCodeUnit = str.kmwCodePointToCodeUnit(beginSlice); + var endSliceCodeUnit = str.kmwCodePointToCodeUnit(endSlice); + if(beginSliceCodeUnit === null || endSliceCodeUnit === null) + return ''; + else + return str.slice(beginSliceCodeUnit, endSliceCodeUnit); } - if(endCodeUnit === null) - return str.substring(startCodeUnit); - else - return str.substring(startCodeUnit, endCodeUnit); -} -/** - * Returns the characters in a string between two indexes into the string. - * - * @param {number} indexA The start code point index in the string to - * extract from - * @param {number} indexB The end code point index in the string to - * extract to - * @return {string} The substring as selected by indexA and indexB - */ -String.prototype.kmwSubstring = function(indexA, indexB) -{ - var str = String(this),indexACodeUnit,indexBCodeUnit; - - if(typeof(indexB) == 'undefined') + /** + * Returns the characters in a string beginning at the specified location through + * the specified number of characters. + * + * @param {number} start The start code point index in the string to + * extract from + * @param {number=} length Optional length to extract + * @return {string} The substring as selected by start and length + */ + String.prototype.kmwSubstr = function(start, length?) { - indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); - indexBCodeUnit = str.length; - } - else - { - if(indexA > indexB) { var c = indexA; indexA = indexB; indexB = c; } - - indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); - indexBCodeUnit = str.kmwCodePointToCodeUnit(indexB); + var str = String(this); + if(start < 0) + { + start = str.kmwLength() + start; + } + if(start < 0) start = 0; + var startCodeUnit = str.kmwCodePointToCodeUnit(start); + var endCodeUnit = startCodeUnit; + + if(startCodeUnit === null) return ''; + + if(arguments.length < 2) { + endCodeUnit = str.length; + } else { + for(var i = 0; i < length; i++) endCodeUnit = str.kmwNextChar(endCodeUnit); + } + if(endCodeUnit === null) + return str.substring(startCodeUnit); + else + return str.substring(startCodeUnit, endCodeUnit); } - if(isNaN(indexACodeUnit) || indexACodeUnit === null) indexACodeUnit = 0; - if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) indexBCodeUnit = str.length; - return str.substring(indexACodeUnit, indexBCodeUnit); -} + /** + * Returns the characters in a string between two indexes into the string. + * + * @param {number} indexA The start code point index in the string to + * extract from + * @param {number} indexB The end code point index in the string to + * extract to + * @return {string} The substring as selected by indexA and indexB + */ + String.prototype.kmwSubstring = function(indexA, indexB) + { + var str = String(this),indexACodeUnit,indexBCodeUnit; -/* - Helper functions -*/ - -/** - * Returns the code unit index for the next code point in the string, accounting for - * supplementary pairs - * - * @param {number|null} codeUnitIndex The code unit position to increment - * @return {number|null} The index of the next code point in the string, - * in code units - */ -String.prototype.kmwNextChar = function(codeUnitIndex) { - var str = String(this); + if(typeof(indexB) == 'undefined') + { + indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); + indexBCodeUnit = str.length; + } + else + { + if(indexA > indexB) { var c = indexA; indexA = indexB; indexB = c; } + + indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); + indexBCodeUnit = str.kmwCodePointToCodeUnit(indexB); + } + if(isNaN(indexACodeUnit) || indexACodeUnit === null) indexACodeUnit = 0; + if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) indexBCodeUnit = str.length; - if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { - return null; + return str.substring(indexACodeUnit, indexBCodeUnit); } - - var first = str.charCodeAt(codeUnitIndex); - if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - var second = str.charCodeAt(codeUnitIndex + 1); - if (second >= 0xDC00 && second <= 0xDFFF) { - if(codeUnitIndex == str.length - 2) { - return null; + + /* + Helper functions + */ + + /** + * Returns the code unit index for the next code point in the string, accounting for + * supplementary pairs + * + * @param {number|null} codeUnitIndex The code unit position to increment + * @return {number|null} The index of the next code point in the string, + * in code units + */ + String.prototype.kmwNextChar = function(codeUnitIndex) { + var str = String(this); + + if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { + return null; + } + + var first = str.charCodeAt(codeUnitIndex); + if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + var second = str.charCodeAt(codeUnitIndex + 1); + if (second >= 0xDC00 && second <= 0xDFFF) { + if(codeUnitIndex == str.length - 2) { + return null; + } + return codeUnitIndex + 2; } - return codeUnitIndex + 2; } + return codeUnitIndex + 1; } - return codeUnitIndex + 1; -} -/** - * Returns the code unit index for the previous code point in the string, accounting - * for supplementary pairs - * - * @param {number|null} codeUnitIndex The code unit position to decrement - * @return {number|null} The index of the previous code point in the - * string, in code units -*/ -String.prototype.kmwPrevChar = function(codeUnitIndex) { - var str = String(this); - - if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { - return null; - } - - var second = str.charCodeAt(codeUnitIndex - 1); - if (second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { - var first = str.charCodeAt(codeUnitIndex - 2); - if(first >= 0xD800 && first <= 0xDBFF) { - return codeUnitIndex - 2; + /** + * Returns the code unit index for the previous code point in the string, accounting + * for supplementary pairs + * + * @param {number|null} codeUnitIndex The code unit position to decrement + * @return {number|null} The index of the previous code point in the + * string, in code units + */ + String.prototype.kmwPrevChar = function(codeUnitIndex) { + var str = String(this); + + if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { + return null; + } + + var second = str.charCodeAt(codeUnitIndex - 1); + if (second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { + var first = str.charCodeAt(codeUnitIndex - 2); + if(first >= 0xD800 && first <= 0xDBFF) { + return codeUnitIndex - 2; + } } + return codeUnitIndex - 1; } - return codeUnitIndex - 1; -} -/** - * Returns the corresponding code unit index to the code point index passed - * - * @param {number|null} codePointIndex A code point index in the string - * @return {number|null} The corresponding code unit index - */ -String.prototype.kmwCodePointToCodeUnit = function(codePointIndex) { - - if(codePointIndex === null) return null; - - var str = String(this); - var codeUnitIndex = 0; - - if(codePointIndex < 0) { - codeUnitIndex = str.length; - for(var i = 0; i > codePointIndex; i--) - codeUnitIndex = str.kmwPrevChar(codeUnitIndex); + /** + * Returns the corresponding code unit index to the code point index passed + * + * @param {number|null} codePointIndex A code point index in the string + * @return {number|null} The corresponding code unit index + */ + String.prototype.kmwCodePointToCodeUnit = function(codePointIndex) { + + if(codePointIndex === null) return null; + + var str = String(this); + var codeUnitIndex = 0; + + if(codePointIndex < 0) { + codeUnitIndex = str.length; + for(var i = 0; i > codePointIndex; i--) + codeUnitIndex = str.kmwPrevChar(codeUnitIndex); + return codeUnitIndex; + } + + if(codePointIndex == str.kmwLength()) return str.length; + + for(var i = 0; i < codePointIndex; i++) + codeUnitIndex = str.kmwNextChar(codeUnitIndex); return codeUnitIndex; } - - if(codePointIndex == str.kmwLength()) return str.length; - - for(var i = 0; i < codePointIndex; i++) - codeUnitIndex = str.kmwNextChar(codeUnitIndex); - return codeUnitIndex; -} -/** - * Returns the corresponding code point index to the code unit index passed - * - * @param {number|null} codeUnitIndex A code unit index in the string - * @return {number|null} The corresponding code point index - */ -String.prototype.kmwCodeUnitToCodePoint = function(codeUnitIndex) { - var str = String(this); - - if(codeUnitIndex === null) - return null; - else if(codeUnitIndex == 0) - return 0; - else if(codeUnitIndex < 0) - return str.substr(codeUnitIndex).kmwLength(); - else - return str.substr(0,codeUnitIndex).kmwLength(); -} + /** + * Returns the corresponding code point index to the code unit index passed + * + * @param {number|null} codeUnitIndex A code unit index in the string + * @return {number|null} The corresponding code point index + */ + String.prototype.kmwCodeUnitToCodePoint = function(codeUnitIndex) { + var str = String(this); -/** - * Returns the character at a the code point index passed - * - * @param {number} codePointIndex A code point index in the string - * @return {string} The corresponding character - */ -String.prototype.kmwCharAt = function(codePointIndex) { - var str = String(this); - - if(codePointIndex >= 0) return str.kmwSubstr(codePointIndex,1); else return ''; -} + if(codeUnitIndex === null) + return null; + else if(codeUnitIndex == 0) + return 0; + else if(codeUnitIndex < 0) + return str.substr(codeUnitIndex).kmwLength(); + else + return str.substr(0,codeUnitIndex).kmwLength(); + } -/** - * String prototype library extensions for basic plane characters, - * to simplify enabling or disabling supplementary plane functionality (I3319) - */ - -/** - * Returns the code unit index for the next code point in the string - * - * @param {number} codeUnitIndex A code point index in the string - * @return {number|null} The corresponding character - */ -String.prototype.kmwBMPNextChar = function(codeUnitIndex) -{ - var str = String(this); - if(codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { - return null; + /** + * Returns the character at a the code point index passed + * + * @param {number} codePointIndex A code point index in the string + * @return {string} The corresponding character + */ + String.prototype.kmwCharAt = function(codePointIndex) { + var str = String(this); + + if(codePointIndex >= 0) return str.kmwSubstr(codePointIndex,1); else return ''; } - return codeUnitIndex + 1; -} - -/** - * Returns the code unit index for the previous code point in the string - * - * @param {number} codeUnitIndex A code unit index in the string - * @return {number|null} The corresponding character - */ -String.prototype.kmwBMPPrevChar = function(codeUnitIndex) -{ - var str = String(this); - - if(codeUnitIndex <= 0 || codeUnitIndex > str.length) { - return null; + + /** + * String prototype library extensions for basic plane characters, + * to simplify enabling or disabling supplementary plane functionality (I3319) + */ + + /** + * Returns the code unit index for the next code point in the string + * + * @param {number} codeUnitIndex A code point index in the string + * @return {number|null} The corresponding character + */ + String.prototype.kmwBMPNextChar = function(codeUnitIndex) + { + var str = String(this); + if(codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { + return null; + } + return codeUnitIndex + 1; } - return codeUnitIndex - 1; -} - -/** - * Returns the code unit index for a code point index - * - * @param {number} codePointIndex A code point index in the string - * @return {number} The corresponding character - */ -String.prototype.kmwBMPCodePointToCodeUnit = function(codePointIndex) -{ - return codePointIndex; -} -/** - * Returns the code point index for a code unit index - * - * @param {number} codeUnitIndex A code point index in the string - * @return {number} The corresponding character - */ -String.prototype.kmwBMPCodeUnitToCodePoint = function(codeUnitIndex) -{ - return codeUnitIndex; -} + /** + * Returns the code unit index for the previous code point in the string + * + * @param {number} codeUnitIndex A code unit index in the string + * @return {number|null} The corresponding character + */ + String.prototype.kmwBMPPrevChar = function(codeUnitIndex) + { + var str = String(this); -/** - * Returns the length of a BMP string - * - * @return {number} The length in code points - */ -String.prototype.kmwBMPLength = function() -{ - var str = String(this); - return str.length; -} + if(codeUnitIndex <= 0 || codeUnitIndex > str.length) { + return null; + } + return codeUnitIndex - 1; + } + /** + * Returns the code unit index for a code point index + * + * @param {number} codePointIndex A code point index in the string + * @return {number} The corresponding character + */ + String.prototype.kmwBMPCodePointToCodeUnit = function(codePointIndex) + { + return codePointIndex; + } -/** - * Returns a substring - * - * @param {number} n - * @param {number=} ln - * @return {string} - */ -String.prototype.kmwBMPSubstr = function(n,ln?) -{ - var str=String(this); - if(n > -1) - return str.substr(n,ln); - else - return str.substr(str.length+n,-n); -} + /** + * Returns the code point index for a code unit index + * + * @param {number} codeUnitIndex A code point index in the string + * @return {number} The corresponding character + */ + String.prototype.kmwBMPCodeUnitToCodePoint = function(codeUnitIndex) + { + return codeUnitIndex; + } -/** - * Enable or disable supplementary plane string handling - * - * @param {boolean} bEnable - */ -String.kmwEnableSupplementaryPlane = function(bEnable) -{ - var p=String.prototype; - String._kmwFromCharCode = bEnable ? String.kmwFromCharCode : String.fromCharCode; - p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; - p._kmwCharCodeAt = bEnable ? p.kmwCharCodeAt : p.charCodeAt; - p._kmwIndexOf = bEnable ? p.kmwIndexOf :p.indexOf; - p._kmwLastIndexOf = bEnable ? p.kmwLastIndexOf : p.lastIndexOf ; - p._kmwSlice = bEnable ? p.kmwSlice : p.slice; - p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; - p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; - p._kmwLength = bEnable ? p.kmwLength : p.kmwBMPLength; - p._kmwNextChar = bEnable ? p.kmwNextChar : p.kmwBMPNextChar; - p._kmwPrevChar = bEnable ? p.kmwPrevChar : p.kmwBMPPrevChar; - p._kmwCodePointToCodeUnit = bEnable ? p.kmwCodePointToCodeUnit : p.kmwBMPCodePointToCodeUnit; - p._kmwCodeUnitToCodePoint = bEnable ? p.kmwCodeUnitToCodePoint : p.kmwBMPCodeUnitToCodePoint; + /** + * Returns the length of a BMP string + * + * @return {number} The length in code points + */ + String.prototype.kmwBMPLength = function() + { + var str = String(this); + return str.length; + } + + /** + * Returns a substring + * + * @param {number} n + * @param {number=} ln + * @return {string} + */ + String.prototype.kmwBMPSubstr = function(n,ln?) + { + var str=String(this); + if(n > -1) + return str.substr(n,ln); + else + return str.substr(str.length+n,-n); + } + + /** + * Enable or disable supplementary plane string handling + * + * @param {boolean} bEnable + */ + String.kmwEnableSupplementaryPlane = function(bEnable) + { + var p=String.prototype; + String._kmwFromCharCode = bEnable ? String.kmwFromCharCode : String.fromCharCode; + p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; + p._kmwCharCodeAt = bEnable ? p.kmwCharCodeAt : p.charCodeAt; + p._kmwIndexOf = bEnable ? p.kmwIndexOf :p.indexOf; + p._kmwLastIndexOf = bEnable ? p.kmwLastIndexOf : p.lastIndexOf ; + p._kmwSlice = bEnable ? p.kmwSlice : p.slice; + p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; + p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; + p._kmwLength = bEnable ? p.kmwLength : p.kmwBMPLength; + p._kmwNextChar = bEnable ? p.kmwNextChar : p.kmwBMPNextChar; + p._kmwPrevChar = bEnable ? p.kmwPrevChar : p.kmwBMPPrevChar; + p._kmwCodePointToCodeUnit = bEnable ? p.kmwCodePointToCodeUnit : p.kmwBMPCodePointToCodeUnit; + p._kmwCodeUnitToCodePoint = bEnable ? p.kmwCodeUnitToCodePoint : p.kmwBMPCodeUnitToCodePoint; + } + + // Ensure that _all_ String extensions are established, even if disabled by default. + if(!String._kmwFromCharCode) { + String.kmwEnableSupplementaryPlane(false); + } } +// For side-effect imports: +extendString(); \ No newline at end of file diff --git a/common/web/utils/src/managedPromise.ts b/common/web/utils/src/managedPromise.ts new file mode 100644 index 0000000000000000000000000000000000000000..ac616b9a350f0d527af2981859a623d2f77ea106 --- /dev/null +++ b/common/web/utils/src/managedPromise.ts @@ -0,0 +1,64 @@ +type ResolveSignature = (value: Type | PromiseLike) => void; +type RejectSignature = (reason?: any) => void; + +export default class ManagedPromise { + public resolve: ResolveSignature; + public reject: RejectSignature; + + private _hasResolved: boolean = false; + private _hasRejected: boolean = false; + + public get hasResolved(): boolean { + return this._hasResolved; + } + + public get hasRejected(): boolean { + return this._hasRejected; + } + + public get hasFinalized(): boolean { + return this.hasResolved || this.hasRejected; + } + + private _promise: Promise; + + constructor(); + constructor(executor: (resolve: ResolveSignature, reject: RejectSignature) => Type); + constructor(executor?: (resolve: ResolveSignature, reject: RejectSignature) => Type) { + this._promise = new Promise((resolve, reject) => { + this.resolve = (value) => { + this._hasResolved = true; + resolve(value); + }; + + this.reject = (reason) => { + this._hasRejected = true; + reject(reason); + }; + + if(executor) { + executor(this.resolve, this.reject); + } + }); + } + + // Cannot actually extend the Promise class in ES5; attempt to use it will throw errors. + // So, we just implement a Promise-like interface. + + then(onfulfilled?: ((value: Type) => TResult1 | PromiseLike) | undefined | null, onrejected?: ((reason: any) => TResult2 | PromiseLike) | undefined | null): Promise { + return this._promise.then(onfulfilled, onrejected); + } + + catch(onrejected?: (reason: any) => PromiseLike): Promise { + return this._promise.catch(onrejected); + } + + finally(onfinally?: () => void): Promise { + return this._promise.finally(onfinally); + } + + // And for things that actually need to provide something typed to Promise... well... + get corePromise(): Promise { + return this._promise; + } +} \ No newline at end of file diff --git a/common/web/utils/src/test/managedPromise.js b/common/web/utils/src/test/managedPromise.js new file mode 100644 index 0000000000000000000000000000000000000000..c189303276309625b188ac253d6cc74284c0db95 --- /dev/null +++ b/common/web/utils/src/test/managedPromise.js @@ -0,0 +1,49 @@ +import { assert } from 'chai'; + +import ManagedPromise from '../../build/obj/managedPromise.js'; + +describe("ManagedPromise", () => { + it(' resolution', async () => { + let promise = new ManagedPromise(); + + promise.resolve(); + + await promise.corePromise; + }); + + it(' resolution', async () => { + let promise = new ManagedPromise(); + + promise.resolve("foobar"); + + let text = await promise.corePromise; + + assert.equal(text, "foobar"); + }); + + it('rejection', async () => { + let promise = new ManagedPromise(); + + promise.reject(new Error("foobar")); + + try { + await promise.corePromise; + assert.fail(); + } catch (err) { + assert.equal(err.message, "foobar"); + } + }); + + it('then()', async () => { + let promise = new ManagedPromise(); + + let thenPromise = promise.then((val) => { + return {text: val + "bar"} + }); + + promise.resolve("foo"); + let val = await thenPromise; + + assert.deepEqual(val, {text: "foobar"}); + }); +}); \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/cases/versions.js b/common/web/utils/src/test/versions.js similarity index 59% rename from common/web/keyboard-processor/tests/cases/versions.js rename to common/web/utils/src/test/versions.js index 2e7f6ae142cd1c4c6fb6a007e4928567d97e511e..e6cf422dc4ac0bd0e1b421d9468450295afaebcd 100644 --- a/common/web/keyboard-processor/tests/cases/versions.js +++ b/common/web/utils/src/test/versions.js @@ -1,33 +1,31 @@ -var assert = require('chai').assert; -let KeyboardProcessor = require('../../build/index.bundled.js'); +import { assert } from 'chai'; -// Required initialization setup. -global.com = KeyboardProcessor.com; // exports all keyboard-processor namespacing. +import Version from '../../build/obj/version.js'; describe('Version Logic', function() { it('Should provide a default, fallback value when nothing is specified', function() { - var fallback = new com.keyman.utils.Version(undefined); - assert.isTrue(fallback.equals(com.keyman.utils.Version.DEVELOPER_VERSION_FALLBACK)); + var fallback = new Version(undefined); + assert.isTrue(fallback.equals(Version.DEVELOPER_VERSION_FALLBACK)); }); it('Should properly process a simple major.minor version string.', function() { - var version = new com.keyman.utils.Version("1.2"); + var version = new Version("1.2"); assert.equal(version.major, 1); assert.equal(version.minor, 2); }); it('Should handle long/deep version specifications.', function() { - var version = new com.keyman.utils.Version("1.2.3.4.5.6"); + var version = new Version("1.2.3.4.5.6"); assert.equal(version.components.length, 6); assert.equal(version.major, 1); assert.equal(version.minor, 2); }); it('Should properly compare two versions.', function() { - var v9_0_1 = new com.keyman.utils.Version("9.0.1"); - var v9_1_0 = new com.keyman.utils.Version("9.1.0"); - var v10_0 = new com.keyman.utils.Version("10.0"); - var v10_0_0 = new com.keyman.utils.Version("10.0.0"); + var v9_0_1 = new Version("9.0.1"); + var v9_1_0 = new Version("9.1.0"); + var v10_0 = new Version("10.0"); + var v10_0_0 = new Version("10.0.0"); // "Precede" checks assert.equal(v9_0_1.compareTo(v9_1_0), -1); diff --git a/common/web/utils/src/version.ts b/common/web/utils/src/version.ts index ab392938c1349ce7b2819b79c9abfafcfab60825..27e865ee1e8a66ccc3a14dc034d5b2c0becb62d8 100644 --- a/common/web/utils/src/version.ts +++ b/common/web/utils/src/version.ts @@ -1,112 +1,109 @@ -// Ensure that this class contains no reference into core KMW code - it is referenced -// by components intended to be modular and possible to separate from core KMW. - -namespace com.keyman.utils { - // Dotted-decimal version - export class Version { - public static readonly CURRENT = new Version(com.keyman.KEYMAN_VERSION.VERSION_RELEASE); - - // Represents a default version value for keyboards compiled before this was compiled into keyboards. - // The exact version is unknown at this point, but the value is "good enough" for what we need. - public static readonly DEVELOPER_VERSION_FALLBACK = new Version([9, 0, 0]); - - // For 12.0, the old default behavior of adding missing keycaps to the default layers was removed, - // as it results in unexpected, bug-like behavior for keyboard designers when it is unwanted. - public static readonly NO_DEFAULT_KEYCAPS = new Version([12, 0]); - - public static readonly MAC_POSSIBLE_IPAD_ALIAS = new Version([10, 15]); - - private readonly components: number[] - - /** - * Parses version information, preparing it for use in comparisons. - * @param text Either a string representing a version number (ex: "9.0.0") or an array representing - * its components (ex: [9, 0, 0]). - */ - constructor(text: String | number[]) { - // If a keyboard doesn't specify a version, use the DEVELOPER_VERSION_FALLBACK values. - if(text === undefined || text === null) { - this.components = [].concat(Version.DEVELOPER_VERSION_FALLBACK.components); +import KEYMAN_VERSION from "@keymanapp/keyman-version"; + +// Dotted-decimal version +export default class Version { + public static readonly CURRENT = new Version(KEYMAN_VERSION.VERSION_RELEASE); + + // Represents a default version value for keyboards compiled before this was compiled into keyboards. + // The exact version is unknown at this point, but the value is "good enough" for what we need. + public static readonly DEVELOPER_VERSION_FALLBACK = new Version([9, 0, 0]); + + // For 12.0, the old default behavior of adding missing keycaps to the default layers was removed, + // as it results in unexpected, bug-like behavior for keyboard designers when it is unwanted. + public static readonly NO_DEFAULT_KEYCAPS = new Version([12, 0]); + + public static readonly MAC_POSSIBLE_IPAD_ALIAS = new Version([10, 15]); + + private readonly components: number[] + + /** + * Parses version information, preparing it for use in comparisons. + * @param text Either a string representing a version number (ex: "9.0.0") or an array representing + * its components (ex: [9, 0, 0]). + */ + constructor(text: String | number[]) { + // If a keyboard doesn't specify a version, use the DEVELOPER_VERSION_FALLBACK values. + if(text === undefined || text === null) { + this.components = [].concat(Version.DEVELOPER_VERSION_FALLBACK.components); + return; + } + + if(Array.isArray(text)) { + let components = text as number[]; + if(components.length < 2) { + throw new Error("Version string must have at least a major and minor component!"); + } else { + this.components = [].concat(components); return; } + } - if(Array.isArray(text)) { - let components = text as number[]; - if(components.length < 2) { - throw new Error("Version string must have at least a major and minor component!"); - } else { - this.components = [].concat(components); - return; - } - } + // else, standard constructor path. + let parts = text.split('.'); + let componentArray: number[] = []; - // else, standard constructor path. - let parts = text.split('.'); - let componentArray: number[] = []; + if(parts.length < 2) { + throw new Error("Version string must have at least a major and minor component!"); + } - if(parts.length < 2) { - throw new Error("Version string must have at least a major and minor component!"); + for(let i=0; i < parts.length; i++) { + let value = parseInt(parts[i], 10); + if(isNaN(value)) { + throw new Error("Version string components must be numerical!"); } - for(let i=0; i < parts.length; i++) { - let value = parseInt(parts[i], 10); - if(isNaN(value)) { - throw new Error("Version string components must be numerical!"); - } + componentArray.push(value); + } - componentArray.push(value); - } + this.components = componentArray; + } - this.components = componentArray; - } + get major(): number { + return this.components[0]; + } - get major(): number { - return this.components[0]; - } + get minor(): number { + return this.components[1]; + } - get minor(): number { - return this.components[1]; - } + toString(): string { + return this.components.join('.'); + } - toString(): string { - return this.components.join('.'); - } + toJSON(): string { + return this.toString(); + } - toJSON(): string { - return this.toString(); - } + equals(other: Version): boolean { + return this.compareTo(other) == 0; + } - equals(other: Version): boolean { - return this.compareTo(other) == 0; - } + precedes(other: Version): boolean { + return this.compareTo(other) < 0; + } - precedes(other: Version): boolean { - return this.compareTo(other) < 0; - } + compareTo(other: Version): number { + // If the version info depth differs, we need a flag to indicate which instance is shorter. + var isShorter: boolean = this.components.length < other.components.length; + var maxDepth: number = (this.components.length < other.components.length) ? this.components.length : other.components.length; - compareTo(other: Version): number { - // If the version info depth differs, we need a flag to indicate which instance is shorter. - var isShorter: boolean = this.components.length < other.components.length; - var maxDepth: number = (this.components.length < other.components.length) ? this.components.length : other.components.length; - - var i: number; - for(i = 0; i < maxDepth; i++) { - let delta = this.components[i] - other.components[i]; - if(delta != 0) { - return delta; - } + var i: number; + for(i = 0; i < maxDepth; i++) { + let delta = this.components[i] - other.components[i]; + if(delta != 0) { + return delta; } + } - var longList = isShorter ? other.components : this.components; - do { - if(longList[i] > 0) { - return isShorter ? -1 : 1; - } - i++; - } while (i < longList.length); + var longList = isShorter ? other.components : this.components; + do { + if(longList[i] > 0) { + return isShorter ? -1 : 1; + } + i++; + } while (i < longList.length); - // Equal. - return 0; - } + // Equal. + return 0; } } \ No newline at end of file diff --git a/common/web/utils/tsconfig.json b/common/web/utils/tsconfig.json index 4bdd955c5fd83d1696900a4260a60d9000111e89..e19eadb2d3efe6a14b9648489b578aec941539fa 100644 --- a/common/web/utils/tsconfig.json +++ b/common/web/utils/tsconfig.json @@ -2,19 +2,27 @@ "extends": "../../../tsconfig-base.json", "compilerOptions": { "allowJs": true, - "module": "none", + "module": "es6", + "moduleResolution": "Node16", + "importHelpers": true, "inlineSources": true, "sourceMap": true, "declaration": true, "target": "es5", + "tsBuildInfoFile": "./build/obj/tsconfig.tsbuildinfo", "types": ["node"], "lib": ["es6"], - "outFile": "build/index.js" + "baseUrl": "./", + "outDir": "./build/obj/", + "rootDir": "./src" }, "references": [ { "path": "../keyman-version"} ], "include": [ "src/*.ts" + ], + "exclude": [ + "src/test/**/*.js" ] } diff --git a/common/windows/cpp/include/legacy_kmx_file.h b/common/windows/cpp/include/legacy_kmx_file.h index ae18543312f2303575ac276e1521c7bff9c20eb7..d961250912f78fc5bab8726af0d652f6b6db86e6 100644 --- a/common/windows/cpp/include/legacy_kmx_file.h +++ b/common/windows/cpp/include/legacy_kmx_file.h @@ -409,7 +409,6 @@ typedef COMP_GROUP *PCOMP_GROUP; typedef struct _COMPILER_OPTIONS { DWORD dwSize; BOOL ShouldAddCompilerVersion; - BOOL UseKmcmpLib; } COMPILER_OPTIONS; typedef COMPILER_OPTIONS *PCOMPILER_OPTIONS; diff --git a/core/doc/hotdoc.json b/core/doc/hotdoc.json index 61a0028fb0bd464c8e01ff40a8d4b22a32b41d51..4ece50d0f52c505abb5a21b46c5da2e8199d9227 100644 --- a/core/doc/hotdoc.json +++ b/core/doc/hotdoc.json @@ -1,10 +1,14 @@ { "project_name": "@project_name@", "project_version": "@project_version@", - "sitemap": "../../doc/sitemap.txt", - "index": "../../doc/markdown_files/index.md", + "sitemap": "@doc_dir@/sitemap.txt", + "index": "@doc_dir@/markdown_files/index.md", "c_sources": [ - "../include/keyboardprocessor.h" + "@include_dir@/keyman/keyboardprocessor.h" + ], + "c_include_directories": [ + "@include_dir@", + "@doc_dir@/../../common/include" ], "c_smart_index" : true, "output": ".", diff --git a/core/doc/meson.build b/core/doc/meson.build index 2772819ce7ff8b54cea5774e59f13cbea7802a00..01fcb5c98ad60250fb7e3ac3561c5ac5d9791a22 100644 --- a/core/doc/meson.build +++ b/core/doc/meson.build @@ -12,18 +12,20 @@ if hotdoc.found() cfg = configuration_data() cfg.set('project_name', meson.project_name()) cfg.set('project_version', meson.project_version()) + cfg.set('doc_dir', meson.current_source_dir()) + cfg.set('include_dir', meson.current_source_dir() / '../include') configure_file(input: 'hotdoc.json', output: 'hotdoc.json', configuration: cfg) deps = files( - '../include/keyman/keyboardprocessor.h.in', + '../include/keyman/keyboardprocessor.h', '../src/jsonpp.hpp', '../src/utfcodec.hpp' ) docs = custom_target('docs', output: ['html'], - input: ['sitemap.txt', 'markdown_files/index.md'], + input: ['sitemap.txt', 'markdown_files/index.md', deps], command: [hotdoc, '--verbose', '--conf-file=doc/hotdoc.json', '--output=doc', 'run'], depend_files: deps, install: true, diff --git a/core/include/keyman/keyboardprocessor.h b/core/include/keyman/keyboardprocessor.h index 3513cdede2e88223e47ada46547975b3d3bc856a..429a8ae6cbf9118850a12460db2a289c262920f1 100644 --- a/core/include/keyman/keyboardprocessor.h +++ b/core/include/keyman/keyboardprocessor.h @@ -779,7 +779,7 @@ km_kbp_keyboard_get_key_list(km_kbp_keyboard const *keyboard, km_kbp_keyboard_key **out); -/* +/** ``` ### `km_kbp_keyboard_key_list_dispose` ##### Description: @@ -794,28 +794,41 @@ returned by `km_kbp_keyboard_get_key_list`. KMN_API void km_kbp_keyboard_key_list_dispose(km_kbp_keyboard_key *key_list); - /** - * Returns the list of IMX libraries and function names that are referenced by - * the keyboard. The matching dispose call needs to be called to free the memory. + * km_kbp_keyboard_get_imx_list: + * + * Returns: the list of IMX libraries and function names that are referenced by + * the keyboard.The matching dispose call needs to be called to free the memory. */ KMN_API -km_kbp_status km_kbp_keyboard_get_imx_list(km_kbp_keyboard const *keyboard, km_kbp_keyboard_imx** imx_list); +km_kbp_status km_kbp_keyboard_get_imx_list(km_kbp_keyboard const *keyboard, km_kbp_keyboard_imx **imx_list); /** + * km_kbp_keyboard_imx_list_dispose: + * * Disposes of the IMX list + * + * Returns: -- */ KMN_API void km_kbp_keyboard_imx_list_dispose(km_kbp_keyboard_imx *imx_list); /** + * km_kbp_state_imx_register_callback: + * * Register the IMX callback endpoint for the client. + * + * Returns: -- */ KMN_API void km_kbp_state_imx_register_callback(km_kbp_state *state, km_kbp_keyboard_imx_platform imx_callback, void *callback_object); /** + * km_kbp_state_imx_deregister_callback: + * * De-register IMX callback endpoint for the client. + * + * Returns: -- */ KMN_API void km_kbp_state_imx_deregister_callback(km_kbp_state *state); @@ -1042,6 +1055,8 @@ enum km_kbp_tech_value { }; /** + * km_kbp_event_flags: + * * Bit flags to be used with the event_flags parameter of km_kbp_process_event */ enum km_kbp_event_flags { @@ -1162,10 +1177,8 @@ km_kbp_event( ); enum km_kbp_event_code { - /** - * A keyboard has been activated by the user. The processor may use this - * event, for example, to switch caps lock state or provide other UX. - */ + // A keyboard has been activated by the user. The processor may use this + // event, for example, to switch caps lock state or provide other UX. KM_KBP_EVENT_KEYBOARD_ACTIVATED = 1, //future: KM_KBP_EVENT_KEYBOARD_DEACTIVATED = 2, }; diff --git a/developer/src/common/delphi/compiler/compile.pas b/developer/src/common/delphi/compiler/compile.pas index b89ef0eb72f63f230ba04a9142175b0ff8c278b0..0124150ef6297561d33f62191f572bc06cf03945 100644 --- a/developer/src/common/delphi/compiler/compile.pas +++ b/developer/src/common/delphi/compiler/compile.pas @@ -176,7 +176,6 @@ type TCompilerOptions = record dwSize: DWORD; ShouldAddCompilerVersion: BOOL; - UseKmcmpLib: BOOL; end; COMPILER_OPTIONS = TCompilerOptions; diff --git a/developer/src/common/include/kmn_compiler_errors.h b/developer/src/common/include/kmn_compiler_errors.h index 6cfae03fc9ea00c559fe53f1b55ddb0e5759b66a..31c5faf0c7b33a806b43ce14d013714ac07f4e54 100644 --- a/developer/src/common/include/kmn_compiler_errors.h +++ b/developer/src/common/include/kmn_compiler_errors.h @@ -224,8 +224,8 @@ #define CWARN_KeyShouldIncludeNCaps 0x000020AD #define CHINT_UnreachableRule 0x000010AE -#define CHINT_FilenameHasDifferingCase 0x000010AF -#define CWARN_MissingFile 0x000020B0 +#define CHINT_FilenameHasDifferingCase 0x000010AF // only used in kmcmpdll +#define CWARN_MissingFile 0x000020B0 // only used in kmcmpdll #define CERR_BufferOverflow 0x000080C0 #define CERR_Break 0x000080C1 diff --git a/developer/src/common/web/test-helpers/index.ts b/developer/src/common/web/test-helpers/index.ts index 7f34107dc91a8406b6d82c395f9fc0b0a6f21e36..a134aa67dfd40551100ede0ac74d57ac2ea0a075 100644 --- a/developer/src/common/web/test-helpers/index.ts +++ b/developer/src/common/web/test-helpers/index.ts @@ -36,7 +36,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { /* CompilerCallbacks */ - loadFile(filename: string | URL): Buffer { + loadFile(filename: string): Uint8Array { try { return fs.readFileSync(filename); } catch(e) { @@ -57,7 +57,10 @@ export class TestCompilerCallbacks implements CompilerCallbacks { } resolveFilename(baseFilename: string, filename: string): string { - const basePath = path.dirname(baseFilename); + const basePath = + baseFilename.endsWith('/') || baseFilename.endsWith('\\') ? + baseFilename : + path.dirname(baseFilename); // Transform separators to platform separators -- we are agnostic // in our use here but path prefers files may use // either / or \, although older kps files were always \. @@ -77,7 +80,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { this.messages.push(event); } - loadSchema(schema: CompilerSchema): Buffer { + loadSchema(schema: CompilerSchema): Uint8Array { return fs.readFileSync(new URL(SCHEMA_BASE + schema + '.schema.json', import.meta.url)); } diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index c46b894cb3caa3be6ee6cb430f9befc671468cac..9962ea6abd5048b6a1561de0dcae2605163f36fb 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -42,62 +42,26 @@ const baseOptions: CompilerOptions = { */ let callbackProcIdentifier = 0; -/** - * Pointer in wasm-space - */ -type WasmPtr = number; - -/** - * The wrapped functions - */ -class WasmWrapper { - Module: any; - - compileKeyboardFile?: (pszInfile: string, pszOutfile: string, aSaveDebug: number, aCompilerWarningsAsErrors: number, aWarnDeprecatedCode: number, msgProc: string) => boolean; - parseUnicodeSet?: (pat: string, buf: WasmPtr, length: number) => number; - setCompilerOptions?: (shouldAddCompilerVersion: number) => boolean; - - constructor(wasmModule: any) { - this.Module = wasmModule; - if (!wasmModule) { - throw Error(`wasm host did not load`); - } - this.compileKeyboardFile = this.Module.cwrap('kmcmp_Wasm_CompileKeyboardFile', 'boolean', ['string', 'string', 'number', 'number', 'number', 'string']); - this.parseUnicodeSet = this.Module.cwrap('kmcmp_Wasm_ParseUnicodeSet', 'number', ['string', 'number', 'number']); - this.setCompilerOptions = this.Module.cwrap('kmcmp_Wasm_SetCompilerOptions', 'boolean', ['number']); - - if (this.parseUnicodeSet === undefined - || this.setCompilerOptions === undefined - || this.compileKeyboardFile === undefined) { - throw Error(`some wasm functions did not load properly.`); - } - } - - /** - * Entry point into Wasm functions - * @returns WasmWrapper - */ - public static async load() : Promise { - return new WasmWrapper(await loadWasmHost()); - } -}; +const + callbackPrefix = 'kmnCompilerCallbacks_'; export class KmnCompiler { - callbackName: string; + private Module: any; + callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; - wasm: WasmWrapper; constructor() { - this.callbackName = 'kmnCompilerCallback' + callbackProcIdentifier; + this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); callbackProcIdentifier++; } public async init(callbacks: CompilerCallbacks): Promise { this.callbacks = callbacks; - if(!this.wasm) { + if(!this.Module) { try { - this.wasm = await WasmWrapper.load(); + this.Module = await loadWasmHost(); } catch(e: any) { + /* c8 ignore next 3 */ this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({e})); return false; } @@ -114,7 +78,9 @@ export class KmnCompiler { // Can't report a message here. throw Error('Must call Compiler.init(callbacks) before proceeding'); } - if(!this.wasm) { // fail if wasm not loaded or function not found + if(!this.Module) { + /* c8 ignore next 4 */ + // fail if wasm not loaded or function not found this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({})); return false; } @@ -122,16 +88,7 @@ export class KmnCompiler { } public run(infile: string, outfile: string, options?: CompilerOptions): boolean { - if(!this.verifyInitialized()) { - return false; - } - - options = {...baseOptions, ...options}; - (globalThis as any)[this.callbackName] = this.compilerMessageCallback; - // TODO: use callbacks for file access -- so kmc-kmn is entirely fs agnostic let result = this.runCompiler(infile, outfile, options); - delete (globalThis as any)[this.callbackName]; - //TODO: write the file out! if(result) { if(result.kmx) { this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); @@ -151,17 +108,66 @@ export class KmnCompiler { return 1; } - private runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult { + private cachedFile: {filename: string; data: Uint8Array} = { + filename: null, + data: null + }; + + private loadFileCallback = (filename: string, baseFilename: string, buffer: number, bufferSize: number): number => { + let resolvedFilename = this.callbacks.resolveFilename(baseFilename, filename); + let data: Uint8Array; + if(this.cachedFile.filename == resolvedFilename) { + data = this.cachedFile.data; + } + else { + data = this.callbacks.loadFile(resolvedFilename); + if(!data) { + return -1; + } + this.cachedFile.filename = resolvedFilename; + this.cachedFile.data = data; + } + + if(buffer == 0) { + /* We need to return buffer size required */ + return data.byteLength; + } + + if(bufferSize != data.byteLength) { + /* c8 ignore next 2 */ + throw new Error(`loadFileCallback: second call, expected file size ${bufferSize} == ${data.byteLength}`); + } + + this.Module.HEAP8.set(data, buffer); + + return 1; + } + + public runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult { + if(!this.verifyInitialized()) { + /* c8 ignore next 2 */ + return null; + } + + options = {...baseOptions, ...options}; + + (globalThis as any)[this.callbackID] = { + message: this.compilerMessageCallback, + loadFile: this.loadFileCallback + }; + let result: CompilerResult = {}; - let wasm_interface = new this.wasm.Module.CompilerInterface(); + let wasm_interface = new this.Module.CompilerInterface(); + let wasm_options = new this.Module.CompilerOptions(); let wasm_result = null; try { - wasm_interface.saveDebug = options.saveDebug; - wasm_interface.compilerWarningsAsErrors = options.compilerWarningsAsErrors; - wasm_interface.warnDeprecatedCode = options.warnDeprecatedCode; - wasm_interface.messageCallback = this.callbackName; - wasm_interface.loadFileCallback = this.callbackName; // TODO: this is wrong, needs to be a new callback; not yet used though - wasm_result = this.wasm.Module.kmcmp_compile(infile, wasm_interface); + wasm_options.saveDebug = options.saveDebug; + wasm_options.compilerWarningsAsErrors = options.compilerWarningsAsErrors; + wasm_options.warnDeprecatedCode = options.warnDeprecatedCode; + wasm_options.shouldAddCompilerVersion = options.shouldAddCompilerVersion; + wasm_options.target = 0; //CKF_KEYMAN; TODO, support KMW + wasm_interface.callbacksKey = this.callbackID; // key of object on globalThis + wasm_result = this.Module.kmcmp_compile(infile, wasm_options, wasm_interface); if(!wasm_result.result) { return null; } @@ -175,11 +181,12 @@ export class KmnCompiler { result.kmx = { filename: outfile, - data: new Uint8Array(this.wasm.Module.HEAP8.buffer, wasm_result.kmx, wasm_result.kmxSize) + data: new Uint8Array(this.Module.HEAP8.buffer, wasm_result.kmx, wasm_result.kmxSize) }; return result; } catch(e) { + /* c8 ignore next 3 */ this.callbacks.reportMessage(CompilerMessages.Fatal_UnexpectedException({e:e})); return null; } finally { @@ -187,6 +194,8 @@ export class KmnCompiler { wasm_result.delete(); } wasm_interface.delete(); + wasm_options.delete(); + delete (globalThis as any)[this.callbackID]; } } @@ -194,20 +203,19 @@ export class KmnCompiler { // The compiler detected a .kvks file, which needs to be captured let reader = new KvksFileReader(); kvksFilename = this.callbacks.resolveFilename(kmnFilename, kvksFilename); - let kvks = reader.read(this.callbacks.loadFile(kvksFilename)); + let filename = this.callbacks.path.basename(kvksFilename); + let kvks = null; try { + kvks = reader.read(this.callbacks.loadFile(kvksFilename)); reader.validate(kvks, this.callbacks.loadSchema('kvks')); } catch(e) { - console.log(e); - // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); + this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({filename, e})); return null; } - let errors: any = []; //TODO: KVKSParseError[]; - let vk = reader.transform(kvks, errors); - if(!vk || errors.length) { - console.dir(errors); - // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); - return null; + let invalidVkeys: string[] = []; + let vk = reader.transform(kvks, invalidVkeys); + for(let invalidVkey of invalidVkeys) { + this.callbacks.reportMessage(CompilerMessages.Warn_InvalidVkeyInKvksFile({filename, invalidVkey})); } let writer = new KvkFileWriter(); return { @@ -224,22 +232,23 @@ export class KmnCompiler { */ public parseUnicodeSet(pattern: string, bufferSize: number) : UnicodeSet | null { if(!this.verifyInitialized()) { + /* c8 ignore next 2 */ return null; } if (!bufferSize) { + /* c8 ignore next 2 */ bufferSize = 100; // TODO-LDML: Preflight mode? Reuse buffer? } - const { Module } = this.wasm; - const buf = Module.asm.malloc(bufferSize * 2 * Module.HEAPU32.BYTES_PER_ELEMENT); + const buf = this.Module.asm.malloc(bufferSize * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT); // TODO-LDML: Catch OOM - const rc = this.wasm.parseUnicodeSet(pattern, buf, bufferSize); + const rc = this.Module.kmcmp_parseUnicodeSet(pattern, buf, bufferSize); if (rc >= 0) { const ranges = []; - const startu = (buf / Module.HEAPU32.BYTES_PER_ELEMENT); + const startu = (buf / this.Module.HEAPU32.BYTES_PER_ELEMENT); for (let i = 0; i < rc; i++) { - const low = Module.HEAPU32[startu + (i * 2) + 0]; - const high = Module.HEAPU32[startu + (i * 2) + 1]; + const low = this.Module.HEAPU32[startu + (i * 2) + 0]; + const high = this.Module.HEAPU32[startu + (i * 2) + 1]; ranges.push([low, high]); } // TODO-LDML: no free?? @@ -276,6 +285,7 @@ function getUnicodeSetError(rc: number) : CompilerEvent { case KMCMP_FATAL_OUT_OF_RANGE: return CompilerMessages.Fatal_UnicodeSetOutOfRange(); default: + /* c8 ignore next */ return CompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}); } } diff --git a/developer/src/kmc-kmn/src/compiler/messages.ts b/developer/src/kmc-kmn/src/compiler/messages.ts index 10578870d7c1499350936abe16981a503cc8167a..0769096e070014b221a545ce4c34736ec95b9d02 100644 --- a/developer/src/kmc-kmn/src/compiler/messages.ts +++ b/developer/src/kmc-kmn/src/compiler/messages.ts @@ -67,6 +67,14 @@ export class CompilerMessages { static Error_UnicodeSetSyntaxError = () => m(this.ERROR_UnicodeSetSyntaxError, `UnicodeSet had a Syntax Error while parsing`); static ERROR_UnicodeSetSyntaxError = SevError | 0x1007; + + static Error_InvalidKvksFile = (o:{filename: string, e: any}) => m(this.ERROR_InvalidKvksFile, + `Error encountered parsing ${o.filename}: ${o.e}`); + static ERROR_InvalidKvksFile = SevError | 0x1008; + + static Warn_InvalidVkeyInKvksFile = (o:{filename: string, invalidVkey: string}) => m(this.WARN_InvalidVkeyInKvksFile, + `Invalid virtual key ${o.invalidVkey} found in ${o.filename}`); + static WARN_InvalidVkeyInKvksFile = SevWarn | 0x1009; } export function mapErrorFromKmcmplib(line: number, code: number, msg: string): CompilerEvent { diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn new file mode 100644 index 0000000000000000000000000000000000000000..f37d6397c5099dbbb61bfb3fa464c90c29dc9958 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmc throws an error with an invalid .kvks file + +store(&NAME) 'error_invalid_kvks_file' +store(&VERSION) '9.0' +store(&VISUALKEYBOARD) 'error_invalid_kvks_file.kvks' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks new file mode 100644 index 0000000000000000000000000000000000000000..c681064f5b188af0703f5c1a31c66332abccf68c --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks @@ -0,0 +1,9 @@ + + +

+ 10.0 + caps_lock_layer_3620 + + +
+ diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn new file mode 100644 index 0000000000000000000000000000000000000000..18a88973e93d128ea46c93c30f5346cc0395c3b1 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmc reports a warning with an invalid vkey in the kvks + +store(&NAME) 'warn_invalid_vkey_in_kvks_file' +store(&VERSION) '9.0' +store(&VISUALKEYBOARD) 'warn_invalid_vkey_in_kvks_file.kvks' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks new file mode 100644 index 0000000000000000000000000000000000000000..f7b247db34e1f9ea8072c40c3a57289d3b461380 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks @@ -0,0 +1,14 @@ + + +
+ 10.0 + caps_lock_layer_3620 + +
+ + + + ážž + + +
diff --git a/developer/src/kmc-kmn/test/helpers/index.ts b/developer/src/kmc-kmn/test/helpers/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..8250dfd101a0a07fd2966cf3e5537ac3386a92f5 --- /dev/null +++ b/developer/src/kmc-kmn/test/helpers/index.ts @@ -0,0 +1,17 @@ +/** + * Helpers and utilities for the Mocha tests. + */ +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Builds a path to the fixture with the given path components. + * + * e.g., makePathToFixture('example.qaa.trivial') + * e.g., makePathToFixture('example.qaa.trivial', 'model.ts') + * + * @param components One or more path components. + */ + export function makePathToFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); +} diff --git a/developer/src/kmc-kmn/test/test-compiler.ts b/developer/src/kmc-kmn/test/test-compiler.ts index 925656f8f554e938141b13bbb35f4f7de817ff76..8c353983613f6d766f6fb8780b91153558c8b51c 100644 --- a/developer/src/kmc-kmn/test/test-compiler.ts +++ b/developer/src/kmc-kmn/test/test-compiler.ts @@ -86,20 +86,20 @@ describe('Compiler class', function() { const kmxFixture = fixtureDir + '/binary/caps_lock_layer_3620.kmx'; const kvkFixture = fixtureDir + '/binary/caps_lock_layer_3620.kvk'; - const kmxfile = __dirname + '/caps_lock_layer_3620.kmx'; - const kvkfile = __dirname + '/caps_lock_layer_3620.kvk'; + const resultingKmxfile = __dirname + '/caps_lock_layer_3620.kmx'; + const resultingKvkfile = __dirname + '/caps_lock_layer_3620.kvk'; - assert(compiler.run(infile, kmxfile, {saveDebug: true, shouldAddCompilerVersion: false})); + assert(compiler.run(infile, resultingKmxfile, {saveDebug: true, shouldAddCompilerVersion: false})); - assert(fs.existsSync(kmxfile)); - assert(fs.existsSync(kvkfile)); + assert(fs.existsSync(resultingKmxfile)); + assert(fs.existsSync(resultingKvkfile)); - const kmxData = fs.readFileSync(kmxfile); + const kmxData = fs.readFileSync(resultingKmxfile); const kmxFixtureData = fs.readFileSync(kmxFixture); assert.equal(kmxData.byteLength, kmxFixtureData.byteLength); assert.deepEqual(kmxData, kmxFixtureData); - const kvkData = fs.readFileSync(kvkfile); + const kvkData = fs.readFileSync(resultingKvkfile); const kvkFixtureData = fs.readFileSync(kvkFixture); assert.equal(kvkData.byteLength, kvkFixtureData.byteLength); assert.deepEqual(kvkData, kvkFixtureData); diff --git a/developer/src/kmc-kmn/test/test-messages.ts b/developer/src/kmc-kmn/test/test-messages.ts index 0dee53f12112d47b67ca19bc266630a791dd4bf9..4d724daaf5338213125c96876ebef03af699093b 100644 --- a/developer/src/kmc-kmn/test/test-messages.ts +++ b/developer/src/kmc-kmn/test/test-messages.ts @@ -1,9 +1,55 @@ import 'mocha'; +import path from 'path'; +import { assert } from 'chai'; import { CompilerMessages } from '../src/compiler/messages.js'; -import { verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; +import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; +import { makePathToFixture } from './helpers/index.js'; +import { KmnCompiler } from '../src/main.js'; describe('CompilerMessages', function () { + const callbacks = new TestCompilerCallbacks(); + it('should have a valid CompilerMessages object', function() { return verifyCompilerMessagesObject(CompilerMessages); }); + + // + // Message tests + // + + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number) { + context.timeout(10000); + + callbacks.clear(); + + const compiler = new KmnCompiler(); + assert(await compiler.init(callbacks)); + assert(compiler.verifyInitialized()); + + const kmnPath = makePathToFixture(...fixture); + const outfile = path.basename(kmnPath, '.kmn') + '.kmx'; + + // Note: throwing away compile results (just to memory) + compiler.runCompiler(kmnPath, outfile, {saveDebug: true, shouldAddCompilerVersion: false}); + + if(messageId) { + assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); + assert.lengthOf(callbacks.messages, 1); + } else { + assert.lengthOf(callbacks.messages, 0, `messages should be empty, but instead got: `+JSON.stringify(callbacks.messages,null,2)); + } + } + + // ERROR_InvalidKvksFile + + it('should generate ERROR_InvalidKvksFile if the kvks is not valid XML', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_invalid_kvks_file.kmn'], CompilerMessages.ERROR_InvalidKvksFile); + }); + + // WARN_InvalidVkeyInKvksFile + + it('should generate WARN_InvalidVkeyInKvksFile if the kvks contains an invalid virtual key', async function() { + await testForMessage(this, ['invalid-keyboards', 'warn_invalid_vkey_in_kvks_file.kmn'], CompilerMessages.WARN_InvalidVkeyInKvksFile); + }); + }); diff --git a/developer/src/kmc-kmn/test/tsconfig.json b/developer/src/kmc-kmn/test/tsconfig.json index 115412f57f397294d7bb8336dfed238cd9b3040c..eb43dc68f19d86f3f6b9b1d0637e6de450c6bc03 100644 --- a/developer/src/kmc-kmn/test/tsconfig.json +++ b/developer/src/kmc-kmn/test/tsconfig.json @@ -13,10 +13,11 @@ }, }, "include": [ - "**/test-*.ts" + "**/test-*.ts", + "./helpers/index.ts" ], "references": [ - { "path": "../../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../../common/web/keyman-version/" }, { "path": "../../../../common/web/types/" }, { "path": "../../../../common/tools/hextobin/" }, { "path": "../../common/web/test-helpers/" }, diff --git a/developer/src/kmc-kmn/tsconfig.json b/developer/src/kmc-kmn/tsconfig.json index b828a5076487c095a83cfaf700f73801f44d758b..04683902eae928e1f470248642611aa843d3f2e5 100644 --- a/developer/src/kmc-kmn/tsconfig.json +++ b/developer/src/kmc-kmn/tsconfig.json @@ -16,7 +16,7 @@ "src/import/kmcmplib/wasm-host.js" ], "references": [ - { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../common/web/keyman-version/tsconfig.json" }, { "path": "../../../common/web/types/" }, ] } diff --git a/developer/src/kmc-ldml/src/compiler/compiler-options.ts b/developer/src/kmc-ldml/src/compiler/compiler-options.ts index 3d29e6393ad3de65cca0db87f464c8cc4f91c8f1..da81ef2f7d5489f7ccf3b742a4bf0d7088fc6e13 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler-options.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler-options.ts @@ -1,4 +1,4 @@ - +import { LDMLKeyboardXMLSourceFileReaderOptions } from "@keymanapp/common-types"; export interface CompilerOptions { /** @@ -10,4 +10,9 @@ export interface CompilerOptions { * Add metadata about the compiler version to .kmx file when compiling */ addCompilerVersion?: boolean; + + /** + * Paths and other options required for reading .xml files + */ + readerOptions: LDMLKeyboardXMLSourceFileReaderOptions; }; diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b7b46e49b2a2e9304f0b2616ff9d22eab6ab0268..4b1472a58530dac22ca620b783c7c4d8b68290df 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,16 +30,14 @@ const SECTION_COMPILERS = [ export class LdmlKeyboardCompiler { private readonly callbacks: CompilerCallbacks; - // private readonly options: CompilerOptions; // not currently used + private readonly options: CompilerOptions; - constructor (callbacks: CompilerCallbacks, _options?: CompilerOptions) { - /* + constructor (callbacks: CompilerCallbacks, options: CompilerOptions) { this.options = { debug: false, addCompilerVersion: true, ...options }; - */ this.callbacks = callbacks; } @@ -54,7 +52,7 @@ export class LdmlKeyboardCompiler { * @returns the source file, or null if invalid */ public load(filename: string): LDMLKeyboardXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.callbacks); + const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); const data = this.callbacks.loadFile(filename); if(!data) { this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: 'Unable to read XML file'})); @@ -84,7 +82,7 @@ export class LdmlKeyboardCompiler { * @returns the source file, or null if invalid */ public loadTestData(filename: string): LDMLKeyboardTestDataXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.callbacks); + const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); const data = this.callbacks.loadFile(filename); if(!data) { this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: 'Unable to read XML file'})); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 8ba8a28aa84ded8a6549461d6ea8b657302bb431..f10af945c82488d9e1543efbe9e623a138488c12 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -33,6 +33,12 @@ export function makePathToFixture(...components: string[]): string { export const compilerTestCallbacks = new TestCompilerCallbacks(); +export const compilerTestOptions: CompilerOptions = { + readerOptions: { + importsPath: fileURLToPath(LDMLKeyboardXMLSourceFileReader.defaultImportsURL) + } +}; + beforeEach(function() { compilerTestCallbacks.clear(); }); @@ -43,14 +49,13 @@ afterEach(function() { } }); - export function loadSectionFixture(compilerClass: typeof SectionCompiler, filename: string, callbacks: TestCompilerCallbacks): Section { callbacks.messages = []; const inputFilename = makePathToFixture(filename); const data = callbacks.loadFile(inputFilename); assert.isNotNull(data); - const reader = new LDMLKeyboardXMLSourceFileReader(callbacks); + const reader = new LDMLKeyboardXMLSourceFileReader(compilerTestOptions.readerOptions, callbacks); const source = reader.load(data); assert.isNotNull(source); diff --git a/developer/src/kmc-ldml/test/test-compiler-e2e.ts b/developer/src/kmc-ldml/test/test-compiler-e2e.ts index ec711549051a4f4a5d43911f8ae8b2924996b962..7d8b179f3125782b8ae557143558a344afa61b86 100644 --- a/developer/src/kmc-ldml/test/test-compiler-e2e.ts +++ b/developer/src/kmc-ldml/test/test-compiler-e2e.ts @@ -2,7 +2,7 @@ import 'mocha'; import {assert} from 'chai'; import x_hextobin from '@keymanapp/hextobin'; import { KMXBuilder } from '@keymanapp/common-types'; -import {checkMessages, compileKeyboard, makePathToFixture} from './helpers/index.js'; +import {checkMessages, compileKeyboard, compilerTestOptions, makePathToFixture} from './helpers/index.js'; const hextobin = (x_hextobin as any).default; @@ -17,7 +17,7 @@ describe('compiler-tests', function() { const binaryFilename = makePathToFixture('basic.txt'); // Compile the keyboard - const kmx = compileKeyboard(inputFilename, {debug: true, addCompilerVersion: false}); + const kmx = compileKeyboard(inputFilename, {...compilerTestOptions, debug: true, addCompilerVersion: false}); assert.isNotNull(kmx); // Use the builder to generate the binary output file diff --git a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts index fb4f28daeeedf1fef9a6c191df57b55e0c5f2933..711820e5e4051a751304f1620369e7f1f131d151 100644 --- a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts +++ b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { checkMessages, compilerTestCallbacks, makePathToFixture } from './helpers/index.js'; +import { checkMessages, compilerTestCallbacks, compilerTestOptions, makePathToFixture } from './helpers/index.js'; import { LdmlKeyboardKeymanWebCompiler } from '../src/compiler/keymanweb-compiler.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import * as fs from 'fs'; @@ -16,7 +16,7 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { // Load input data; we'll use the LDML keyboard compiler loader to save us // effort here - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {debug: true, addCompilerVersion: false}); + const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {...compilerTestOptions, debug: true, addCompilerVersion: false}); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -27,7 +27,7 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { assert.isTrue(valid, 'k.validate should not have failed'); // Actual test: compile to javascript - const jsCompiler = new LdmlKeyboardKeymanWebCompiler(compilerTestCallbacks, {debug: true}); + const jsCompiler = new LdmlKeyboardKeymanWebCompiler(compilerTestCallbacks, {...compilerTestOptions, debug: true}); const output = jsCompiler.compile('basic.xml', source); assert.isNotNull(output); @@ -36,7 +36,7 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { assert.strictEqual(output, outputFixture); // Second test: compile to javascript without debug formatting - const jsCompilerNoDebug = new LdmlKeyboardKeymanWebCompiler(compilerTestCallbacks, {debug: false}); + const jsCompilerNoDebug = new LdmlKeyboardKeymanWebCompiler(compilerTestCallbacks, {...compilerTestOptions, debug: false}); const outputNoDebug = jsCompilerNoDebug.compile('basic.xml', source); assert.isNotNull(outputNoDebug); diff --git a/developer/src/kmc-ldml/test/test-metadata-compiler.ts b/developer/src/kmc-ldml/test/test-metadata-compiler.ts index 59500d09950c313568948fb70b8322ea0799b0d7..ede77ed4bccfba209a0d6f1e570ffa9bd56e7755 100644 --- a/developer/src/kmc-ldml/test/test-metadata-compiler.ts +++ b/developer/src/kmc-ldml/test/test-metadata-compiler.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { checkMessages, compileKeyboard, makePathToFixture } from './helpers/index.js'; +import { checkMessages, compileKeyboard, compilerTestOptions, makePathToFixture } from './helpers/index.js'; import { KMX } from '@keymanapp/common-types'; import KEYMAN_VERSION from '@keymanapp/keyman-version'; @@ -13,7 +13,7 @@ describe('kmx metadata compiler', function () { const inputFilename = makePathToFixture('basic.xml'); // Compile the keyboard - const kmx = compileKeyboard(inputFilename, {debug:true, addCompilerVersion:true}); + const kmx = compileKeyboard(inputFilename, {...compilerTestOptions, debug:true, addCompilerVersion:true}); checkMessages(); assert.isNotNull(kmx); @@ -48,7 +48,7 @@ describe('kmx metadata compiler', function () { const inputFilename = makePathToFixture('basic.xml'); // Compile the keyboard - const kmx = compileKeyboard(inputFilename, {debug:true, addCompilerVersion:false}); + const kmx = compileKeyboard(inputFilename, {...compilerTestOptions, debug:true, addCompilerVersion:false}); checkMessages(); assert.isNotNull(kmx); diff --git a/developer/src/kmc-ldml/test/test-testdata-e2e.ts b/developer/src/kmc-ldml/test/test-testdata-e2e.ts index 25ac1963213a5fed475292821837b4fbb8c005fb..ece4ce85608174bac50522c48810ca3f89fac3df 100644 --- a/developer/src/kmc-ldml/test/test-testdata-e2e.ts +++ b/developer/src/kmc-ldml/test/test-testdata-e2e.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import 'mocha'; import {assert} from 'chai'; -import {loadTestdata, makePathToFixture} from './helpers/index.js'; +import {compilerTestOptions, loadTestdata, makePathToFixture} from './helpers/index.js'; describe('testdata-tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while @@ -14,7 +14,7 @@ describe('testdata-tests', function() { const jsonFilename = makePathToFixture('test-fr.json'); // Compile the keyboard - const testData = loadTestdata(inputFilename, {debug: true, addCompilerVersion: false}); + const testData = loadTestdata(inputFilename, {...compilerTestOptions, debug: true, addCompilerVersion: false}); assert.isNotNull(testData); const jsonData = JSON.parse(readFileSync(jsonFilename, 'utf-8')); diff --git a/developer/src/kmc-ldml/test/test-visual-keyboard-compiler-e2e.ts b/developer/src/kmc-ldml/test/test-visual-keyboard-compiler-e2e.ts index 1b1e34069e5ac2b13982f5ee18654b80d1bacb6f..4af68278387c8dd89283108fcbe6d031ee8b51a8 100644 --- a/developer/src/kmc-ldml/test/test-visual-keyboard-compiler-e2e.ts +++ b/developer/src/kmc-ldml/test/test-visual-keyboard-compiler-e2e.ts @@ -2,7 +2,7 @@ import 'mocha'; import {assert} from 'chai'; import x_hextobin from '@keymanapp/hextobin'; import { KvkFileWriter } from '@keymanapp/common-types'; -import {checkMessages, compileVisualKeyboard, makePathToFixture} from './helpers/index.js'; +import {checkMessages, compilerTestOptions, compileVisualKeyboard, makePathToFixture} from './helpers/index.js'; const hextobin = (x_hextobin as any).default; @@ -17,7 +17,7 @@ describe('visual-keyboard-compiler', function() { const binaryFilename = makePathToFixture('basic-kvk.txt'); // Compile the visual keyboard - const vk = compileVisualKeyboard(inputFilename, {debug: true, addCompilerVersion: false}); + const vk = compileVisualKeyboard(inputFilename, {...compilerTestOptions, debug: true, addCompilerVersion: false}); assert.isNotNull(vk); // Use the builder to generate the binary output file diff --git a/developer/src/kmc-ldml/test/tsconfig.json b/developer/src/kmc-ldml/test/tsconfig.json index cfc7583fa48620c06c92c20039f6b11dfa724f4d..f47c5c5f71f31821ad1ee65b7cb9a29929404617 100644 --- a/developer/src/kmc-ldml/test/tsconfig.json +++ b/developer/src/kmc-ldml/test/tsconfig.json @@ -19,7 +19,7 @@ "./helpers/index.ts" ], "references": [ - { "path": "../../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../../common/web/keyman-version" }, { "path": "../../../../common/web/types/" }, { "path": "../../../../common/tools/hextobin/" }, { "path": "../../common/web/test-helpers/" }, diff --git a/developer/src/kmc-ldml/tsconfig.json b/developer/src/kmc-ldml/tsconfig.json index a9dcdbbb2a8e8039b5bd7a448ab80c93434df207..d7c0d4841eafe24cc3fbbab2e77219c17d40f562 100644 --- a/developer/src/kmc-ldml/tsconfig.json +++ b/developer/src/kmc-ldml/tsconfig.json @@ -18,7 +18,7 @@ "src/**/*.ts" ], "references": [ - { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../common/web/keyman-version" }, { "path": "../../../common/web/types/" }, { "path": "../../../core/include/ldml/"}, ] diff --git a/developer/src/kmc-model/src/main.ts b/developer/src/kmc-model/src/main.ts index 3d808057d5981cef795bd59545735b1ef6cc1e31..32e2b03afd317532e68c22c63f205f3073743987 100644 --- a/developer/src/kmc-model/src/main.ts +++ b/developer/src/kmc-model/src/main.ts @@ -8,6 +8,8 @@ import LexicalModelCompiler from './lexical-model-compiler.js'; import { LexicalModelSource } from './lexical-model.js'; import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from './model-compiler-errors.js'; +export { default as LexicalModelCompiler } from './lexical-model-compiler.js'; + /** * Compiles a model.ts file, using paths relative to its location. * diff --git a/developer/src/kmc-model/tsconfig.json b/developer/src/kmc-model/tsconfig.json index 8ad34413fa273854677e75ad7d41264a6434e234..724ebd74d0b6dba26ff2942965ebae101f06062c 100644 --- a/developer/src/kmc-model/tsconfig.json +++ b/developer/src/kmc-model/tsconfig.json @@ -11,7 +11,7 @@ "src/**/*.ts" ], "references": [ - { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../common/web/keyman-version" }, { "path": "../../../common/models/types" }, { "path": "../../../common/web/types" }, ] diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index b660641df9c564972064f4ed644433dd793c3f65..198a1ef2de2823b43902867c517aa9b85aa66b91 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -16,7 +16,12 @@ export class KmpCompiler { public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile { // Load the KPS data from XML as JS structured data. - const data = this.callbacks.fs.readFileSync(kpsFilename, 'utf-8'); + const buffer = this.callbacks.loadFile(kpsFilename); + if(!buffer) { + this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename: kpsFilename})); + return null; + } + const data = new TextDecoder().decode(buffer); const kpsPackage = (() => { let a: KpsFile.KpsPackage; @@ -24,7 +29,8 @@ export class KmpCompiler { tagNameProcessors: [xml2js.processors.firstCharLowerCase], explicitArray: false }); - parser.parseString(data, (e: unknown, r: unknown) => { a = r as KpsFile.KpsPackage }); + // TODO: add unit test for xml errors parsing .kps file + parser.parseString(data, (e: unknown, r: unknown) => { if(e) throw e; a = r as KpsFile.KpsPackage }); return a; })(); @@ -284,9 +290,20 @@ export class KmpCompiler { * we want that to remain the responsibility of the keyboard compiler, so we'll warn the * few users who are still doing this */ - private warnIfKvkFileIsNotBinary(filename: string, data: Buffer) { - // TODO: Buffer is not available on web - if(filename.match(/\.kvk$/) && data.compare(Buffer.from(KvkFile.KVK_HEADER_IDENTIFIER_BYTES), 0, 3, 0, 3) != 0) { + private warnIfKvkFileIsNotBinary(filename: string, data: Uint8Array) { + if(!filename.match(/\.kvk$/)) { + return; + } + + if(data.byteLength < 4) { + // TODO: Not a valid .kvk file; should we be reporting this? + return; + } + + if(data[0] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[0] || + data[1] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[1] || + data[2] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[2] || + data[3] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[3]) { this.callbacks.reportMessage(CompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename})); } } diff --git a/developer/src/kmc-package/test/tsconfig.json b/developer/src/kmc-package/test/tsconfig.json index 9700969d6220af9e2837776691cc084cda01cf31..05d10505504675c2732a7c608f32b600cb80f884 100644 --- a/developer/src/kmc-package/test/tsconfig.json +++ b/developer/src/kmc-package/test/tsconfig.json @@ -18,7 +18,7 @@ "references": [ { "path": "../" }, { "path": "../../../../common/web/types" }, - { "path": "../../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../../common/web/keyman-version" }, { "path": "../../common/web/test-helpers/" }, ] } \ No newline at end of file diff --git a/developer/src/kmc-package/tsconfig.json b/developer/src/kmc-package/tsconfig.json index 8913057047eadc303181a6e070dbb16889d459ed..0ab78a91ffbf7025199113da2b69fcdbc06900f4 100644 --- a/developer/src/kmc-package/tsconfig.json +++ b/developer/src/kmc-package/tsconfig.json @@ -10,7 +10,7 @@ "src/**/*.ts", ], "references": [ - { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../common/web/keyman-version" }, { "path": "../../../common/web/types" }, ] } diff --git a/developer/src/kmc/src/commands/build/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/build/BuildLdmlKeyboard.ts index 024f7f534002f6231322b561c7b36c0315debe04..40b9513d320e1971824a93f30c62c3f4ea206604 100644 --- a/developer/src/kmc/src/commands/build/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/build/BuildLdmlKeyboard.ts @@ -1,8 +1,9 @@ import * as path from 'path'; import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { KvkFileWriter, CompilerCallbacks } from '@keymanapp/common-types'; +import { KvkFileWriter, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; import { BuildActivity, BuildActivityOptions } from './BuildActivity.js'; +import { fileURLToPath } from 'url'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -46,11 +47,14 @@ function buildLdmlKeyboardToMemory(inputFilename: string, callbacks: CompilerCal let compilerOptions: kmcLdml.CompilerOptions = { debug: options.debug ?? false, addCompilerVersion: options.compilerVersion ?? true, + readerOptions: { + importsPath: fileURLToPath(LDMLKeyboardXMLSourceFileReader.defaultImportsURL) + } // TODO: warnDeprecatedCode: options.warnDeprecatedCode, // TODO: treatWarningsAsErrors: options.treatWarningsAsErrors, } - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, options); + const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, compilerOptions); let source = k.load(inputFilename); if (!source) { return [null, null, null]; diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 0990b91edb55de750acdb544ffa0a6e1105cbd09..b6ebf34a739326e797565407d2dddb217a39a622 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -1,8 +1,9 @@ import * as fs from 'fs'; import * as path from 'path'; import * as kmc from '@keymanapp/kmc-ldml'; -import { CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile } from '@keymanapp/common-types'; +import { CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../messages/NodeCompilerCallbacks.js'; +import { fileURLToPath } from 'url'; export interface BuildTestDataOptions { outFile?: string; @@ -11,7 +12,10 @@ export interface BuildTestDataOptions { export function buildTestData(infile: string, options: BuildTestDataOptions) { let compilerOptions: kmc.CompilerOptions = { debug: false, - addCompilerVersion: false + addCompilerVersion: false, + readerOptions: { + importsPath: fileURLToPath(LDMLKeyboardXMLSourceFileReader.defaultImportsURL) + } }; let testData = loadTestData(infile, compilerOptions); diff --git a/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts b/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts index ceae18c4c59e7f8165a39460fab40558621fd211..dfcb7e2ecbfae760a3c72f66b9b1baad50819040 100644 --- a/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts @@ -1,14 +1,52 @@ import * as fs from 'fs'; import * as path from 'path'; import { CompilerCallbacks, CompilerSchema, CompilerEvent, compilerErrorSeverityName, CompilerPathCallbacks, CompilerFileSystemCallbacks } from '@keymanapp/common-types'; +import { InfrastructureMessages } from './messages.js'; /** * Concrete implementation for CLI use */ +// TODO: Make a common class for all the CompilerCallbacks implementations + export class NodeCompilerCallbacks implements CompilerCallbacks { - // TODO: REMOVE! - loadFile(filename: string | URL): Buffer { + /* NodeCompilerCallbacks */ + + messages: CompilerEvent[] = []; + silent: boolean; + + constructor(silent?: boolean) { + this.silent = !!silent; + } + + clear() { + this.messages = []; + } + + hasMessage(code: number): boolean { + return this.messages.find((item) => item.code == code) === undefined ? false : true; + } + + private verifyFilenameConsistency(originalFilename: string): void { + if(fs.existsSync(originalFilename)) { + // Note, we only check this if the file exists, because + // if it is not found, that will be returned as an error + // from loadFile anyway. + const filename = fs.realpathSync(originalFilename); + const nativeFilename = fs.realpathSync.native(filename); + if(filename != nativeFilename) { + this.reportMessage(InfrastructureMessages.Hint_FilenameHasDifferingCase({ + reference: originalFilename, + filename: nativeFilename + })); + } + } + } + + /* CompilerCallbacks */ + + loadFile(filename: string): Uint8Array { + this.verifyFilenameConsistency(filename); try { return fs.readFileSync(filename); } catch (e) { @@ -29,6 +67,10 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { } reportMessage(event: CompilerEvent): void { + this.messages.push(event); + if(this.silent) { + return; + } const code = event.code.toString(16); if(event.line) { console.log(`${compilerErrorSeverityName(event.code)} ${code} [${event.line}]: ${event.message}`); @@ -41,7 +83,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { console.debug(msg); } - loadSchema(schema: CompilerSchema) { + loadSchema(schema: CompilerSchema): Uint8Array { let schemaPath = new URL('../util/' + schema + '.schema.json', import.meta.url); return fs.readFileSync(schemaPath); } @@ -51,7 +93,10 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { } resolveFilename(baseFilename: string, filename: string) { - const basePath = path.dirname(baseFilename); + const basePath = + baseFilename.endsWith('/') || baseFilename.endsWith('\\') ? + baseFilename : + path.dirname(baseFilename); // Transform separators to platform separators -- we are agnostic // in our use here but path prefers files may use // either / or \, although older kps files were always \. diff --git a/developer/src/kmc/src/messages/messages.ts b/developer/src/kmc/src/messages/messages.ts index 1f1761e715d6f70f523d95d074b2eebab1fe1500..358a59406bb3a80f1c025f9c88ed80a9e1945009 100644 --- a/developer/src/kmc/src/messages/messages.ts +++ b/developer/src/kmc/src/messages/messages.ts @@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m const Namespace = CompilerErrorNamespace.Infrastructure; const SevInfo = CompilerErrorSeverity.Info | Namespace; -// const SevHint = CompilerErrorSeverity.Hint | Namespace; +const SevHint = CompilerErrorSeverity.Hint | Namespace; // const SevWarn = CompilerErrorSeverity.Warn | Namespace; const SevError = CompilerErrorSeverity.Error | Namespace; const SevFatal = CompilerErrorSeverity.Fatal | Namespace; @@ -39,5 +39,9 @@ export class InfrastructureMessages { static Error_InvalidProjectFile = (o:{message:string}) => m(this.ERROR_InvalidProjectFile, `Project file is not valid: ${o.message}`); static ERROR_InvalidProjectFile = SevError | 0x0008; + + static Hint_FilenameHasDifferingCase = (o:{reference:string, filename:string}) => m(this.HINT_FilenameHasDifferingCase, + `File ${o.filename} differs in case from reference ${o.reference}; this will fail on platforms with case-sensitive filesystems.`); + static HINT_FilenameHasDifferingCase = SevHint | 0x0009; } diff --git a/developer/src/kmc/test/fixtures/invalid-keyboards/hint_filename_has_differing_case.kmn b/developer/src/kmc/test/fixtures/invalid-keyboards/hint_filename_has_differing_case.kmn new file mode 100644 index 0000000000000000000000000000000000000000..5792016e10383d7bee622a0a49d6ec5e9ee03e09 --- /dev/null +++ b/developer/src/kmc/test/fixtures/invalid-keyboards/hint_filename_has_differing_case.kmn @@ -0,0 +1,6 @@ +store(&name) 'hint_filename_has_differing_case' +store(&version) '7.0' + +begin unicode > use(main) + +group(main) using keys diff --git a/developer/src/kmc/test/test-messages.ts b/developer/src/kmc/test/test-messages.ts index 7b591ad72de7a2e3dec348e55f52155c77030cbb..045718075daaf989c6871ede3648af8fe5e0d675 100644 --- a/developer/src/kmc/test/test-messages.ts +++ b/developer/src/kmc/test/test-messages.ts @@ -1,9 +1,80 @@ import 'mocha'; +import { assert } from 'chai'; import { InfrastructureMessages } from '../src/messages/messages.js'; import { verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; +import { makePathToFixture } from './helpers/index.js'; +import { NodeCompilerCallbacks } from '../src/messages/NodeCompilerCallbacks.js'; describe('InfrastructureMessages', function () { it('should have a valid InfrastructureMessages object', function() { return verifyCompilerMessagesObject(InfrastructureMessages); }); + + // + // Message tests + // + + /* + TODO: + + let callbacks = new TestCompilerCallbacks(); + + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number) { + context.timeout(10000); + + callbacks.clear(); + + const builder = new BuildKmnKeyboard(); + const path = makePathToFixture(...fixture); + let result = await builder.build(path, callbacks, { + compilerVersion: false, + compilerWarningsAsErrors: true, + debug: false, + warnDeprecatedCode: true, + }); + + if(messageId) { + assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); + assert.lengthOf(callbacks.messages, 1); + } else { + assert.lengthOf(callbacks.messages, 0, `messages should be empty, but instead got: `+JSON.stringify(callbacks.messages,null,2)); + assert.isTrue(result); + } + } + + // ERROR_FileDoesNotExist + + it('should generate ERROR_FileDoesNotExist if a file does not exist', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_file_does_not_exist.kmn'], CompilerMessages.ERROR_FileDoesNotExist); + }); + + // ERROR_FileTypeNotRecognized + + it('should generate ERROR_FileTypeNotRecognized if a file is not a recognized type', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_file_type_not_recognized.xxx'], CompilerMessages.ERROR_FileTypeNotRecognized); + }); + + // ERROR_OutFileNotValidForProjects + + it('should generate ERROR_OutFileNotValidForProjects if an output file is specified for a project build', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_out_file_not_valid_for_projects.kpj'], CompilerMessages.ERROR_OutFileNotValidForProjects); + }); + + // ERROR_InvalidProjectFile + + it('should generate ERROR_InvalidProjectFile if a project file is invalid', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_invalid_project_file.kpj'], CompilerMessages.ERROR_InvalidProjectFile); + }); + */ + + // HINT_FilenameHasDifferingCase + + it('should generate HINT_FilenameHasDifferingCase if a referenced file has differing case', async function() { + // This message is generated by NodeCompilerCallbacks, because that's where the filesystem is visible, + // so we can't use our usual testForMessage pattern. + const ncb = new NodeCompilerCallbacks(true); + ncb.loadFile(makePathToFixture('invalid-keyboards', 'Hint_Filename_Has_Differing_Case.kmn')); + assert.isTrue(ncb.hasMessage(InfrastructureMessages.HINT_FilenameHasDifferingCase), + `HINT_FilenameHasDifferingCase not generated, instead got: `+JSON.stringify(ncb.messages,null,2)); + }); }); diff --git a/developer/src/kmc/tsconfig.json b/developer/src/kmc/tsconfig.json index 9b6eb601acd121a775c5d7943a0c7477bb9e9fda..0493499987ecca63a769351bf2fd4577da8c060e 100644 --- a/developer/src/kmc/tsconfig.json +++ b/developer/src/kmc/tsconfig.json @@ -18,7 +18,7 @@ "src/**/*.ts" ], "references": [ - { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, + { "path": "../../../common/web/keyman-version" }, { "path": "../../../common/web/types" }, { "path": "../kmc-kmn" }, { "path": "../kmc-ldml" }, diff --git a/developer/src/kmcmpdll/Compiler.cpp b/developer/src/kmcmpdll/Compiler.cpp index 8cc28d2ab7f18bfdc5e6e65dd78b99bf1fe8afe4..100e43fc5bd4356d1fe7cd95b427b6e28a6afe6d 100644 --- a/developer/src/kmcmpdll/Compiler.cpp +++ b/developer/src/kmcmpdll/Compiler.cpp @@ -90,8 +90,6 @@ #include "UnreachableRules.h" #include "CheckForDuplicates.h" -#include "../kmcmplib/include/kmcmplibapi.h" - int xatoi(PWSTR *p); int atoiW(PWSTR p); void safe_wcsncpy(PWSTR out, PWSTR in, int cbMax); @@ -291,33 +289,15 @@ BOOL AddCompileMessage(DWORD msg) return FALSE; } -bool flag_use_new_kmcomp = true; // flag to switch to new kmcompx - extern "C" BOOL __declspec(dllexport) SetCompilerOptions(PCOMPILER_OPTIONS options) { if(!options || options->dwSize < sizeof(COMPILER_OPTIONS)) { return FALSE; } - flag_use_new_kmcomp = options->UseKmcmpLib; - - //printf("\n---> started in SetCompilerOptions() of kmcmpdll\n"); - if ( flag_use_new_kmcomp ) - { - KMCMP_COMPILER_OPTIONS kmcmp_options = {0}; - kmcmp_options.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); - kmcmp_options.ShouldAddCompilerVersion = options->ShouldAddCompilerVersion; - return kmcmp_SetCompilerOptions(&kmcmp_options); - } - //printf("---> stayed in SetCompilerOptions() of kmcmpdll\n"); - FShouldAddCompilerVersion = options->ShouldAddCompilerVersion; return TRUE; } -int kmcmpMsgproc(int line, uint32_t dwMsgCode, char* szText, void* context) { - return static_cast(context)(line, dwMsgCode, szText); -} - extern "C" BOOL __declspec(dllexport) CompileKeyboardFile(PSTR pszInfile, PSTR pszOutfile, BOOL ASaveDebug, BOOL ACompilerWarningsAsErrors, BOOL AWarnDeprecatedCode, CompilerMessageProc pMsgProc) // I4865 // I4866 { HANDLE hInfile = INVALID_HANDLE_VALUE, hOutfile = INVALID_HANDLE_VALUE; @@ -325,15 +305,6 @@ extern "C" BOOL __declspec(dllexport) CompileKeyboardFile(PSTR pszInfile, PSTR p DWORD len; char str[260]; - //printf("\n---> started in CompileKeyboardFile() of kmcmpdll\n"); - - if ( flag_use_new_kmcomp ) - { - return kmcmp_CompileKeyboardFile(pszInfile, pszOutfile, ASaveDebug, ACompilerWarningsAsErrors,AWarnDeprecatedCode, kmcmpMsgproc, (void*) pMsgProc); - } - - //printf("---> stayed in CompileKeyboardFile() of kmcmpdll\n"); - FSaveDebug = ASaveDebug; FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 FWarnDeprecatedCode = AWarnDeprecatedCode; // I4866 @@ -418,15 +389,6 @@ extern "C" BOOL __declspec(dllexport) CompileKeyboardFileToBuffer(PSTR pszInfile DWORD len; char str[260]; - //printf("\n---> started in CompileKeyboardFileToBuffer() of kmcmpdll\n"); - if ( flag_use_new_kmcomp ) - { - return kmcmp_CompileKeyboardFileToBuffer( pszInfile, (void*) pfkBuffer, ACompilerWarningsAsErrors, AWarnDeprecatedCode, kmcmpMsgproc, (void*) pMsgProc, Target); - } - - //printf("---> stayed in CompileKeyboardFileToBuffer() of kmcmpdll\n"); - - FSaveDebug = TRUE; // I3681 FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 FWarnDeprecatedCode = AWarnDeprecatedCode; // I4866 diff --git a/developer/src/kmcmpdll/json-validation.cpp b/developer/src/kmcmpdll/json-validation.cpp index e35dc0485dcae1480b00b1f002bad5fa75b56d26..c3c1af0484318b176cacef53dec22f58c7f2f118 100644 --- a/developer/src/kmcmpdll/json-validation.cpp +++ b/developer/src/kmcmpdll/json-validation.cpp @@ -1,14 +1,11 @@ #include #include -#include "../kmcmplib/include/kmcmplibapi.h" #include typedef bool (*kmcmp_ValidateJsonMessageProc)(int64_t offset, const char* szText, void* context); -extern bool flag_use_new_kmcomp; - using nlohmann::json; using nlohmann::json_uri; using nlohmann::json_schema_draft4::json_validator; @@ -35,13 +32,6 @@ bool kmcmpMessageProc(int64_t offset, const char* szText, void* context) { extern "C" BOOL __declspec(dllexport) ValidateJsonFile(PWSTR pwszSchemaFile, PWSTR pwszJsonFile, ValidateJsonMessageProc MessageProc) { - if ( flag_use_new_kmcomp ) - { - std::fstream f(pwszSchemaFile); - std::fstream fd(pwszJsonFile); - return kmcmp_ValidateJsonFile(f, fd, kmcmpMessageProc, (void*)MessageProc); - } - std::fstream f(pwszSchemaFile); if (!f.good()) { MessageProc(-1, "Schema file could not be loaded."); diff --git a/developer/src/kmcmpdll/kmcmpdll.vcxproj b/developer/src/kmcmpdll/kmcmpdll.vcxproj index c7f7e1f48f062c068d8cc15e1846f86f25942d9f..2bcc038bd00df813dc7fc9cd7db7a3f3b268d034 100644 --- a/developer/src/kmcmpdll/kmcmpdll.vcxproj +++ b/developer/src/kmcmpdll/kmcmpdll.vcxproj @@ -89,25 +89,25 @@ $(VC_IncludePath);$(WindowsSDK_IncludePath);$(KEYMAN_ROOT)\developer\src\common\include;$(KEYMAN_ROOT)\developer\src\ext\json;$(KEYMAN_ROOT)\developer\src\ext\json-schema-validator $(ProjectDir)bin\$(Platform)\$(Configuration)\ - $(KEYMAN_ROOT)\developer\src\kmcmplib\build\x86\$(Configuration)\src;$(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) + $(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) $(VC_IncludePath);$(WindowsSDK_IncludePath);$(KEYMAN_ROOT)\developer\src\common\include;$(KEYMAN_ROOT)\developer\src\ext\json;$(KEYMAN_ROOT)\developer\src\ext\json-schema-validator $(ProjectDir)obj\$(Platform)\$(Configuration)\ $(ProjectName).x64 $(ProjectDir)bin\$(Platform)\$(Configuration)\ - $(KEYMAN_ROOT)\developer\src\kmcmplib\build\x64\$(Configuration)\src;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) + $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) $(VC_IncludePath);$(WindowsSDK_IncludePath);$(KEYMAN_ROOT)\developer\src\common\include;$(KEYMAN_ROOT)\developer\src\ext\json;$(KEYMAN_ROOT)\developer\src\ext\json-schema-validator - $(KEYMAN_ROOT)\developer\src\kmcmplib\build\x86\$(Configuration)\src;$(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) + $(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) $(VC_IncludePath);$(WindowsSDK_IncludePath);$(KEYMAN_ROOT)\developer\src\common\include;$(KEYMAN_ROOT)\developer\src\ext\json;$(KEYMAN_ROOT)\developer\src\ext\json-schema-validator $(ProjectDir)obj\$(Platform)\$(Configuration)\ $(ProjectName).x64 $(ProjectDir)bin\$(Platform)\$(Configuration)\ - $(KEYMAN_ROOT)\developer\src\kmcmplib\build\x64\$(Configuration)\src;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) + $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) @@ -142,7 +142,7 @@ $(KEYMAN_ROOT)\windows\src\global\inc - libkmcmplib.a;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true true @@ -190,7 +190,7 @@ $(KEYMAN_ROOT)\windows\src\global\inc - libkmcmplib.a;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true true Windows @@ -233,7 +233,7 @@ $(KEYMAN_ROOT)\windows\src\global\inc - libkmcmplib.a;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true true true @@ -277,7 +277,7 @@ $(KEYMAN_ROOT)\windows\src\global\inc - libkmcmplib.a;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true true true diff --git a/developer/src/kmcmplib/checkout-keyboards.inc.sh b/developer/src/kmcmplib/checkout-keyboards.inc.sh index defcb479ef4070a759b2767115d1949e7cc78ac1..6c76d53caa1fc9f39eeef1a9bcdce0140c57ed86 100644 --- a/developer/src/kmcmplib/checkout-keyboards.inc.sh +++ b/developer/src/kmcmplib/checkout-keyboards.inc.sh @@ -9,8 +9,8 @@ function locate_keyboards_repo() { builder_die "keyboards_commit_ref.txt does not exist, run prep.sh" fi - if [[ ! -d "$THIS_SCRIPT_PATH/tests/fixtures" ]]; then - builder_die "fixtures folder does not exist, run prep.sh" + if [[ ! -d "$THIS_SCRIPT_PATH/tests/fixtures/keyboards-repo" ]]; then + builder_die "fixtures/keyboards-repo folder does not exist, run prep.sh" fi if [[ -z "${KEYBOARDS_ROOT+x}" ]]; then diff --git a/developer/src/kmcmplib/include/kmcmplibapi.h b/developer/src/kmcmplib/include/kmcmplibapi.h index 2fb35979b2def0839a4f36fa552c76abe8604cfe..0b41af69909254876d3a92c0d3bf61d37e143162 100644 --- a/developer/src/kmcmplib/include/kmcmplibapi.h +++ b/developer/src/kmcmplib/include/kmcmplibapi.h @@ -16,53 +16,52 @@ #define EXTERN EMSCRIPTEN_KEEPALIVE #endif -typedef struct _KMCMP_COMPILER_OPTIONS { - uint32_t dwSize; - bool ShouldAddCompilerVersion; -} KMCMP_COMPILER_OPTIONS; - -EXTERN bool kmcmp_SetCompilerOptions( - KMCMP_COMPILER_OPTIONS* options -); - -typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, char* szText, void* context); - -EXTERN bool kmcmp_CompileKeyboardFile( - char* pszInfile, - char* pszOutfile, - bool ASaveDebug, - bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, - kmcmp_CompilerMessageProc pMsgproc, - void* AmsgprocContext -); - /* Compile target */ #define CKF_KEYMAN 0 #define CKF_KEYMANWEB 1 -EXTERN bool kmcmp_CompileKeyboardFileToBuffer( - char* pszInfile, - void* pfkBuffer, - bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, - kmcmp_CompilerMessageProc pMsgproc, - void* AmsgprocContext, - int Target -); - -typedef bool (*kmcmp_ValidateJsonMessageProc)(int64_t offset, const char* szText, void* context); - -EXTERN bool kmcmp_ValidateJsonFile( - std::fstream& f, - std::fstream& fd, - kmcmp_ValidateJsonMessageProc MessageProc, - void* context +struct KMCMP_COMPILER_OPTIONS { + bool saveDebug; + bool compilerWarningsAsErrors; + bool warnDeprecatedCode; + bool shouldAddCompilerVersion; + int target; // CKF_KEYMAN, CKF_KEYMANWEB +}; + +struct KMCMP_COMPILER_RESULT { + void* kmx; + size_t kmxSize; + std::string kvksFilename; +}; + +// TODO: parameters in UTF-8 #8887 +typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, const char* szText, void* context); + +// parameters in UTF-8 +// TODO typical usage: +// if(!kmcmp_LoadFileProc("filename.ico", "/tmp/filename.kmn", nullptr, &size)) { +// return error; +// } +// buf = new unsigned char[size]; +// if(!kmcmp_LoadFileProc("filename.ico", "/tmp/filename.kmn", buf, &size)) { +// delete[] buf; +// return error; +// } +typedef bool (*kmcmp_LoadFileProc)(const char* loadFilename, const char* baseFilename, void* buffer, int* bufferSize, void* context); + +// Parameters in UTF-8 +EXTERN bool kmcmp_CompileKeyboard( + const char* pszInfile, + const KMCMP_COMPILER_OPTIONS& options, + kmcmp_CompilerMessageProc messageProc, + kmcmp_LoadFileProc loadFileProc, + const void* procContext, + KMCMP_COMPILER_RESULT& result ); /** - * kmcmp_ParseUnicodeSet is successful if it returns >= USET_OK + * kmcmp_parseUnicodeSet is successful if it returns >= USET_OK */ static const int KMCMP_USET_OK = 0; @@ -83,24 +82,19 @@ static const int KMCMP_ERROR_UNSUPPORTED_PROPERTY = -3; */ static const int KMCMP_FATAL_OUT_OF_RANGE = -4; -/** - * Function pointer to kmcmp_ParseUnicodeSet - */ -typedef int (*kmcmp_ParseUnicodeSetProc)(const char* szText, uint32_t* output, uint32_t outputLength); - /** * Parse a UnicodeSet into 32-bit ranges. * For example, "[]" will return 0 (KMCMP_USET_OK) as a zero-length set. * "[" will return KMCMP_ERROR_SYNTAX_ERR, * and "[x A-C]" will return 2 and [0x41, 0x43, 0x78, 0x78] - * @param szText input txt, null terminated, in UTF-8 format + * @param text input txt, null terminated, in UTF-8 format * @param outputBuffer output buffer, owned by caller: Pairs of ranges in order * @param outputBufferSize length of output buffer. Needs to be twice the number of expected ranges * @return If >= KMCMP_USET_OK, number of ranges, otherwise one of the negative error values. */ -EXTERN int kmcmp_ParseUnicodeSet( - const char* szText, - uint32_t* outputBuffer, +EXTERN int kmcmp_parseUnicodeSet( + const std::string text, + uintptr_t outputBuffer_, uint32_t outputBufferSize ); diff --git a/developer/src/kmcmplib/include/meson.build b/developer/src/kmcmplib/include/meson.build index f98a4d8c762da2134f33054a4b18948aa3a5e658..dde8e69f6b0cd52d1c8f1ce66f2a65c413e3809f 100644 --- a/developer/src/kmcmplib/include/meson.build +++ b/developer/src/kmcmplib/include/meson.build @@ -20,7 +20,5 @@ configure_file( inc = include_directories( '.', '../../common/include', - '../../../../common/include', - '../../ext/json', - '../../ext/json-schema-validator' + '../../../../common/include' ) diff --git a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp b/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp deleted file mode 100644 index 627298bf375de6155aeae55fb1ddff96a157df14..0000000000000000000000000000000000000000 --- a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp +++ /dev/null @@ -1,145 +0,0 @@ - -#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING 1 -#include "pch.h" -#include "compfile.h" -#include -#include "kmcmplib.h" -#include -#include "CheckFilenameConsistency.h" -#include "kmx_u16.h" -#include "filesystem.h" - -#ifdef _MSC_VER -#include -#endif - -namespace kmcmp { - extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer -} -bool IsRelativePath(KMX_CHAR const * p) { - // Relative path (returns TRUE): - // ..\...\BITMAP.BMP - // PATH\BITMAP.BMP - // BITMAP.BMP - - // Semi-absolute path (returns FALSE): - // \...\BITMAP.BMP - - // Absolute path (returns FALSE): - // C:\...\BITMAP.BMP - // \\SERVER\SHARE\...\BITMAP.BMP - - if ((*p == '\\') || (*p == '/')) return FALSE; - if (*p && *(p + 1) == ':') return FALSE; - - return TRUE; -} - -bool IsRelativePath(KMX_WCHAR const * p) { - // Relative path (returns TRUE): - // ..\...\BITMAP.BMP - // PATH\BITMAP.BMP - // BITMAP.BMP - - // Semi-absolute path (returns FALSE): - // \...\BITMAP.BMP - - // Absolute path (returns FALSE): - // C:\...\BITMAP.BMP - // \\SERVER\SHARE\...\BITMAP.BMP - - if ((*p == u'\\') || (*p == u'/'))return FALSE; - if (*p && *(p + 1) == u':') return FALSE; - - return TRUE; -} - -KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissingFile) { - PKMX_WCHAR WFilename = strtowstr(( KMX_CHAR *)Filename); - KMX_DWORD const result = CheckFilenameConsistency(WFilename, ReportMissingFile); - delete WFilename; - return result; -} - -KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile) { - // not ready yet: needs more attention-> common includes for non-Windows platforms - KMX_WCHAR Name[260]; // TODO: fixed buffer sizes bad - - - if (IsRelativePath(Filename)) { - PKMX_WCHAR WCompileDir = strtowstr(kmcmp::CompileDir); - u16ncpy(Name, WCompileDir, _countof(Name)); // I3481 - u16ncat(Name, Filename, _countof(Name)); // I3481 - delete[] WCompileDir; - } else { - u16ncpy(Name, Filename, _countof(Name)); // I3481 - } - -#ifndef _MSC_VER - // Filename consistency only needs to be checked on Windows, because other - // platforms are going to fail if the filename is inconsistent anyway! - if(!kmcmp_FileExists(Name)) { - if (ReportMissingFile) { - u16cpy(ErrExtraW, u"referenced file '"); - u16ncat(ErrExtraW, Filename, 256); - u16ncat(ErrExtraW, u"'", 256); - strcpy(ErrExtraLIB, string_from_u16string(ErrExtraW).c_str()); - AddWarning(CWARN_MissingFile); - } - return CERR_None; - } - return CERR_None; -#else - _wfinddata_t fi; - intptr_t n; - if ((n = _wfindfirst((const wchar_t*) Name, &fi)) == -1) { - if (ReportMissingFile) { - sprintf(ErrExtraLIB, "referenced file '%ls'", (wchar_t*) Filename); - AddWarning(CWARN_MissingFile); - } - return CERR_None; - } - - _findclose(n); - - KMX_WCHAR FName[_MAX_FNAME], Ext[_MAX_EXT]; - wchar_t WChName[_MAX_PATH]; - _wsplitpath_s((const wchar_t*)Filename, nullptr, 0, nullptr, 0, (wchar_t*) FName, _MAX_FNAME, (wchar_t*) Ext, _MAX_EXT); - _wmakepath_s(WChName, _MAX_PATH, nullptr, nullptr, (const wchar_t*) FName, (const wchar_t*) Ext); - if (wcscmp(WChName, fi.name) != 0) { - sprintf(ErrExtraLIB, "reference '%ls' does not match actual filename '%ls'", WChName, fi.name); - - AddWarning(CHINT_FilenameHasDifferingCase); - - } -#endif - - return CERR_None; -} - -KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk) { - // call() statements depend on a fairly ugly hack for js, - // where store(DllFunction) "my.dll:func" will look for a - // file called function.call_js. ( or should this be func.call_js ? ) - // This is ripe for rewrite! - // But let's check what we have anyway - - PFILE_STORE sp; - KMX_DWORD i, msg; - for (i = 0, sp = fk->dpStoreArray; i < fk->cxStoreArray; i++, sp++) { - if (!sp->fIsCall) continue; - - const std::u16string callsite(sp->dpString); - const auto colon = callsite.find(':'); - if (colon == std::u16string::npos) continue; - - auto func1 = callsite.substr(colon + 1); - std::u16string str_js(u".call_js"); - std::u16string func = func1+ str_js; - - if ((msg = CheckFilenameConsistency(func.c_str(), FALSE)) != CERR_None) { - return msg; - } - } - return CERR_None; -} diff --git a/developer/src/kmcmplib/src/CheckFilenameConsistency.h b/developer/src/kmcmplib/src/CheckFilenameConsistency.h deleted file mode 100644 index 39f86db391b1ac6b870d7a521f297372059bd9b4..0000000000000000000000000000000000000000 --- a/developer/src/kmcmplib/src/CheckFilenameConsistency.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "compfile.h" -#include "kmcmplib.h" - -KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk); -KMX_DWORD CheckFilenameConsistency(KMX_CHAR const * Filename, bool ReportMissingFile); -KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile); - -bool IsRelativePath(KMX_CHAR const * p); -bool IsRelativePath(KMX_WCHAR const * p); diff --git a/developer/src/kmcmplib/src/CompMsg.cpp b/developer/src/kmcmplib/src/CompMsg.cpp index 2fa0d68322dc1217207bb8541a62e304fee40f76..dd41336a0e9d262a96a0b4026cbaac54b80683a3 100644 --- a/developer/src/kmcmplib/src/CompMsg.cpp +++ b/developer/src/kmcmplib/src/CompMsg.cpp @@ -111,7 +111,6 @@ const struct CompilerError CompilerErrors[] = { { CERR_DuplicateStore , "A store with this name has already been defined."}, { CERR_RepeatedBegin , "Begin has already been set"}, - { CHINT_FilenameHasDifferingCase , "Casing differences may fail on some platforms: "}, { CHINT_UnreachableRule , "This rule will never be matched as another rule takes precedence"}, { CWARN_TooManyWarnings , "Too many warnings or errors"}, @@ -142,7 +141,6 @@ const struct CompilerError CompilerErrors[] = { { CWARN_NulNotFirstStatementInContext , "nul must be the first statement in the context"}, { CWARN_IfShouldBeAtStartOfContext , "if, platform and baselayout should be at start of context (after nul, if present)"}, { CWARN_KeyShouldIncludeNCaps , "Other rules which reference this key include CAPS or NCAPS modifiers, so this rule must include NCAPS modifier to avoid inconsistent matches"}, - { CWARN_MissingFile , "The referenced file could not be found: "}, { 0, nullptr } }; diff --git a/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp b/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e90938f18a735ffca4ab986ee53004bb3703a62a --- /dev/null +++ b/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp @@ -0,0 +1,160 @@ +#include "pch.h" +#include +#include "kmcmplib.h" +#include "CheckNCapsConsistency.h" +#include "DeprecationChecks.h" +#include "versioning.h" +#include "CompileKeyboardBuffer.h" +#include "../../../../common/windows/cpp/include/keymanversion.h" + +bool CompileKeyboardBuffer(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk) +{ + PKMX_WCHAR str, p; + + KMX_DWORD msg; + + kmcmp::FMnemonicLayout = FALSE; + + if (!fk) { + AddCompileError(CERR_SomewhereIGotItWrong); + return FALSE; + } + + str = new KMX_WCHAR[LINESIZE]; + if (!str) { + AddCompileError(CERR_CannotAllocateMemory); + return FALSE; + } + + fk->KeyboardID = 0; + fk->version = 0; + fk->dpStoreArray = NULL; + fk->dpGroupArray = NULL; + fk->cxStoreArray = 0; + fk->cxGroupArray = 0; + fk->StartGroup[0] = fk->StartGroup[1] = -1; + fk->szName[0] = 0; + fk->szCopyright[0] = 0; + fk->dwFlags = KF_AUTOMATICVERSION; + fk->currentGroup = 0xFFFFFFFF; + fk->currentStore = 0; + fk->cxDeadKeyArray = 0; + fk->dpDeadKeyArray = NULL; + fk->cxVKDictionary = 0; // I3438 + fk->dpVKDictionary = NULL; // I3438 + fk->extra->kvksFilename = u""; +/* fk->szMessage[0] = 0; + fk->szLanguageName[0] = 0;*/ + fk->dwBitmapSize = 0; + fk->dwHotKey = 0; + + kmcmp::BeginLine[BEGIN_ANSI] = -1; + kmcmp::BeginLine[BEGIN_UNICODE] = -1; + kmcmp::BeginLine[BEGIN_NEWCONTEXT] = -1; + kmcmp::BeginLine[BEGIN_POSTKEYSTROKE] = -1; + + + /* Add a store for the Keyman 6.0 copyright information string */ + + if(kmcmp::FShouldAddCompilerVersion) { + u16sprintf(str,LINESIZE, L"Created with Keyman Developer version %d.%d.%d.%d", KEYMAN_VersionMajor, KEYMAN_VersionMinor, KEYMAN_VersionPatch, 0); + AddStore(fk, TSS_KEYMANCOPYRIGHT, str); + } + + /* Add a system store for the Keyman edition number */ + AddStore(fk, TSS_CUSTOMKEYMANEDITION, u"0"); + AddStore(fk, TSS_CUSTOMKEYMANEDITIONNAME, u"Keyman"); + + int offset = 0; + + // must preprocess for group and store names -> this isn't really necessary, but never mind! + while ((msg = ReadLine(infile, sz, offset, str, TRUE)) == CERR_None) + { + p = str; + switch (LineTokenType(&p)) + { + case T_VERSION: + *(p + 4) = 0; + if ((msg = AddStore(fk, TSS_VERSION, p)) != CERR_None) { + AddCompileError(msg); + return FALSE; + } + break; + + case T_GROUP: + if ((msg = ProcessGroupLine(fk, p)) != CERR_None) { + AddCompileError(msg); + return FALSE; + } + break; + + case T_STORE: + if ((msg = ProcessStoreLine(fk, p)) != CERR_None) { + AddCompileError(msg); + return FALSE; + } + break; + + default: + break; + } + } + + if (msg != CERR_EndOfFile) { + AddCompileError(msg); + return FALSE; + } + + offset = 0; + kmcmp::currentLine = 0; + + /* Reindex the list of codeconstants after stores added */ + + kmcmp::CodeConstants->reindex(); + + /* ReadLine will automatically skip over $Keyman lines, and parse wrapped lines */ + while ((msg = ReadLine(infile, sz, offset, str, FALSE)) == CERR_None) + { + msg = ParseLine(fk, str); + if (msg != CERR_None) { + AddCompileError(msg); + return FALSE; + } + } + + if (msg != CERR_EndOfFile) { + AddCompileError(msg); + return FALSE; + } + + ProcessGroupFinish(fk); + + if (kmcmp::FSaveDebug) kmcmp::RecordDeadkeyNames(fk); + + /* Add the compiler version as a system store */ + if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) { + AddCompileError(msg); + return FALSE; + } + + if ((msg = BuildVKDictionary(fk)) != CERR_None) { + AddCompileError(msg); + return FALSE; + } + + delete str; + + if (!kmcmp::CheckKeyboardFinalVersion(fk)) { + return FALSE; + } + + /* Warn on inconsistent use of NCAPS */ + if (!kmcmp::FMnemonicLayout) { + CheckNCapsConsistency(fk); + } + + /* Flag presence of deprecated features */ + kmcmp::CheckForDeprecatedFeatures(fk); + + return TRUE; +} diff --git a/developer/src/kmcmplib/src/CompileKeyboardBuffer.h b/developer/src/kmcmplib/src/CompileKeyboardBuffer.h new file mode 100644 index 0000000000000000000000000000000000000000..78bac6665399d721b52baf563c62de3732a570fb --- /dev/null +++ b/developer/src/kmcmplib/src/CompileKeyboardBuffer.h @@ -0,0 +1,5 @@ +#pragma once + +#include "compfile.h" + +bool CompileKeyboardBuffer(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk); diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index b09e36cbc1079fdf47feeb9ac882eaf640b3f425..e5d5f94cbc7d0381c8aaec67f59cdbdd43379035 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -94,11 +94,12 @@ #include "CasedKeys.h" #include #include -#include "CheckFilenameConsistency.h" +#include +#include + #include "UnreachableRules.h" #include "CheckForDuplicates.h" #include "kmx_u16.h" -#include "filesystem.h" #include /* These macros are adapted from winnt.h and legacy use only */ @@ -122,7 +123,6 @@ namespace kmcmp{ KMX_BOOL FMnemonicLayout = FALSE; KMX_BOOL FOldCharPosMatching = FALSE; int CompileTarget; - KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer int BeginLine[4]; KMX_BOOL IsValidCallStore(PFILE_STORE fs); @@ -230,6 +230,8 @@ enum LinePrefixType { lptNone, lptKeymanAndKeymanWeb, lptKeymanWebOnly, lptKeyma /* Compile target */ kmcmp_CompilerMessageProc msgproc = NULL; +kmcmp_LoadFileProc loadfileproc = NULL; + void* msgprocContext = NULL; int kmcmp::currentLine = 0; @@ -862,7 +864,6 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE int i, j; KMX_DWORD msg; PKMX_WCHAR p, q; - KMX_CHAR *pp; if (!pssBuf) pssBuf = new KMX_WCHAR[GLOBAL_BUFSIZE]; PKMX_WCHAR buf = pssBuf; @@ -912,13 +913,9 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE case TSS_INCLUDECODES: VERIFY_KEYBOARD_VERSION(fk, VERSION_60, CERR_60FeatureOnly_NamedCodes); - pp = wstrtostr(sp->dpString); - if (!kmcmp::CodeConstants->LoadFile(pp)) - { - delete[] pp; + if (!kmcmp::CodeConstants->LoadFile(fk, sp->dpString)) { return CERR_CannotLoadIncludeFile; } - delete[] pp; kmcmp::CodeConstants->reindex(); // I4982 break; @@ -1047,10 +1044,6 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE delete[] sp->dpString; sp->dpString = q; - - if ((msg = CheckFilenameConsistency( (sp->dpString), FALSE)) != CERR_None) { - return msg; - } } break; case TSS_KMW_RTL: @@ -1061,16 +1054,10 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE case TSS_KMW_HELPFILE: case TSS_KMW_EMBEDJS: VERIFY_KEYBOARD_VERSION(fk, VERSION_70, CERR_70FeatureOnly); - if ((msg = CheckFilenameConsistency(sp->dpString, FALSE)) != CERR_None) { - return msg; - } break; case TSS_KMW_EMBEDCSS: VERIFY_KEYBOARD_VERSION(fk, VERSION_90, CERR_90FeatureOnlyEmbedCSS); - if ((msg = CheckFilenameConsistency(sp->dpString, FALSE)) != CERR_None) { - return msg; - } break; case TSS_TARGETS: // I4504 @@ -1120,9 +1107,6 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE case TSS_LAYOUTFILE: // I3483 VERIFY_KEYBOARD_VERSION(fk, VERSION_90, CERR_90FeatureOnlyLayoutFile); // I4140 - if ((msg = CheckFilenameConsistency(sp->dpString, FALSE)) != CERR_None) { - return msg; - } // Used by KMW compiler break; @@ -3068,7 +3052,7 @@ KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, KMX_BYTE**data, size_t& dataS return CERR_None; } -KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) +KMX_DWORD ReadLine(KMX_BYTE* infile, int sz, int& offset, PKMX_WCHAR wstr, KMX_BOOL PreProcess) { KMX_DWORD len; PKMX_WCHAR p; @@ -3076,21 +3060,26 @@ KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) KMX_DWORD n; KMX_WCHAR currentQuotes = 0; KMX_WCHAR str[LINESIZE + 3]; - len = (KMX_DWORD)fread( str , 1 ,LINESIZE * 2,fp_in); - if (ferror(fp_in) ) return CERR_CannotReadInfile; - len /= 2; - str[len] = 0; auto cur = ftell(fp_in); - fseek(fp_in, 0, SEEK_END); - auto fsize = ftell(fp_in); - fseek(fp_in, cur, SEEK_SET); - if (cur == fsize) + if(offset >= sz) { + return CERR_EndOfFile; + } - // \r\n is still added here even though Linux doesn`t use \r. - // This is to ensure to still have a working windows-only-version - u16ncat(str, u"\r\n", _countof(str)); // I3481 // Always a "\r\n" to the EOF, avoids funny bugs + len = offset + LINESIZE*2 > sz ? sz-offset : LINESIZE*2; + memcpy(str, infile+offset, len); + offset += len; + len /= 2; + str[len] = 0; - if (len == 0) return CERR_EndOfFile; + if(offset == sz) { + // \r\n is still added here even though Linux doesn`t use \r. + // This is to ensure to still have a working windows-only-version + u16ncat(str, u"\r\n", _countof(str)); // I3481 // Always a "\r\n" to the EOF, avoids funny bugs + } + + if (len == 0) { + return CERR_EndOfFile; + } // neccessary to add this block for using on non-windows platforms (removes all \r for platforms that use \n instead of \r\n) for (p = str, n = 0; n < len; n++, p++) { @@ -3172,9 +3161,13 @@ KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) return (PreProcess ? CERR_None : CERR_LineTooLong); } - if (*p == L'\n') kmcmp::currentLine++; + kmcmp::currentLine++; - fseek(fp_in, -(int)(len * 2 - (int)(p - str) * 2 - 2), SEEK_CUR); + offset -= (int)(len * 2 - (int)(p - str) * 2 - 2); + if(offset >= sz) { + // If we've appended a \n, we can go past EOF + offset = sz; + } p--; while (p >= str && iswspace(*p)) p--; @@ -3220,58 +3213,38 @@ KMX_BOOL IsSameToken(PKMX_WCHAR *p, KMX_WCHAR const * token) return FALSE; } -KMX_DWORD ImportBitmapFile(PFILE_KEYBOARD fk, PKMX_WCHAR szName, PKMX_DWORD FileSize, PKMX_BYTE *Buf) +static bool endsWith(const std::string& str, const std::string& suffix) { - FILE *fp; - KMX_WCHAR szNewName[260]; - - if (IsRelativePath(szName)) - { - PKMX_WCHAR WCompileDir = strtowstr(kmcmp::CompileDir); - u16ncpy(szNewName, WCompileDir, _countof(szNewName)); // I3481 - u16ncat(szNewName,szName, _countof(szNewName )); // I3481 - } - else - u16ncpy(szNewName, szName, _countof(szNewName)); // I3481 - - fp=Open_File(szNewName, u"rb"); - - if ( fp == NULL) - { - // else if filename.bmp is not in the folder -> attempt to open filename.bmp.bmp ! - if ( u16cmp(szNewName+u16len(szNewName)-4, u".bmp") ) - u16ncat(szNewName, u".bmp", _countof(szNewName)); // I3481 + return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix); +} - fp= Open_File(szNewName, u"rb"); +KMX_DWORD ImportBitmapFile(PFILE_KEYBOARD fk, PKMX_WCHAR szName, PKMX_DWORD FileSize, PKMX_BYTE *Buf) +{ + auto szNameUtf8 = string_from_u16string(szName); - if ( fp == NULL) + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) { + // Append .bmp and try again + if(endsWith(szNameUtf8, ".bmp")) { return CERR_CannotReadBitmapFile; + } + szNameUtf8.append(".bmp"); + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) { + return CERR_CannotReadBitmapFile; + } } - KMX_DWORD msg; - if ((msg = CheckFilenameConsistency(szNewName, FALSE)) != CERR_None) { - return msg; - } - - fseek(fp, 0, SEEK_END); - *FileSize = (KMX_DWORD)ftell(fp); - fseek(fp ,0,SEEK_SET); - if (*FileSize < 0) { - fclose(fp); + if(*FileSize < 2) { + // Zero-byte file is invalid; 2 byte file is too, but we only really care + // about the prolog at this point so we don't overrun our buffer return CERR_CannotReadBitmapFile; } - if (*FileSize < 2) return CERR_CannotReadBitmapFile; *Buf = new KMX_BYTE[*FileSize]; - - if (fread(*Buf, 1, *FileSize, fp) < (size_t) *FileSize) { - delete[] * Buf; - *Buf = NULL; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), *Buf, (int*) FileSize, msgprocContext)) { + delete[] *Buf; return CERR_CannotReadBitmapFile; } - fclose(fp); - /* Test for version 7.0 icon support */ if (*((PKMX_CHAR)*Buf) != 'B' && *(((PKMX_CHAR)*Buf) + 1) != 'M') { VERIFY_KEYBOARD_VERSION(fk, VERSION_70, CERR_70FeatureOnly); @@ -3427,81 +3400,54 @@ KMX_BOOL kmcmp::IsValidCallStore(PFILE_STORE fs) return i == 1; } -FILE* CreateTempFile() -{ - return tmpfile(); +/////////////////// + +bool hasPreamble(std::u16string result) { + return result.size() > 0 && result[0] == 0xFEFF; } -/////////////////// +#include "unicode/ucnv.h" -FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble) -{ - FILE *fp_out = CreateTempFile(); - if(fp_out == NULL) // I3228 // I3510 - { - fclose(fp_in); - return NULL; //return INVALID_HANDLE_VALUE; _S2 can I exchange that? +bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) { + if(sz == 0) { + return FALSE; } - PKMX_BYTE buf, p; - PKMX_WCHAR outbuf, poutbuf; - KMX_DWORD len; - KMX_DWORD len2; - KMX_WCHAR prolog = 0xFEFF; - fwrite(&prolog,2, 1, fp_out); - - fseek(fp_in, 0, SEEK_END); - len = (KMX_DWORD)ftell(fp_in); - fseek(fp_in, 0, SEEK_SET); - if (hasPreamble) { - fseek( fp_in,3,SEEK_SET); // Cut off UTF-8 marker - len -= 3; - } - - buf = new KMX_BYTE[len + 1]; // null terminated - outbuf = new KMX_WCHAR[len + 1]; - - len2= (KMX_DWORD)fread(buf,1,len,fp_in); - if (len2) { - buf[len2] = 0; - p = buf; - poutbuf = outbuf; - if (hasPreamble) { - // We have a preamble, so we attempt to read as UTF-8 and allow conversion errors to be filtered. This is not great for a - // compiler but matches existing behaviour -- in future versions we may not do lenient conversion. - ConvertUTF8toUTF16(&p, &buf[len2], (UTF16 **)&poutbuf, (const UTF16 *)&outbuf[len], lenientConversion); - fwrite(outbuf, (KMX_DWORD)(poutbuf - outbuf) * 2 , 1, fp_out); + std::u16string result; + + try { + std::wstring_convert, char16_t> converter; + result = converter.from_bytes((char*)infile, (char*)infile+sz); + } catch(std::range_error e) { + UErrorCode status = U_ZERO_ERROR; + // TODO: we need ICU data files here #8884 + UConverter* conv = ucnv_open("windows-1252", &status); + if(U_FAILURE(status)) { + return FALSE; } - else { - // No preamble, so we attempt to read as strict UTF-8 and fall back to ANSI if that fails - ConversionResult cr = ConvertUTF8toUTF16(&p, &buf[len2], (UTF16 **)&poutbuf, (const UTF16 *)&outbuf[len], strictConversion); - if (cr == sourceIllegal) { - // Not a valid UTF-8 file, so fall back to ANSI - // AddCompileError(CHINT_NonUnicodeFile); - // note, while this message is defined, for now we will not emit it - // because we don't support HINT/INFO messages yet and we don't want - // this to cause a blocking compile at this stage - // do strtowstr only when no invalid characters are found - if( p==0){ - poutbuf = strtowstr((PKMX_STR)buf); - fwrite(poutbuf, (KMX_DWORD)u16len(poutbuf) * 2 , 1, fp_out); - delete[] poutbuf; - } - else - AddCompileError(CERR_InvalidCharacter); - } - else { - fwrite(outbuf, (KMX_DWORD)(poutbuf - outbuf) * 2 , 1, fp_out); - } + char16_t* dest = new char16_t[sz*2]; + ucnv_toUChars(conv, dest, sz*2, (char*)infile, sz, &status); + if(U_FAILURE(status)) { + delete[] dest; + return FALSE; } + + result = dest; + delete[] dest; } - fclose( fp_in); - delete[] buf; - delete[] outbuf; - fseek( fp_out,2,SEEK_SET); - return fp_out; + if(hasPreamble(result)) { + *sz16 = result.size() * 2 - 2; + *tempfile = new KMX_BYTE[*sz16]; + memcpy(*tempfile, result.c_str() + 1, *sz16); + } else { + *sz16 = result.size() * 2; + *tempfile = new KMX_BYTE[*sz16]; + memcpy(*tempfile, result.c_str(), *sz16); + } + + return TRUE; } PFILE_STORE FindSystemStore(PFILE_KEYBOARD fk, KMX_DWORD dwSystemID) diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp index a1609d35449562695c622e689649f962f6f6c57f..fdfbdf19613cc26d3de1161ca47e2dfdeb9c7bd7 100644 --- a/developer/src/kmcmplib/src/CompilerInterfaces.cpp +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -1,453 +1,110 @@ #include "pch.h" - #include #include #include "kmcmplib.h" -#include "filesystem.h" -#include "CheckFilenameConsistency.h" -#include "CheckNCapsConsistency.h" -#include "DeprecationChecks.h" -#include "versioning.h" #include "../../../../common/windows/cpp/include/ConvertUTF.h" -#include "../../../../common/windows/cpp/include/keymanversion.h" - -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); -bool CompileKeyboard(const char* pszInfile, - void* pfkBuffer, bool ASaveDebug, bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, const void* AmsgprocContext, - int Target); - -EXTERN bool kmcmp_SetCompilerOptions(KMCMP_COMPILER_OPTIONS* options) { - //printf("°°-> changed to SetCompilerOptions() of kmcmplib \n"); - if(!options || options->dwSize < sizeof(KMCMP_COMPILER_OPTIONS)) { - return FALSE; - } - kmcmp::FShouldAddCompilerVersion = options->ShouldAddCompilerVersion; - return TRUE; -} - -#ifdef __EMSCRIPTEN__ - -/* - WASM interface for compiler message callback -*/ -EM_JS(int, wasm_msgproc, (int line, int msgcode, char* text, char* context), { - const proc = globalThis[UTF8ToString(context)]; - if(!proc || typeof proc != 'function') { - console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); - return 0; - } else { - return proc(line, msgcode, UTF8ToString(text)); - } -}); - -int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, char* szText, void* context) { - char* msgProc = static_cast(context); - return wasm_msgproc(line, dwMsgCode, szText, msgProc); -} - -//DEPRECATED -EXTERN bool kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { - KMCMP_COMPILER_OPTIONS options; - options.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); - options.ShouldAddCompilerVersion = ShouldAddCompilerVersion; - return kmcmp_SetCompilerOptions(&options); -} - -//DEPRECATED -EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, - char* pszOutfile, int ASaveDebug, int ACompilerWarningsAsErrors, - int AWarnDeprecatedCode, char* msgProc +#include "CompileKeyboardBuffer.h" + +EXTERN bool kmcmp_CompileKeyboard( + const char* pszInfile, + const KMCMP_COMPILER_OPTIONS& options, + kmcmp_CompilerMessageProc messageProc, + kmcmp_LoadFileProc loadFileProc, + const void* procContext, + KMCMP_COMPILER_RESULT& result ) { - return kmcmp_CompileKeyboardFile( - pszInfile, - pszOutfile, - ASaveDebug, - ACompilerWarningsAsErrors, - AWarnDeprecatedCode, - wasm_CompilerMessageProc, - msgProc - ); -} - -EXTERN int kmcmp_Wasm_ParseUnicodeSet(char* pat, - uint32_t* buf, int length -) { - return kmcmp_ParseUnicodeSet( - pat, buf, length - ); -} - - -struct COMPILER_INTERFACE { - bool saveDebug; - bool compilerWarningsAsErrors; - bool warnDeprecatedCode; - bool shouldAddCompilerVersion; - int target; // CKF_KEYMAN, CKF_KEYMANWEB - std::string messageCallback; // int line, uint32_t dwMsgCode, char* szText - std::string loadFileCallback; // char* infile, char* filenameRelativeToInfile --> buffer -}; - -struct COMPILER_RESULT { - bool result; - // Following are pointer offsets in heap + buffer size - int kmx; - int kmxSize; - // Following are compiler side-channel data, required for - // follow-on transform - std::string kvksFilename; - // TODO: additional data to be passed back -}; - -COMPILER_RESULT kmcmp_compile(std::string pszInfile, const COMPILER_INTERFACE intf) { - COMPILER_RESULT r = {false}; FILE_KEYBOARD fk; + fk.extra = new FILE_KEYBOARD_EXTRA; + fk.extra->kmnFilename = pszInfile; - // TODO: this should be included in CompileKeyboard? - kmcmp::FShouldAddCompilerVersion = intf.shouldAddCompilerVersion; - - r.result = CompileKeyboard( - pszInfile.c_str(), - &fk, - intf.saveDebug, - intf.compilerWarningsAsErrors, - intf.warnDeprecatedCode, - wasm_CompilerMessageProc, - intf.messageCallback.c_str(), - intf.target); - - if(!r.result) { - return r; - } - - KMX_DWORD msg; - KMX_BYTE* data = nullptr; - size_t dataSize = 0; - msg = WriteCompiledKeyboard(&fk, &data, dataSize); - //TODO: FreeKeyboardPointers(fk); - - if(msg != CERR_None) { - AddCompileError(msg); - r.result = FALSE; - return r; - } - - r.kmx = (int) data; - r.kmxSize = (int) dataSize; - r.kvksFilename = string_from_u16string(fk.extra->kvksFilename); // convert to UTF8 - - return r; -} - -EMSCRIPTEN_BINDINGS(compiler_interface) { - emscripten::class_("CompilerInterface") - .constructor<>() - .property("saveDebug", &COMPILER_INTERFACE::saveDebug) - .property("compilerWarningsAsErrors", &COMPILER_INTERFACE::compilerWarningsAsErrors) - .property("warnDeprecatedCode", &COMPILER_INTERFACE::warnDeprecatedCode) - .property("shouldAddCompilerVersion", &COMPILER_INTERFACE::shouldAddCompilerVersion) - .property("target", &COMPILER_INTERFACE::target) - .property("messageCallback", &COMPILER_INTERFACE::messageCallback) - .property("loadFileCallback", &COMPILER_INTERFACE::loadFileCallback) - ; - - emscripten::class_("CompilerResult") - .constructor<>() - .property("result", &COMPILER_RESULT::result) - .property("kmx", &COMPILER_RESULT::kmx) - .property("kmxSize", &COMPILER_RESULT::kmxSize) - .property("kvksFilename", &COMPILER_RESULT::kvksFilename) - ; - - emscripten::function("kmcmp_compile", &kmcmp_compile); -} - -#endif - -bool CompileKeyboard(const char* pszInfile, - void* pfkBuffer, bool ASaveDebug, bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, const void* AmsgprocContext, - int Target) { - - FILE* fp_in = NULL; - KMX_CHAR str[260]; + kmcmp::FSaveDebug = options.saveDebug; // I3681 + kmcmp::FCompilerWarningsAsErrors = options.compilerWarningsAsErrors; // I4865 + AWarnDeprecatedCode_GLOBAL_LIB = options.warnDeprecatedCode; + kmcmp::FShouldAddCompilerVersion = options.shouldAddCompilerVersion; + kmcmp::CompileTarget = options.target; - kmcmp::FSaveDebug = ASaveDebug; // I3681 - kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 - AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; - - kmcmp::CompileTarget = Target; - - if (!pMsgproc || !pszInfile || !pfkBuffer) { + if (!messageProc || !loadFileProc || !pszInfile) { AddCompileError(CERR_BadCallParams); return FALSE; } - PKMX_STR p; - - if ((p = strrchr_slash((char*)pszInfile)) != nullptr) - { - strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 - kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; - } - else - kmcmp::CompileDir[0] = 0; - - msgproc = pMsgproc; - msgprocContext = (void*)AmsgprocContext; + msgproc = messageProc; + loadfileproc = loadFileProc; + msgprocContext = (void*)procContext; kmcmp::currentLine = 0; kmcmp::nErrors = 0; - fp_in = Open_File(pszInfile, "rb"); - - if (fp_in == NULL) { + int sz; + if(!loadFileProc(pszInfile, "", nullptr, &sz, msgprocContext)) { AddCompileError(CERR_InfileNotExist); return FALSE; } - // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? - // What about really large files? Transfer to a temp file... - if (!fread(str, 1, 3, fp_in)) { - fclose(fp_in); + if(sz < 3) { + // Technically, a 3 byte file can never be a valid .kmn, so we can shortcut + // here and avoid testing outside memory bounds for looking at BOM AddCompileError(CERR_CannotReadInfile); return FALSE; } - fseek(fp_in, 0, SEEK_SET); - if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) - fp_in = UTF16TempFromUTF8(fp_in, TRUE); - else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) - fseek(fp_in, 2, SEEK_SET); - else - fp_in = UTF16TempFromUTF8(fp_in, FALSE); - if (fp_in == NULL) { - AddCompileError(CERR_CannotCreateTempfile); + KMX_BYTE* infile = new KMX_BYTE[sz+1]; + if(!infile) { + AddCompileError(CERR_CannotAllocateMemory); return FALSE; } - - kmcmp::CodeConstants = new kmcmp::NamedCodeConstants; - bool result = CompileKeyboardHandle(fp_in, static_cast(pfkBuffer)); - delete kmcmp::CodeConstants; - - fclose(fp_in); - - if (kmcmp::nErrors > 0) { + if(!loadFileProc(pszInfile, "", infile, &sz, msgprocContext)) { + delete[] infile; + AddCompileError(CERR_CannotReadInfile); return FALSE; } + infile[sz] = 0; // zero-terminate for safety, not technically needed but helps avoid memory bugs - return result; -} - -EXTERN bool kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, - kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext, int Target) // I4865 // I4866 -{ - if (!pMsgproc || !pszInfile || !pfkBuffer) { - AddCompileError(CERR_BadCallParams); - return FALSE; + int offset = 0; + if(infile[0] == (KMX_BYTE) UTF16Sig[0] && infile[1] == (KMX_BYTE) UTF16Sig[1]) { + // UTF-16 source file + offset = 2; + } else { + // UTF-8 source file + KMX_BYTE* infile16; + int sz16; + if(!UTF16TempFromUTF8(infile, sz, &infile16, &sz16)) { + delete[] infile; + AddCompileError(CERR_CannotCreateTempfile); + return FALSE; + } + delete[] infile; + infile = infile16; + sz = sz16; } - return CompileKeyboard(pszInfile, pfkBuffer, TRUE, ACompilerWarningsAsErrors, AWarnDeprecatedCode, - pMsgproc, AmsgprocContext, Target); -} + kmcmp::CodeConstants = new kmcmp::NamedCodeConstants; + bool success = CompileKeyboardBuffer(infile+offset, sz-offset, &fk); + delete kmcmp::CodeConstants; -EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, - char* pszOutfile, bool ASaveDebug, bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext) // I4865 // I4866 -{ - if (!pMsgproc || !pszInfile || !pszOutfile) { - AddCompileError(CERR_BadCallParams); - return FALSE; - } + delete[] infile; - FILE_KEYBOARD fk; - if(!CompileKeyboard(pszInfile, &fk, ASaveDebug, ACompilerWarningsAsErrors, AWarnDeprecatedCode, - pMsgproc, AmsgprocContext, CKF_KEYMAN)) { - // any errors will have been reported directly by CompileKeyboard - return FALSE; - } - - FILE* fp_out = Open_File(pszOutfile, "wb"); - if (fp_out == NULL) { - AddCompileError(CERR_CannotCreateOutfile); + if (kmcmp::nErrors > 0 || !success) { return FALSE; } + // fill in result data KMX_DWORD msg; KMX_BYTE* data = nullptr; size_t dataSize = 0; - if ((msg = WriteCompiledKeyboard(&fk, &data, dataSize)) != CERR_None) { - AddCompileError(msg); - } else { - if(fwrite(data, 1, dataSize, fp_out) != dataSize) { - AddCompileError(CERR_UnableToWriteFully); - } - delete[] data; - } - - fclose(fp_out); - - if (kmcmp::nErrors > 0) { - remove(pszOutfile); - return FALSE; - } - - return TRUE; -} - -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) -{ - PKMX_WCHAR str, p; - - KMX_DWORD msg; - - kmcmp::FMnemonicLayout = FALSE; - - if (!fk) { - AddCompileError(CERR_SomewhereIGotItWrong); - return FALSE; - } - - str = new KMX_WCHAR[LINESIZE]; - if (!str) { - AddCompileError(CERR_CannotAllocateMemory); - return FALSE; - } - - fk->KeyboardID = 0; - fk->version = 0; - fk->dpStoreArray = NULL; - fk->dpGroupArray = NULL; - fk->cxStoreArray = 0; - fk->cxGroupArray = 0; - fk->StartGroup[0] = fk->StartGroup[1] = -1; - fk->szName[0] = 0; - fk->szCopyright[0] = 0; - fk->dwFlags = KF_AUTOMATICVERSION; - fk->currentGroup = 0xFFFFFFFF; - fk->currentStore = 0; - fk->cxDeadKeyArray = 0; - fk->dpDeadKeyArray = NULL; - fk->cxVKDictionary = 0; // I3438 - fk->dpVKDictionary = NULL; // I3438 - fk->extra = new FILE_KEYBOARD_EXTRA; - fk->extra->kvksFilename = u""; -/* fk->szMessage[0] = 0; - fk->szLanguageName[0] = 0;*/ - fk->dwBitmapSize = 0; - fk->dwHotKey = 0; - - kmcmp::BeginLine[BEGIN_ANSI] = -1; - kmcmp::BeginLine[BEGIN_UNICODE] = -1; - kmcmp::BeginLine[BEGIN_NEWCONTEXT] = -1; - kmcmp::BeginLine[BEGIN_POSTKEYSTROKE] = -1; - - - /* Add a store for the Keyman 6.0 copyright information string */ - - if(kmcmp::FShouldAddCompilerVersion) { - u16sprintf(str,LINESIZE, L"Created with Keyman Developer version %d.%d.%d.%d", KEYMAN_VersionMajor, KEYMAN_VersionMinor, KEYMAN_VersionPatch, 0); - AddStore(fk, TSS_KEYMANCOPYRIGHT, str); - } - - /* Add a system store for the Keyman edition number */ - AddStore(fk, TSS_CUSTOMKEYMANEDITION, u"0"); - AddStore(fk, TSS_CUSTOMKEYMANEDITIONNAME, u"Keyman"); - - // must preprocess for group and store names -> this isn't really necessary, but never mind! - while ((msg = ReadLine(fp_in, str, TRUE)) == CERR_None) - { - p = str; - switch (LineTokenType(&p)) - { - case T_VERSION: - *(p + 4) = 0; - if ((msg = AddStore(fk, TSS_VERSION, p)) != CERR_None) { - AddCompileError(msg); - return FALSE; - } - break; - - case T_GROUP: - if ((msg = ProcessGroupLine(fk, p)) != CERR_None) { - AddCompileError(msg); - return FALSE; - } - break; - - case T_STORE: - if ((msg = ProcessStoreLine(fk, p)) != CERR_None) { - AddCompileError(msg); - return FALSE; - } - break; - - default: - break; - } - } - - if (msg != CERR_EndOfFile) { - AddCompileError(msg); - return FALSE; - } - - fseek( fp_in,2,SEEK_SET); - kmcmp::currentLine = 0; - - /* Reindex the list of codeconstants after stores added */ - - kmcmp::CodeConstants->reindex(); - - /* ReadLine will automatically skip over $Keyman lines, and parse wrapped lines */ - while ((msg = ReadLine(fp_in, str, FALSE)) == CERR_None) - { - msg = ParseLine(fk, str); - if (msg != CERR_None) { - AddCompileError(msg); - return FALSE; - } - } - - if (msg != CERR_EndOfFile) { - AddCompileError(msg); - return FALSE; - } - - ProcessGroupFinish(fk); - - if (kmcmp::FSaveDebug) kmcmp::RecordDeadkeyNames(fk); - - /* Add the compiler version as a system store */ - if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) { - AddCompileError(msg); - return FALSE; - } + msg = WriteCompiledKeyboard(&fk, &data, dataSize); - if ((msg = BuildVKDictionary(fk)) != CERR_None) { - AddCompileError(msg); - return FALSE; - } + //TODO: FreeKeyboardPointers(fk); - if ((msg = CheckFilenameConsistencyForCalls(fk)) != CERR_None) { + if(msg != CERR_None) { AddCompileError(msg); return FALSE; } - delete str; - - if (!kmcmp::CheckKeyboardFinalVersion(fk)) { - return FALSE; - } - - /* Warn on inconsistent use of NCAPS */ - if (!kmcmp::FMnemonicLayout) { - CheckNCapsConsistency(fk); - } - - /* Flag presence of deprecated features */ - kmcmp::CheckForDeprecatedFeatures(fk); + result.kmx = data; + result.kmxSize = dataSize; + result.kvksFilename = string_from_u16string(fk.extra->kvksFilename); // convert to UTF8 return TRUE; } + diff --git a/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp b/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0aab9ba51d4abf4262595356feed46819d2c0c4a --- /dev/null +++ b/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp @@ -0,0 +1,117 @@ +#include "pch.h" +#include + +#ifdef __EMSCRIPTEN__ + +/* + WASM interface for compiler message callback +*/ +EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context), { + const proc = globalThis[UTF8ToString(context)].message; + if(!proc || typeof proc != 'function') { + console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); + return 0; + } else { + return proc(line, msgcode, UTF8ToString(text)); + } +}); + +EM_JS(int, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int bufferSize, char* context), { + const proc = globalThis[UTF8ToString(context)].loadFile; + if(!proc || typeof proc != 'function') { + return 0; + } else { + if(buffer == 0) { + return proc(UTF8ToString(filename), UTF8ToString(baseFilename), 0, 0); + } else { + return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize); + } + } +}); + +bool wasm_LoadFileProc(const char* filename, const char* baseFilename, void* buffer, int* bufferSize, void* context) { + char* msgProc = static_cast(context); + if(buffer == nullptr) { + *bufferSize = wasm_loadfileproc(filename, baseFilename, 0, 0, msgProc); + return *bufferSize != -1; + } else { + return wasm_loadfileproc(filename, baseFilename, buffer, *bufferSize, msgProc) == 1; + } +} + +int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, void* context) { + char* msgProc = static_cast(context); + return wasm_msgproc(line, dwMsgCode, szText, msgProc); +} + +struct WASM_COMPILER_INTERFACE { + std::string callbacksKey; // key of callbacks object on globalThis +}; + +struct WASM_COMPILER_RESULT { + bool result; + // Following are pointer offsets in heap + buffer size + int kmx; + int kmxSize; + // Following are compiler side-channel data, required for + // follow-on transform + std::string kvksFilename; + // TODO: additional data to be passed back +}; + +WASM_COMPILER_RESULT kmcmp_wasm_compile(std::string pszInfile, const KMCMP_COMPILER_OPTIONS options, const WASM_COMPILER_INTERFACE intf) { + WASM_COMPILER_RESULT r = {false}; + KMCMP_COMPILER_RESULT kr; + + r.kmx = 0; + r.kmxSize = 0; + r.kvksFilename = ""; + + r.result = kmcmp_CompileKeyboard( + pszInfile.c_str(), + options, + wasm_CompilerMessageProc, + wasm_LoadFileProc, + intf.callbacksKey.c_str(), + kr + ); + + if(r.result) { + // TODO: additional data as required by kmc_kmw + r.kmx = (int) kr.kmx; + r.kmxSize = (int) kr.kmxSize; + r.kvksFilename = kr.kvksFilename; + } + + return r; +} + +EMSCRIPTEN_BINDINGS(compiler_interface) { + + emscripten::class_("CompilerOptions") + .constructor<>() + .property("saveDebug", &KMCMP_COMPILER_OPTIONS::saveDebug) + .property("compilerWarningsAsErrors", &KMCMP_COMPILER_OPTIONS::compilerWarningsAsErrors) + .property("warnDeprecatedCode", &KMCMP_COMPILER_OPTIONS::warnDeprecatedCode) + .property("shouldAddCompilerVersion", &KMCMP_COMPILER_OPTIONS::shouldAddCompilerVersion) + .property("target", &KMCMP_COMPILER_OPTIONS::target) + ; + + emscripten::class_("CompilerInterface") + .constructor<>() + .property("callbacksKey", &WASM_COMPILER_INTERFACE::callbacksKey) + ; + + emscripten::class_("CompilerResult") + .constructor<>() + .property("result", &WASM_COMPILER_RESULT::result) + .property("kmx", &WASM_COMPILER_RESULT::kmx) + .property("kmxSize", &WASM_COMPILER_RESULT::kmxSize) + .property("kvksFilename", &WASM_COMPILER_RESULT::kvksFilename) + ; + + emscripten::function("kmcmp_compile", &kmcmp_wasm_compile); + emscripten::function("kmcmp_parseUnicodeSet", &kmcmp_parseUnicodeSet); +} + +#endif diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.cpp b/developer/src/kmcmplib/src/NamedCodeConstants.cpp index e6b1ae6d70ce20846ba23d24e0ab0efd05d2120a..a8af8fa783c23ac87f211452266d1c85cb9b87c8 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.cpp +++ b/developer/src/kmcmplib/src/NamedCodeConstants.cpp @@ -24,18 +24,14 @@ #include "pch.h" #include #include "NamedCodeConstants.h" -#include "CheckFilenameConsistency.h" #include #include "kmcompx.h" -#include "filesystem.h" using namespace kmcmp; int IsHangulSyllable(const KMX_WCHAR *codename, int *code); namespace kmcmp { - extern KMX_CHAR CompileDir[]; - int __cdecl sort_entries(const void *elem1, const void *elem2) { return u16icmp( @@ -117,85 +113,57 @@ char *kmc_strupr(char *s) { return s; } -KMX_BOOL NamedCodeConstants::IntLoadFile(const KMX_CHAR *filename) -{ +KMX_BOOL NamedCodeConstants::LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename) { const int str_size = 256; - FILE *fp = NULL; - if (CheckFilenameConsistency(filename, FALSE) != 0) { + auto szNameUtf8 = string_from_u16string(filename); + + int FileSize; + KMX_BYTE* Buf; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, &FileSize, msgprocContext)) { return FALSE; } - fp = Open_File(filename, "rt"); - if(fp == NULL) { - return FALSE; // I3481 + Buf = new KMX_BYTE[FileSize+1]; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), Buf, &FileSize, msgprocContext)) { + delete[] Buf; + return FALSE; } + Buf[FileSize] = 0; // zero-terminate for strtok - KMX_CHAR str[str_size], *p, *q, *context = NULL; - KMX_BOOL isEol , first = TRUE; + char* filetok; + char* filecontext; + filetok = strtok_r((char*)Buf, "\n", &filecontext); - while(fgets(str, str_size, fp)) - { - isEol = *(strchr(str, 0) - 1) == '\n'; - p = strtok_r(str, ";", &context); // I3481 - q = strtok_r(NULL, ";\n", &context); - if(p && q) - { - if(first && *p == (KMX_CHAR)0xEF && *(p+1) == (KMX_CHAR)0xBB && *(p+2) == (KMX_CHAR)0xBF) p += 3; // I3056 UTF-8 // I3512 - first = FALSE; + if(*filetok == (KMX_CHAR)0xEF && *(filetok+1) == (KMX_CHAR)0xBB && *(filetok+2) == (KMX_CHAR)0xBF) filetok += 3; // I3056 UTF-8 // I3512 + + while(filetok) { + KMX_CHAR str[str_size], *p, *q, *context = NULL; + + if(strlen(filetok) >= str_size) { + delete[] Buf; + // TODO chuck a wobbly + return FALSE; + } + strcpy(str, filetok); + p = strtok_r(str, ";\r", &context); // I3481 + q = strtok_r(nullptr, ";\r", &context); + if(p && q) { kmc_strupr(q); // I3481 // I3641 - long n = strtol(p, NULL, 16); + long n = strtol(p, nullptr, 16); if (*q != '<') { PKMX_WCHAR q0 = strtowstr(q); AddCode_IncludedCodes((int)n, q0); delete[] q0; } } - if(!isEol ) - { - while(fgets(str, str_size, fp)) if(*(strchr(str, 0)-1) == '\n') break; - } + filetok = strtok_r(nullptr, "\n", &filecontext); } - fclose(fp); - - return TRUE; -} - -KMX_BOOL NamedCodeConstants::LoadFile(const KMX_CHAR *filename) -{ - const int buf_size = 260; - KMX_CHAR buf[buf_size]; - // Look in current directory first -- REMOVED AS DANGEROUS - /* strncpy(buf, filename, (buf_size-1)); buf[buf_size-1] = 0; // I3481 - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - */ - // Then look in keyboard file directory (CompileDir) - strncpy(buf, CompileDir, (buf_size-1)); buf[buf_size-1] = 0; // I3481 - strncat(buf, filename, (buf_size-1)-strlen(CompileDir)); buf[buf_size-1] = 0; - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - - //TODO: sort out how to find common includes in non-Windows platforms: - #ifdef _WINDOWS_ - // Finally look in kmcmpdll.dll directory - GetModuleFileName(0, buf, buf_size); - - KMX_CHAR *p = strrchr_slash(buf); - if(p) - p++; - else - p = buf; - *p = 0; - strncat_s(buf, _countof(buf), filename, (buf_size-1)-strlen(buf)); buf[buf_size-1] = 0; // I3481 // I3641 - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - #endif + delete[] Buf; reindex(); - - return FALSE; + return TRUE; } void NamedCodeConstants::reindex() diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.h b/developer/src/kmcmplib/src/NamedCodeConstants.h index 83d707d092cc4535a24cf3676aaa3d2a44d7b859..c537a73361b40f5539a75275fe86787e244b3541 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.h +++ b/developer/src/kmcmplib/src/NamedCodeConstants.h @@ -2,6 +2,8 @@ #ifndef NAMEDCODECONSTANTS_H #define NAMEDCODECONSTANTS_H +#include "compfile.h" + #define MAX_ENAME 128 #define ALLOC_SIZE 256 @@ -23,14 +25,13 @@ namespace kmcmp{ int GetCode_IncludedCodes(const KMX_WCHAR *codename); void AddCode_IncludedCodes(int n, const KMX_WCHAR *p); - KMX_BOOL IntLoadFile(const KMX_CHAR *filename); public: NamedCodeConstants(); ~NamedCodeConstants(); void reindex(); void AddCode(int n, const KMX_WCHAR *p, KMX_DWORD storeIndex); - KMX_BOOL LoadFile(const KMX_CHAR *filename); + KMX_BOOL LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename); int GetCode(const KMX_WCHAR *codename, KMX_DWORD *storeIndex); }; } diff --git a/developer/src/kmcmplib/src/compfile.h b/developer/src/kmcmplib/src/compfile.h index 595d1fececb2cf8b01b25e49a026fda1c7d103ac..c54663aba7659f871ce6271089494aa81420f6d0 100644 --- a/developer/src/kmcmplib/src/compfile.h +++ b/developer/src/kmcmplib/src/compfile.h @@ -123,7 +123,8 @@ typedef FILE_VKDICTIONARY *PFILE_VKDICTIONARY; * Extra metadata for API consumers */ struct FILE_KEYBOARD_EXTRA { - std::u16string kvksFilename; // original TSS_VISUALKEYBOARD value + std::string kmnFilename; // utf-8 + std::u16string kvksFilename; // utf-16, original TSS_VISUALKEYBOARD value }; typedef struct FILE_KEYBOARD_EXTRA* PFILE_KEYBOARD_EXTRA; @@ -197,33 +198,4 @@ struct COMPILEMESSAGES { typedef COMPILEMESSAGES *PCOMPILEMESSAGES; -/* -struct TVersion -{ - //int MinVersion; // 0x0500 usually - //int CompilerVersion[4]; - //int MinCompilerVersion[4]; - int KeyboardVersion; // 0x0500 usually -}; - -extern TVersion FVersionInfo; -*/ - -/* -#define bstrcpy(c,d) (LPBYTE)strcpy((LPSTR)(c),(LPSTR)(d)) -#define bstrlen(c) strlen((LPSTR)(c)) -#define bstrcmp(c,d) strcmp((LPSTR)(c),(LPSTR)(d)) -#define bstrncmp(c,d,n) strncmp((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrnicmp(c,d,n) strnicmp((LPSTR)(c),(LPSTR)(d),(n)) -#define bstricmp(c,d) stricmp((LPSTR)(c),(LPSTR)(d)) -#define bstrchr(c,ch) (LPBYTE)strchr((LPSTR)(c),(char)ch) -#define bstrncpy(c,d,n) (LPBYTE)strncpy((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrtok(c,d) (LPBYTE)strtok((LPSTR)(c),(LPSTR)(d)) -#define bstrcat(c,d) (LPBYTE)strcat((LPSTR)(c),(LPSTR)(d)) -#define bstrncat(c,d,n) (LPBYTE)strncat((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrrev(c) (LPBYTE)strrev((LPSTR)(c)) -#define batoi(c) atoi((LPSTR)(c)) -#define bstrtol(c,d,n) strtol((LPSTR)(c),(LPSTR *)(d),(n)) -*/ - #endif // _COMPFILE_H diff --git a/developer/src/kmcmplib/src/json-validation.cpp b/developer/src/kmcmplib/src/json-validation.cpp deleted file mode 100644 index f58d8c950534943427285a4f400fb39c190217ff..0000000000000000000000000000000000000000 --- a/developer/src/kmcmplib/src/json-validation.cpp +++ /dev/null @@ -1,68 +0,0 @@ - -#include -#include - -#include -#include "kmcompx.h" - -using nlohmann::json; -using nlohmann::json_uri; -using nlohmann::json_schema_draft4::json_validator; - -static void loader(const json_uri &uri, json &schema) -{ - std::fstream lf("." + uri.path()); - if (!lf.good()) - throw std::invalid_argument("could not open " + uri.url() + " tried with " + uri.path()); - - lf >> schema; -} - -EXTERN bool kmcmp_ValidateJsonFile(std::fstream& f, std::fstream& fd, kmcmp_ValidateJsonMessageProc MessageProc, void* context) { - if (!f.good()) { - MessageProc(-1, "Schema file could not be loaded.", context); - return FALSE; - } - - // 1) Read the schema for the document you want to validate - json schema; - try { - f >> schema; - } - catch (std::exception &e) { - MessageProc(f.tellp(), e.what(), context); - return FALSE; - } - - // 2) create the validator and - json_validator validator(loader, [](const std::string &, const std::string &) {}); - - try { - // insert this schema as the root to the validator - // this resolves remote-schemas, sub-schemas and references via the given loader-function - validator.set_root_schema(schema); - } - catch (std::exception &e) { - MessageProc(-2, e.what(), context); - return FALSE; - } - - // 3) do the actual validation of the document - json document; - - if (!fd.good()) { - MessageProc(-3, "Json file could not be loaded.", context); - return FALSE; - } - - try { - fd >> document; - validator.validate(document); - } - catch (std::exception &e) { - MessageProc(fd.tellp(), e.what(), context); - return FALSE; - } - - return TRUE; -} \ No newline at end of file diff --git a/developer/src/kmcmplib/src/kmcmplib.h b/developer/src/kmcmplib/src/kmcmplib.h index 78e945ee0d866af69158dd7a9b1e04e3916a914a..f22c836baeeff7cdf0e72de4a3fb2caab4dfd740 100644 --- a/developer/src/kmcmplib/src/kmcmplib.h +++ b/developer/src/kmcmplib/src/kmcmplib.h @@ -14,7 +14,6 @@ namespace kmcmp { extern KMX_BOOL FMnemonicLayout; extern KMX_BOOL FOldCharPosMatching; extern int CompileTarget; - extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer extern int BeginLine[4]; extern int currentLine; extern NamedCodeConstants *CodeConstants; @@ -24,6 +23,7 @@ namespace kmcmp { } extern kmcmp_CompilerMessageProc msgproc; +extern kmcmp_LoadFileProc loadfileproc; extern void* msgprocContext; extern KMX_BOOL AWarnDeprecatedCode_GLOBAL_LIB; @@ -40,10 +40,10 @@ KMX_BOOL AddCompileError(KMX_DWORD msg); PKMX_WCHAR strtowstr(PKMX_STR in); PFILE_STORE FindSystemStore(PFILE_KEYBOARD fk, KMX_DWORD dwSystemID); -FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble); +bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16); KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, KMX_BYTE**data, size_t& dataSize); KMX_DWORD AddStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, const KMX_WCHAR * str, KMX_DWORD *dwStoreID= NULL); -KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess); +KMX_DWORD ReadLine(KMX_BYTE* infile, int sz, int& offset, PKMX_WCHAR wstr, KMX_BOOL PreProcess); KMX_DWORD ParseLine(PFILE_KEYBOARD fk, PKMX_WCHAR str); KMX_DWORD ProcessGroupLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); KMX_DWORD ProcessGroupFinish(PFILE_KEYBOARD fk); diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 292b158ce276abf20060f38c8770cdbf1f0dcb84..c6ed2b84b01869c07311687fa5445dfed1b9f06d 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -7,6 +7,7 @@ # TODO: is this required? It should be Keyman Core only defns += ['-DKMN_KBP_EXPORTING'] version_res = [] +lib_links = [] if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' warns += [ @@ -24,12 +25,10 @@ endif name_suffix = [] if cpp_compiler.get_id() == 'emscripten' - links += ['-lnodefs.js', '-sMODULARIZE', '-sEXPORT_ES6', '--whole-archive', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'cwrap\', \'UTF8ToString\']'] - # tests are building as ES6 so we need to declare the file extension - # note that meson currently struggles with the sanitycheckc_cross.exe - # program, because it has a hard coded extension (.exe) which is not - # valid for node programs in module mode. - name_suffix = '.mjs' + # wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+ + flags += ['-fwasm-exceptions'] + lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6'] + links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) @@ -38,36 +37,29 @@ icuuc_dep = icu.get_variable('icuuc_dep') lib = library('kmcmplib', 'CasedKeys.cpp', 'CharToKeyConversion.cpp', + 'CheckForDuplicates.cpp', + 'CheckNCapsConsistency.cpp', + 'CompileKeyboardBuffer.cpp', 'Compiler.cpp', 'CompilerInterfaces.cpp', + 'CompilerInterfacesWasm.cpp', + 'CompMsg.cpp', 'DeprecationChecks.cpp', 'Edition.cpp', - 'filesystem.cpp', - 'json-validation.cpp', + 'kmx_u16.cpp', 'NamedCodeConstants.cpp', + 'UnreachableRules.cpp', + 'uset-api.cpp', 'versioning.cpp', 'virtualcharkeys.cpp', 'xstring.cpp', - 'CharToKeyConversion.cpp', - 'CheckFilenameConsistency.cpp', - 'CheckForDuplicates.cpp', - 'CheckNCapsConsistency.cpp', - 'UnreachableRules.cpp', - 'kmx_u16.cpp', - 'CompMsg.cpp', - 'uset-api.cpp', - '../../../../common/windows/cpp/src/ConvertUTF.c', '../../../../common/windows/cpp/src/crc32.cpp', '../../../../common/windows/cpp/src/vkeys.cpp', - 'xstring.cpp', - '../../ext/json-schema-validator/json-schema-draft4.json.cpp', - '../../ext/json-schema-validator/json-uri.cpp', - '../../ext/json-schema-validator/json-validator.cpp', version_res, cpp_args: defns + warns + flags, - link_args: links, + link_args: links + lib_links, version: meson.project_version(), include_directories: inc, install: true, @@ -81,7 +73,7 @@ if cpp_compiler.get_id() == 'emscripten' host = executable('wasm-host', #'wasm-host.cpp', cpp_args: defns, include_directories: inc, - link_args: links, + link_args: links + lib_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep) endif diff --git a/developer/src/kmcmplib/src/pch.h b/developer/src/kmcmplib/src/pch.h index f878ad11be4f892ad4e9f0ece5864712c6af5c6b..358e28998abc8e65bba25cf2b594ca95cbed5b57 100644 --- a/developer/src/kmcmplib/src/pch.h +++ b/developer/src/kmcmplib/src/pch.h @@ -10,6 +10,4 @@ #include "../../../../common/windows/cpp/include/crc32.h" #include -#include - #include diff --git a/developer/src/kmcmplib/src/uset-api.cpp b/developer/src/kmcmplib/src/uset-api.cpp index 1aa22477b2b63465a8cb0db8b7d624409d9ce083..c2c72cab347587e64998ce7ecfed41614ea8fc49 100644 --- a/developer/src/kmcmplib/src/uset-api.cpp +++ b/developer/src/kmcmplib/src/uset-api.cpp @@ -4,16 +4,13 @@ #include "unicode/uniset.h" #include "unicode/unistr.h" -EXTERN int kmcmp_ParseUnicodeSet( - const char* szText, - uint32_t* outputBuffer, +EXTERN int kmcmp_parseUnicodeSet( + const std::string text, + uintptr_t outputBuffer_, uint32_t outputBufferSize ) { - if (szText == nullptr) { - // null string coming in - return KMCMP_ERROR_SYNTAX_ERR; - } - const icu::UnicodeString str = icu::UnicodeString::fromUTF8(szText); + uint32_t* outputBuffer = reinterpret_cast(outputBuffer_); + const icu::UnicodeString str = icu::UnicodeString::fromUTF8(text.c_str()); if (str.isBogus() || str.isEmpty()) { // empty string return KMCMP_ERROR_SYNTAX_ERR; diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp index 4c46b1f1a408c166cda98dda0ef43d7449471a1e..cb0337caef680107910d8b219bca8a69171c5a15 100644 --- a/developer/src/kmcmplib/tests/api-test.cpp +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -18,32 +18,20 @@ #include #include "../src/compfile.h" #include +#include "util_filesystem.h" +#include "util_callbacks.h" void setup(); -void test_kmcmp_CompileKeyboardFile(); -void test_kmcmp_CompileKeyboardFileToBuffer(); - -std::vector error_vec; - -int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) { - error_vec.push_back(dwMsgCode); - const char*t = "unknown"; - switch(dwMsgCode & 0xF000) { - case CERR_HINT: t=" hint"; break; - case CERR_WARNING: t="warning"; break; - case CERR_ERROR: t=" error"; break; - case CERR_FATAL: t=" fatal"; break; - } - printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); - return 1; -} +void test_kmcmp_CompileKeyboard(char *kmn_file); int main(int argc, char *argv[]) { + if(argc < 1) { + puts("Usage: api-test "); + puts("Warning: blank_keyboard will be overwritten"); + return 1; + } setup(); - test_kmcmp_CompileKeyboardFile(); - - setup(); - test_kmcmp_CompileKeyboardFileToBuffer(); + test_kmcmp_CompileKeyboard(argv[1]); return 0; } @@ -52,36 +40,27 @@ void setup() { error_vec.clear(); } -void test_kmcmp_CompileKeyboardFile() { - char kmn_file[L_tmpnam], kmx_file[L_tmpnam]; - tmpnam(kmn_file); - tmpnam(kmx_file); - - // Create an empty file - FILE *fp = fopen(kmn_file, "w"); - fclose(fp); - - // It should fail when a zero-byte file is passed in - assert(!kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr)); - assert(error_vec.size() == 1); - assert(error_vec[0] == CERR_CannotReadInfile); - - unlink(kmn_file); -} - -void test_kmcmp_CompileKeyboardFileToBuffer() { - char kmn_file[L_tmpnam], kmx_file[L_tmpnam]; - tmpnam(kmn_file); - tmpnam(kmx_file); +/* + TODO: tests to run: + 4. ANSI (no BOM of course) #8884 + 8. file without blank last line (cannot compare with fixture due to bug in kmcmpdll...) + Hint to add: k004_ansi.kmn: Hint: 10A6 Keyman Developer has detected that the file has ANSI encoding. Consider converting this file to UTF-8 +*/ +void test_kmcmp_CompileKeyboard(char *kmn_file) { // Create an empty file - FILE *fp = fopen(kmn_file, "w"); + FILE *fp = Open_File(kmn_file, "wb"); fclose(fp); - FILE_KEYBOARD fk; - // It should fail when a zero-byte file is passed in - assert(!kmcmp_CompileKeyboardFileToBuffer(kmn_file, &fk, true, false, msgproc, nullptr, CKF_KEYMAN)); + KMCMP_COMPILER_RESULT result; + KMCMP_COMPILER_OPTIONS options; + options.saveDebug = true; + options.compilerWarningsAsErrors = false; + options.warnDeprecatedCode = true; + options.shouldAddCompilerVersion = false; + options.target = CKF_KEYMAN; + assert(!kmcmp_CompileKeyboard(kmn_file, options, msgproc, loadfileProc, nullptr, result)); assert(error_vec.size() == 1); assert(error_vec[0] == CERR_CannotReadInfile); diff --git a/developer/src/kmcmplib/tests/fixtures/adiga_danef.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/adiga_danef.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/adiga_danef.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/adiga_danef.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ahom_star.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ahom_star.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ahom_star.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ahom_star.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/aksarabali_panlex.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/aksarabali_panlex.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/aksarabali_panlex.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/aksarabali_panlex.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/alephwithbeth.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/alephwithbeth.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/alephwithbeth.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/alephwithbeth.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/anii.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/anii.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/anii.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/anii.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/arabic_izza.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/arabic_izza.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/arabic_izza.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/arabic_izza.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/aramaic_hebrew.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/aramaic_hebrew.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/aramaic_hebrew.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/aramaic_hebrew.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/armenian_mnemonic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/armenian_mnemonic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/armenian_mnemonic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/armenian_mnemonic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/armenian_mnemonic_r.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/armenian_mnemonic_r.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/armenian_mnemonic_r.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/armenian_mnemonic_r.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/athinkra_vai.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/athinkra_vai.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/athinkra_vai.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/athinkra_vai.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/athinkra_vai_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/athinkra_vai_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/athinkra_vai_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/athinkra_vai_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ausephon.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ausephon.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ausephon.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ausephon.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_inpage.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_inpage.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_inpage.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_inpage.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_persian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_persian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_persian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_persian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_scientific.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_scientific.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_scientific.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_scientific.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/balochi_urdu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_urdu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/balochi_urdu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/balochi_urdu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bangla_joy.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_joy.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bangla_joy.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_joy.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bangla_munir.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_munir.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bangla_munir.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_munir.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bangla_probhat.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_probhat.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bangla_probhat.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bangla_probhat.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbda1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbda1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbda2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbda2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbda3.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda3.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbda3.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbda3.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdadlm.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdadlm.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdadlm.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdadlm.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdal.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdal.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdal.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdal.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdarme.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarme.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdarme.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarme.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdarmph.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmph.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdarmph.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmph.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdarmty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdarmty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdarmw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdarmw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdarmw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdaze.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdaze.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdaze.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdaze.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdazel.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdazel.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdazel.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdazel.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdazst.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdazst.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdazst.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdazst.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbash.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbash.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbash.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbash.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbe.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbe.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbe.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbe.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbene.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbene.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbene.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbene.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbgph.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbgph.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbgph.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbgph.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbgph1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbgph1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbgph1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbgph1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbhc.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbhc.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbhc.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbhc.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdblr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdblr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdblr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdblr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbug.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbug.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbug.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbug.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdbulg.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbulg.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdbulg.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdbulg.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdca.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdca.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdca.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdca.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcher.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcher.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcher.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcher.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcherp.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcherp.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcherp.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcherp.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcz.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcz.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcz1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcz1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdcz2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdcz2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdcz2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdda.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdda.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdda.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdda.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbddiv1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddiv1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbddiv1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddiv1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbddiv2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddiv2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbddiv2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddiv2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbddv.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddv.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbddv.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddv.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbddzo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddzo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbddzo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbddzo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdes.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdes.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdes.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdes.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdest.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdest.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdest.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdest.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfi1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfi1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfi1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfi1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdfthrk.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfthrk.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdfthrk.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdfthrk.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgae.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgae.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgae.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgae.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgeo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgeo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgeoer.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeoer.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgeoer.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeoer.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgeome.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeome.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgeome.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeome.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgeooa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeooa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgeooa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeooa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgeoqw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeoqw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgeoqw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgeoqw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgkl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgkl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgkl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgkl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgn.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgn.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgn.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgn.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgr1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgr1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgr1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgr1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgrlnd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgrlnd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgrlnd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgrlnd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdgthc.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgthc.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdgthc.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdgthc.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhau.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhau.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhau.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhau.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhaw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhaw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhaw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhaw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhe.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhe.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhe220.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe220.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhe220.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe220.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhe319.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe319.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhe319.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhe319.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdheb.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdheb.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdheb.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdheb.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhebl3.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhebl3.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhebl3.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhebl3.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhela2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhela2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhela2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhela2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhela3.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhela3.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhela3.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhela3.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhept.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhept.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhept.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhept.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdhu1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhu1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdhu1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdhu1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdibo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdibo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdibo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdibo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinasa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinasa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinasa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinasa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinbe2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinbe2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinbe2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinbe2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinben.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinben.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinben.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinben.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdindev.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdindev.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdindev.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdindev.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinen.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinen.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinen.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinen.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinguj.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinguj.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinguj.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinguj.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinhin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinhin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinhin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinhin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinkan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinkan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinkan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinkan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinmal.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinmal.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinmal.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinmal.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinmar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinmar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinmar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinmar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinori.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinori.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinori.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinori.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinpun.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinpun.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinpun.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinpun.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdintam.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdintam.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdintam.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdintam.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdintel.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdintel.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdintel.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdintel.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdinuk2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinuk2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdinuk2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdinuk2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdir.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdir.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdir.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdir.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdit.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdit.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdit.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdit.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdit142.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdit142.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdit142.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdit142.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdiulat.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdiulat.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdiulat.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdiulat.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdjav.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdjav.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdjav.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdjav.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdkaz.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkaz.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdkaz.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkaz.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdkhmr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkhmr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdkhmr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkhmr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdkni.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkni.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdkni.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkni.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdkurd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkurd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdkurd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkurd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdkyr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkyr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdkyr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdkyr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdla.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdla.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdla.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdla.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlao.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlao.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlao.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlao.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlisub.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlisub.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlisub.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlisub.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlisus.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlisus.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlisus.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlisus.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlt.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlt.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlt1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlt1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlt2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlt2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlt2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlv.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlv.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlv.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlv.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlv1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlv1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlv1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlv1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdlvst.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlvst.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdlvst.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdlvst.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmac.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmac.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmac.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmac.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmacst.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmacst.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmacst.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmacst.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmaori.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmaori.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmaori.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmaori.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmlt47.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmlt47.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmlt47.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmlt47.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmlt48.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmlt48.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmlt48.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmlt48.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmon.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmon.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmon.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmon.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmonmo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmonmo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmonmo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmonmo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmonst.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmonst.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmonst.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmonst.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdmyan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmyan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdmyan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdmyan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdne.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdne.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdne.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdne.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdnepr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnepr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdnepr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnepr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdnko.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnko.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdnko.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnko.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdno.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdno.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdno.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdno.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdno1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdno1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdno1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdno1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdnso.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnso.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdnso.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdnso.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdntl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdntl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdntl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdntl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdogham.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdogham.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdogham.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdogham.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdolch.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdolch.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdolch.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdolch.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdoldit.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdoldit.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdoldit.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdoldit.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdosa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdosa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdosa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdosa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdosm.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdosm.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdosm.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdosm.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdpash.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpash.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdpash.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpash.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdphags.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdphags.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdphags.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdphags.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdpl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdpl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdpl1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpl1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdpl1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpl1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdpo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdpo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdpo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdropr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdropr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdropr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdropr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdrost.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdrost.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdrost.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdrost.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdru.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdru.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdru.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdru.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdru1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdru1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdru1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdru1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdrum.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdrum.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdrum.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdrum.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsf.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsf.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsf.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsf.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsg.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsg.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsg.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsg.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsl1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsl1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsl1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsl1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsmsfi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsmsfi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsmsfi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsmsfi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsmsno.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsmsno.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsmsno.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsmsno.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsn1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsn1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsn1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsn1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsora.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsora.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsora.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsora.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsorex.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsorex.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsorex.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsorex.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsors1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsors1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsors1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsors1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsp.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsp.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsp.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsp.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsw09.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsw09.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsw09.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsw09.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsyr1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsyr1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsyr1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsyr1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdsyr2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsyr2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdsyr2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdsyr2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtaile.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtaile.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtaile.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtaile.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtajik.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtajik.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtajik.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtajik.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtam99.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtam99.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtam99.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtam99.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdth0.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth0.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdth0.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth0.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdth1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdth1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdth2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdth2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdth3.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth3.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdth3.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdth3.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtifi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtifi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtifi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtifi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtifi2.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtifi2.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtifi2.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtifi2.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtiprd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtiprd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtiprd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtiprd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtt102.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtt102.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtt102.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtt102.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtuf.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtuf.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtuf.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtuf.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtuq.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtuq.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtuq.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtuq.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdturme.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdturme.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdturme.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdturme.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdtzm.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtzm.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdtzm.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdtzm.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdughr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdughr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdughr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdughr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdughr1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdughr1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdughr1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdughr1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbduk.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbduk.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbduk.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbduk.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdukx.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdukx.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdukx.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdukx.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdur.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdur.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdur.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdur.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdur1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdur1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdur1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdur1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdurdu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdurdu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdurdu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdurdu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdus.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdus.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdus.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdus.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdusa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdusa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdusl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdusl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdusr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdusr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdusx.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusx.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdusx.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdusx.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbduzb.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbduzb.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbduzb.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbduzb.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdvntc.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdvntc.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdvntc.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdvntc.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdwol.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdwol.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdwol.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdwol.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdyak.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdyak.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdyak.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdyak.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdyba.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdyba.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdyba.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdyba.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdycc.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdycc.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdycc.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdycc.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/basic_kbdycl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdycl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/basic_kbdycl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/basic_kbdycl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/batak.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/batak.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/batak.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/batak.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/baybayin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/baybayin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/baybayin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/baybayin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bhaiksuki_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bhaiksuki_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bhaiksuki_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bhaiksuki_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_cree_east.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_cree_east.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_cree_east_james_bay.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east_james_bay.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_cree_east_james_bay.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east_james_bay.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_cree_east_latn.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east_latn.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_cree_east_latn.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_east_latn.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_cree_west_latn.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_west_latn.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_cree_west_latn.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_west_latn.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_cree_woods.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_woods.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_cree_woods.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_cree_woods.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_innu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_innu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_innu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_innu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_innu_phonemic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_innu_phonemic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_innu_phonemic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_innu_phonemic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_mista_wasaha_cree.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_mista_wasaha_cree.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_mista_wasaha_cree.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_mista_wasaha_cree.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_naskapi_classic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_naskapi_classic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_naskapi_classic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_naskapi_classic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_naskapi_common.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_naskapi_common.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_naskapi_common.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_naskapi_common.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bj_oji_cree.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_oji_cree.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bj_oji_cree.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bj_oji_cree.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/brahmi_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/brahmi_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/brahmi_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/brahmi_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/btl_kenya.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/btl_kenya.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/btl_kenya.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/btl_kenya.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/bu_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/bu_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/bu_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/bu_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/buhid.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/buhid.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/buhid.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/buhid.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/burushaski_girminas.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/burushaski_girminas.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/burushaski_girminas.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/burushaski_girminas.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/cabecar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/cabecar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/cabecar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/cabecar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/carian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/carian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/carian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/carian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/chalchiteko.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/chalchiteko.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/chalchiteko.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/chalchiteko.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/chechen_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/chechen_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/chechen_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/chechen_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/chinuk_wawa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/chinuk_wawa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/chinuk_wawa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/chinuk_wawa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/choctaw_modern.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/choctaw_modern.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/choctaw_modern.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/choctaw_modern.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/cim.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/cim.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/cim.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/cim.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/clavbur9.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/clavbur9.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/clavbur9.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/clavbur9.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/colchis_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/colchis_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/colchis_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/colchis_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/colchis_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/colchis_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/colchis_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/colchis_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/common_devanagari.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/common_devanagari.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/common_devanagari.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/common_devanagari.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/coptic_greek.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/coptic_greek.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/coptic_greek.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/coptic_greek.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/coptic_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/coptic_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/coptic_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/coptic_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/dagbani.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/dagbani.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/dagbani.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/dagbani.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/dega.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/dega.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/dega.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/dega.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/dene.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/dene.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/dene.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/dene.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/deseret.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/deseret.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/deseret.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/deseret.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/dogra_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/dogra_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/dogra_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/dogra_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/easy_chakma.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/easy_chakma.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/easy_chakma.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/easy_chakma.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ekwtamil99uni.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ekwtamil99uni.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ekwtamil99uni.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ekwtamil99uni.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_dinka.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_dinka.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_dinka.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_dinka.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_harari_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_harari_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_harari_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_harari_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_naija.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_naija.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_naija.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_naija.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_nuer.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_nuer.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_nuer.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_nuer.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_pasifika.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_pasifika.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_pasifika.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_pasifika.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/el_yolngu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_yolngu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/el_yolngu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/el_yolngu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/embera_north.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/embera_north.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/embera_north.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/embera_north.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/enga.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/enga.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/enga.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/enga.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/enggano.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/enggano.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/enggano.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/enggano.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/english_shavian_igc.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/english_shavian_igc.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/english_shavian_igc.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/english_shavian_igc.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/english_shavian_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/english_shavian_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/english_shavian_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/english_shavian_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/engram.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/engram.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/engram.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/engram.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/esperuni.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/esperuni.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/esperuni.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/esperuni.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/farsiman.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/farsiman.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/farsiman.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/farsiman.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/finongan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/finongan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/finongan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/finongan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fulfulde_ajami_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fulfulde_ajami_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fulfulde_ajami_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fulfulde_ajami_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fulfulde_latin_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fulfulde_latin_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fulfulde_latin_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fulfulde_latin_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_dakelh.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_dakelh.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_dakelh.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_dakelh.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_dane_zaa_zaage.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_dane_zaa_zaage.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_dane_zaa_zaage.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_dane_zaa_zaage.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_denesuline_epsilon.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_denesuline_epsilon.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_denesuline_epsilon.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_denesuline_epsilon.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_diitiidatx.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_diitiidatx.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_diitiidatx.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_diitiidatx.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_gitsenimx.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_gitsenimx.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_gitsenimx.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_gitsenimx.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_gwichin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_gwichin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_gwichin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_gwichin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_hailzaqvla.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_hailzaqvla.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_hailzaqvla.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_hailzaqvla.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_haisla.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_haisla.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_haisla.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_haisla.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_halqemeylem.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_halqemeylem.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_halqemeylem.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_halqemeylem.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_han.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_han.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_han.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_han.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_henqeminem.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_henqeminem.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_henqeminem.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_henqeminem.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_hlgaagilda_xaayda_kil.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_hlgaagilda_xaayda_kil.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_hlgaagilda_xaayda_kil.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_hlgaagilda_xaayda_kil.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_kanienkeha_e.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kanienkeha_e.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_kanienkeha_e.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kanienkeha_e.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_ktunaxa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_ktunaxa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_ktunaxa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_ktunaxa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_kwakwala.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kwakwala.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_kwakwala.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kwakwala.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_kwakwala_liqwala.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kwakwala_liqwala.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_kwakwala_liqwala.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_kwakwala_liqwala.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_migmaq.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_migmaq.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_migmaq.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_migmaq.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_natwits.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_natwits.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_natwits.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_natwits.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_nisgaa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nisgaa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_nisgaa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nisgaa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_nlekepmxcin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nlekepmxcin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_nlekepmxcin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nlekepmxcin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_northern_tutchone.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_northern_tutchone.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_northern_tutchone.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_northern_tutchone.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_nsilxcen.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nsilxcen.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_nsilxcen.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nsilxcen.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_nuucaanul.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nuucaanul.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_nuucaanul.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_nuucaanul.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_secwepemctsin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_secwepemctsin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_secwepemctsin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_secwepemctsin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_sencoten.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_sencoten.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_sencoten.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_sencoten.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_shashishalhem.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_shashishalhem.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_shashishalhem.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_shashishalhem.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_smalgyax.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_smalgyax.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_smalgyax.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_smalgyax.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_southern_tutchone.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_southern_tutchone.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_southern_tutchone.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_southern_tutchone.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_statimcets.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_statimcets.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_statimcets.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_statimcets.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_stlatlimxec.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_stlatlimxec.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_stlatlimxec.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_stlatlimxec.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_tagizi_dene.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tagizi_dene.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_tagizi_dene.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tagizi_dene.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_taltan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_taltan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_taltan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_taltan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_tlingit.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tlingit.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_tlingit.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tlingit.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_tsekehne.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tsekehne.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_tsekehne.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tsekehne.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_tsilhqotin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tsilhqotin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_tsilhqotin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_tsilhqotin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_uwikala.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_uwikala.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_uwikala.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_uwikala.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/fv_xaislakala.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_xaislakala.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/fv_xaislakala.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/fv_xaislakala.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/galaxie_greek_mnemonic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_greek_mnemonic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/galaxie_greek_mnemonic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_greek_mnemonic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/galaxie_greek_positional.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_greek_positional.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/galaxie_greek_positional.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_greek_positional.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/galaxie_hebrew_mnemonic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_hebrew_mnemonic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/galaxie_hebrew_mnemonic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_hebrew_mnemonic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/galaxie_hebrew_positional.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_hebrew_positional.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/galaxie_hebrew_positional.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/galaxie_hebrew_positional.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gandhari.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gandhari.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gandhari.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gandhari.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/geezbrhan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/geezbrhan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/geezbrhan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/geezbrhan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_amh_7.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_amh_7.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_amh_7.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_amh_7.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_amharic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_amharic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_amharic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_amharic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_blin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_blin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_blin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_blin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_ethiopic_7.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_ethiopic_7.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_ethiopic_7.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_ethiopic_7.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_geez.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_geez.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_geez.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_geez.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_gurage.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_gurage.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_gurage.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_gurage.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_gurage_legacy.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_gurage_legacy.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_gurage_legacy.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_gurage_legacy.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_musnad.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_musnad.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_musnad.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_musnad.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_tigrinya_eritrea.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_tigrinya_eritrea.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_tigrinya_eritrea.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_tigrinya_eritrea.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gff_tigrinya_ethiopia.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_tigrinya_ethiopia.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gff_tigrinya_ethiopia.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gff_tigrinya_ethiopia.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ghana.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ghana.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ghana.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ghana.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gilaki.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gilaki.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gilaki.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gilaki.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gilaki_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gilaki_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gilaki_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gilaki_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gondi_dev.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_dev.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gondi_dev.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_dev.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gondi_gunjala.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_gunjala.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gondi_gunjala.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_gunjala.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/gondi_tel.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_tel.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/gondi_tel.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/gondi_tel.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/greekclassical.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/greekclassical.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/greekclassical.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/greekclassical.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hanunoo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hanunoo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hanunoo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hanunoo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/haroi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/haroi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/haroi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/haroi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hatran_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hatran_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hatran_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hatran_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hausa_ajami_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hausa_ajami_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hausa_ajami_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hausa_ajami_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hausa_kano.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hausa_kano.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hausa_kano.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hausa_kano.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hcesar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hcesar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hcesar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hcesar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hieroglyphic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hieroglyphic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hieroglyphic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hieroglyphic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/himyarit_musnad.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/himyarit_musnad.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/himyarit_musnad.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/himyarit_musnad.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/hindi_modular.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/hindi_modular.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/hindi_modular.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/hindi_modular.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/indigenous_nt.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/indigenous_nt.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/indigenous_nt.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/indigenous_nt.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/indonesia.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/indonesia.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/indonesia.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/indonesia.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/indonesian_suku.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/indonesian_suku.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/indonesian_suku.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/indonesian_suku.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/inuktitut_naqittaut.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/inuktitut_naqittaut.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/inuktitut_naqittaut.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/inuktitut_naqittaut.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ishkashimi_cyrillic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ishkashimi_cyrillic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ishkashimi_cyrillic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ishkashimi_cyrillic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_bengali.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_bengali.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_bengali.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_bengali.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_devanagari_hindi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_devanagari_hindi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_devanagari_hindi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_devanagari_hindi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_devanagari_sanskrit_vedic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_devanagari_sanskrit_vedic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_devanagari_sanskrit_vedic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_devanagari_sanskrit_vedic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_gujarati.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_gujarati.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_gujarati.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_gujarati.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_gurmukhi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_gurmukhi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_gurmukhi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_gurmukhi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_odia.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_odia.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_odia.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_odia.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/itrans_roman.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_roman.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/itrans_roman.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/itrans_roman.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/jawa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/jawa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/jawa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/jawa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/jorai.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/jorai.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/jorai.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/jorai.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/karakalpak_cyrillic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/karakalpak_cyrillic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/karakalpak_cyrillic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/karakalpak_cyrillic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/karakalpak_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/karakalpak_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/karakalpak_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/karakalpak_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/kayan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/kayan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/kayan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/kayan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/kbdsn1.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/kbdsn1.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/kbdsn1.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/kbdsn1.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/kharoshthi_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/kharoshthi_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/kharoshthi_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/kharoshthi_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/khmer_advanced.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/khmer_advanced.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/khmer_advanced.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/khmer_advanced.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/khmer_angkor.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/khmer_angkor.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/khmer_angkor.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/khmer_angkor.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/khojki_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/khojki_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/khojki_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/khojki_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/kmhmu_2008.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/kmhmu_2008.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/kmhmu_2008.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/kmhmu_2008.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/koalibrere.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/koalibrere.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/koalibrere.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/koalibrere.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/korean_rr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/korean_rr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/korean_rr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/korean_rr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/koreguaje.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/koreguaje.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/koreguaje.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/koreguaje.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/krung.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/krung.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/krung.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/krung.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lahu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lahu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lahu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lahu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lamkaang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lamkaang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lamkaang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lamkaang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/landuma.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/landuma.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/landuma.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/landuma.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lao_2008_basic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_2008_basic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lao_2008_basic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_2008_basic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lao_2008_rapid.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_2008_rapid.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lao_2008_rapid.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_2008_rapid.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lao_pali.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_pali.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lao_pali.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_pali.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lao_pali_us.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_pali_us.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lao_pali_us.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_pali_us.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lao_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lao_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lao_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/lazuri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/lazuri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/lazuri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/lazuri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/libtralo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/libtralo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/libtralo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/libtralo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/makasar_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/makasar_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/makasar_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/makasar_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/malar_braille.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_braille.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/malar_braille.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_braille.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/malar_malayalam.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_malayalam.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/malar_malayalam.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_malayalam.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/malar_malayalam_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_malayalam_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/malar_malayalam_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_malayalam_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/malar_tirhuta.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_tirhuta.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/malar_tirhuta.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/malar_tirhuta.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/maltese.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/maltese.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/maltese.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/maltese.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mandaic_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mandaic_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mandaic_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mandaic_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/masaram_gondi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/masaram_gondi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/masaram_gondi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/masaram_gondi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/me_en.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/me_en.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/me_en.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/me_en.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/meitei_legacy.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/meitei_legacy.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/meitei_legacy.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/meitei_legacy.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/miluk_hanis_siuslaw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/miluk_hanis_siuslaw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/miluk_hanis_siuslaw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/miluk_hanis_siuslaw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/modi_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/modi_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/modi_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/modi_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mon_anonta.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mon_anonta.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mon_anonta.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mon_anonta.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mon_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mon_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mon_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mon_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mongolian_cyrillic_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mongolian_cyrillic_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mongolian_cyrillic_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mongolian_cyrillic_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mozhi_malayalam.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mozhi_malayalam.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mozhi_malayalam.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mozhi_malayalam.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/mro_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/mro_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/mro_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/mro_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/multani_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/multani_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/multani_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/multani_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/multi_pak_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/multi_pak_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/multi_pak_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/multi_pak_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/munji.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/munji.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/munji.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/munji.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/myancode_san.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/myancode_san.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/myancode_san.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/myancode_san.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nabataean_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nabataean_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nabataean_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nabataean_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/naijatype.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/naijatype.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/naijatype.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/naijatype.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nailangs.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nailangs.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nailangs.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nailangs.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nasa_yuwe.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nasa_yuwe.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nasa_yuwe.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nasa_yuwe.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nepali_traditional.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nepali_traditional.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nepali_traditional.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nepali_traditional.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/newa_romanized.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/newa_romanized.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/newa_romanized.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/newa_romanized.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/newa_traditional.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/newa_traditional.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/newa_traditional.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/newa_traditional.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nias.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nias.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nias.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nias.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nisenan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nisenan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nisenan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nisenan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nko.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nko.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nko.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nko.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nkonya.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nkonya.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nkonya.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nkonya.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_bengali_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_bengali_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_bengali_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_bengali_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_devanagari_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_devanagari_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_devanagari_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_devanagari_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_gujarati_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_gujarati_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_gujarati_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_gujarati_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_gurmukhi_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_gurmukhi_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_gurmukhi_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_gurmukhi_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_ipa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_ipa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_ipa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_ipa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_kannada_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_kannada_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_kannada_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_kannada_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_malayalam_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_malayalam_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_malayalam_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_malayalam_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_oriya_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_oriya_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_oriya_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_oriya_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_tamil_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_tamil_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_tamil_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_tamil_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nlci_telugu_winscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_telugu_winscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nlci_telugu_winscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nlci_telugu_winscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nrc_makah.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nrc_makah.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nrc_makah.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nrc_makah.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ntl_onekey.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ntl_onekey.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ntl_onekey.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ntl_onekey.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/numanggang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/numanggang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/numanggang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/numanggang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/nw_iranian_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/nw_iranian_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/nw_iranian_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/nw_iranian_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/o_tissi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/o_tissi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/o_tissi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/o_tissi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/obolo_chwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/obolo_chwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/obolo_chwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/obolo_chwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/obolo_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/obolo_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/obolo_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/obolo_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/old_hungarian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/old_hungarian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/old_hungarian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/old_hungarian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/old_turkic_udw21_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/old_turkic_udw21_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/old_turkic_udw21_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/old_turkic_udw21_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/orma.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/orma.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/orma.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/orma.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/osage_nation.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/osage_nation.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/osage_nation.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/osage_nation.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/osage_nation_new.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/osage_nation_new.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/osage_nation_new.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/osage_nation_new.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/otoe_missouria.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/otoe_missouria.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/otoe_missouria.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/otoe_missouria.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/persian_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/persian_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/persian_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/persian_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/pid_piaroa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/pid_piaroa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/pid_piaroa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/pid_piaroa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/pingelap.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/pingelap.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/pingelap.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/pingelap.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/postmodern_english_uk_dualstroke.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_uk_dualstroke.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/postmodern_english_uk_dualstroke.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_uk_dualstroke.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/postmodern_english_uk_natural.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_uk_natural.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/postmodern_english_uk_natural.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_uk_natural.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/postmodern_english_us_dualstroke.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_us_dualstroke.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/postmodern_english_us_dualstroke.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_us_dualstroke.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/postmodern_english_us_natural.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_us_natural.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/postmodern_english_us_natural.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/postmodern_english_us_natural.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/pukapuka.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/pukapuka.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/pukapuka.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/pukapuka.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/qom.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/qom.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/qom.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/qom.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/quinault.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/quinault.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/quinault.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/quinault.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/qwerty_farang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/qwerty_farang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/qwerty_farang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/qwerty_farang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_aer.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_aer.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_aer.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_aer.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_arabic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_arabic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_arabic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_arabic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_balti.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_balti.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_balti.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_balti.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_brahui.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_brahui.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_brahui.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_brahui.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_brahui_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_brahui_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_brahui_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_brahui_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_burushaski.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_burushaski.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_burushaski.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_burushaski.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_dameli.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dameli.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_dameli.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dameli.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_dhatki.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dhatki.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_dhatki.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dhatki.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_dogri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dogri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_dogri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_dogri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_gawar_bati.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_gawar_bati.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_gawar_bati.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_gawar_bati.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_gawri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_gawri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_gawri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_gawri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_hazaragi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_hazaragi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_hazaragi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_hazaragi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_hindko.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_hindko.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_hindko.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_hindko.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_indus_kohistani.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_indus_kohistani.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_indus_kohistani.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_indus_kohistani.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_kalasha.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kalasha.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_kalasha.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kalasha.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_kashmir_shina.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kashmir_shina.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_kashmir_shina.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kashmir_shina.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_kashmiri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kashmiri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_kashmiri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_kashmiri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_khowar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_khowar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_khowar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_khowar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_marwari.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_marwari.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_marwari.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_marwari.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_munji.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_munji.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_munji.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_munji.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_oadki.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_oadki.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_oadki.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_oadki.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_ormuri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_ormuri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_ormuri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_ormuri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_pahari.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pahari.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_pahari.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pahari.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_palula.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_palula.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_palula.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_palula.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_parkari_koli.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_parkari_koli.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_parkari_koli.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_parkari_koli.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_pashai.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pashai.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_pashai.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pashai.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_pashto.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pashto.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_pashto.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_pashto.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_saraiki.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_saraiki.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_saraiki.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_saraiki.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_shina.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_shina.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_shina.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_shina.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_sindhi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_sindhi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_sindhi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_sindhi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_torwali.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_torwali.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_torwali.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_torwali.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_urdu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_urdu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_urdu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_urdu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_ushojo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_ushojo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_ushojo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_ushojo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_uyghur.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_uyghur.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_uyghur.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_uyghur.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_wadiyara.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_wadiyara.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_wadiyara.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_wadiyara.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_wakhi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_wakhi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_wakhi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_wakhi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_western_punjabi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_western_punjabi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_western_punjabi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_western_punjabi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rac_yidgha.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_yidgha.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rac_yidgha.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rac_yidgha.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rawang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rawang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rawang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rawang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rejang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rejang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rejang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rejang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/remington_gail.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/remington_gail.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/remington_gail.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/remington_gail.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rohingya_arab.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rohingya_arab.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rohingya_arab.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rohingya_arab.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/rossel.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/rossel.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/rossel.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/rossel.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/runeboard.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/runeboard.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/runeboard.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/runeboard.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/russian_mnemonic_r.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/russian_mnemonic_r.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/russian_mnemonic_r.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/russian_mnemonic_r.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sabdalipi_assamese.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sabdalipi_assamese.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sabdalipi_assamese.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sabdalipi_assamese.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sahaptin_umatilla.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sahaptin_umatilla.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sahaptin_umatilla.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sahaptin_umatilla.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sahaptin_yakima.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sahaptin_yakima.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sahaptin_yakima.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sahaptin_yakima.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sanjha_punjabi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sanjha_punjabi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sanjha_punjabi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sanjha_punjabi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/santali_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/santali_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/santali_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/santali_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/saraiki.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/saraiki.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/saraiki.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/saraiki.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/satere.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/satere.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/satere.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/satere.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/shahmukhi_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/shahmukhi_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/shahmukhi_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/shahmukhi_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/shan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/shan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/shan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/shan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/shaw_2layer.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/shaw_2layer.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/shaw_2layer.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/shaw_2layer.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/siddham_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/siddham_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/siddham_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/siddham_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_akha_act.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_akha_act.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_akha_act.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_akha_act.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_arabic_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_arabic_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_arabic_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_arabic_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_areare.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_areare.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_areare.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_areare.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bari.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bari.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bari.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bari.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bengali_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bengali_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bengali_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bengali_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bolivia.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bolivia.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bolivia.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bolivia.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_boonkit.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_boonkit.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_boonkit.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_boonkit.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_brao.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_brao.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_brao.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_brao.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bru.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bru.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bru.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bru.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_buang.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_buang.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_buang.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_buang.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bunong.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bunong.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bunong.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bunong.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_busa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_busa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_busa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_busa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_bwe_karen.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bwe_karen.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_bwe_karen.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_bwe_karen.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_cameroon_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cameroon_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_cameroon_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cameroon_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_cameroon_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cameroon_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_cameroon_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cameroon_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_cherokee_nation.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cherokee_nation.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_cherokee_nation.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cherokee_nation.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_cheyenne.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cheyenne.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_cheyenne.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cheyenne.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_cipher_music.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cipher_music.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_cipher_music.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_cipher_music.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_devanagari_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_devanagari_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_devanagari_romanized.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_romanized.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_devanagari_romanized.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_romanized.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_devanagari_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_devanagari_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_devanagari_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_dzongkha.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_dzongkha.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_dzongkha.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_dzongkha.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_eastern_congo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_eastern_congo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_eastern_congo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_eastern_congo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_el_ethiopian_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_el_ethiopian_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_el_ethiopian_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_el_ethiopian_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_ethiopic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ethiopic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_ethiopic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ethiopic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_ethiopic_power_g.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ethiopic_power_g.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_ethiopic_power_g.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ethiopic_power_g.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_euro_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_euro_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_euro_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_euro_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_extended_urdu_np.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_extended_urdu_np.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_extended_urdu_np.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_extended_urdu_np.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_greek_polytonic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_greek_polytonic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_greek_polytonic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_greek_polytonic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_hawaiian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hawaiian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_hawaiian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hawaiian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_hebr_grek_trans.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebr_grek_trans.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_hebr_grek_trans.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebr_grek_trans.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_hebrew.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebrew.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_hebrew.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebrew.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_hebrew_legacy.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebrew_legacy.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_hebrew_legacy.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hebrew_legacy.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_hmd_plrd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hmd_plrd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_hmd_plrd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_hmd_plrd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_indic_roman.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_indic_roman.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_indic_roman.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_indic_roman.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_ipa.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ipa.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_ipa.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ipa.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_jarai.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_jarai.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_jarai.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_jarai.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_kayah_kali.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_kali.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_kayah_kali.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_kali.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_kayah_latn.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_latn.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_kayah_latn.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_latn.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_kayah_mymr.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_mymr.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_kayah_mymr.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kayah_mymr.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_khamti.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khamti.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_khamti.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khamti.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_khmer.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khmer.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_khmer.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khmer.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_khowar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khowar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_khowar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_khowar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_kmhmu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kmhmu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_kmhmu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kmhmu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_korda_jamo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korda_jamo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_korda_jamo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korda_jamo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_korda_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korda_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_korda_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korda_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_korean_morse.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korean_morse.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_korean_morse.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_korean_morse.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_kvl_kayaw.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kvl_kayaw.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_kvl_kayaw.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_kvl_kayaw.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_lepcha.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lepcha.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_lepcha.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lepcha.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_limbu_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_limbu_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_limbu_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_limbu_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_limbu_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_limbu_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_limbu_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_limbu_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_lisu_basic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lisu_basic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_lisu_basic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lisu_basic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_lisu_standard.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lisu_standard.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_lisu_standard.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lisu_standard.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_lpo_plrd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lpo_plrd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_lpo_plrd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_lpo_plrd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_madi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_madi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_madi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_madi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_makuri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_makuri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_makuri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_makuri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_mali_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_mali_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_mali_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_mali_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_mali_qwertz.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_qwertz.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_mali_qwertz.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_mali_qwertz.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_moore.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_moore.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_moore.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_moore.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_myanmar_my3.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_myanmar_my3.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_myanmar_my3.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_myanmar_my3.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_myanmar_mywinext.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_myanmar_mywinext.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_myanmar_mywinext.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_myanmar_mywinext.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_nigeria_dot.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_dot.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_nigeria_dot.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_dot.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_nigeria_odd_vowels.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_odd_vowels.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_nigeria_odd_vowels.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_odd_vowels.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_nigeria_underline.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_underline.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_nigeria_underline.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nigeria_underline.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_nko.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nko.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_nko.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nko.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_nubian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nubian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_nubian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_nubian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_pan_africa_mnemonic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_pan_africa_mnemonic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_pan_africa_mnemonic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_pan_africa_mnemonic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_pan_africa_positional.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_pan_africa_positional.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_pan_africa_positional.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_pan_africa_positional.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_philippines.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_philippines.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_philippines.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_philippines.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_sahu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_sahu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_sahu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_sahu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_bsc_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_bsc_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_bsc_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_bsc_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_cou_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_cou_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_cou_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_cou_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_dyo_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_dyo_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_dyo_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_dyo_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_gsl_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_gsl_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_gsl_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_gsl_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_krx_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_krx_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_krx_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_krx_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_ndv_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_ndv_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_ndv_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_ndv_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_sav_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_sav_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_sav_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_sav_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_snf_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_snf_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_snf_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_snf_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_srr_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_srr_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_srr_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_srr_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_senegal_wo_azerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_wo_azerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_senegal_wo_azerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_senegal_wo_azerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_sgaw_karen.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_sgaw_karen.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_sgaw_karen.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_sgaw_karen.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_shan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_shan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_shan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_shan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tai_dam.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tai_dam.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tai_dam_lao.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_lao.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tai_dam_lao.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_lao.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tai_dam_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tai_dam_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tai_dam_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tai_dam_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tai_dam_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tawallammat.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tawallammat.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tawallammat.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tawallammat.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tchad.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tchad.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tchad.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tchad.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tepehuan.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tepehuan.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tepehuan.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tepehuan.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_torwali.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_torwali.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_torwali.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_torwali.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_tunisian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tunisian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_tunisian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_tunisian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_uganda_tanzania.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_uganda_tanzania.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_uganda_tanzania.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_uganda_tanzania.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_vai.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_vai.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_vai.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_vai.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_wayuu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_wayuu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_wayuu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_wayuu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_ygp_plrd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ygp_plrd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_ygp_plrd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ygp_plrd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yna_plrd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yna_plrd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yna_plrd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yna_plrd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yoruba8.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba8.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yoruba8.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba8.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yoruba_bar.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba_bar.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yoruba_bar.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba_bar.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yoruba_dot.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba_dot.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yoruba_dot.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yoruba_dot.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yupik_cyrillic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yupik_cyrillic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yupik_cyrillic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yupik_cyrillic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_yupik_cyrillic_ru.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yupik_cyrillic_ru.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_yupik_cyrillic_ru.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_yupik_cyrillic_ru.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sil_ywq_plrd.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ywq_plrd.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sil_ywq_plrd.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sil_ywq_plrd.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/slc_saliba.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/slc_saliba.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/slc_saliba.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/slc_saliba.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/soqotri_arabic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/soqotri_arabic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/soqotri_arabic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/soqotri_arabic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/srr_ajami_qwerty.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/srr_ajami_qwerty.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/srr_ajami_qwerty.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/srr_ajami_qwerty.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sundanese.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sundanese.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sundanese.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sundanese.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sundanese_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sundanese_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sundanese_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sundanese_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/swanalekha_malayalam.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/swanalekha_malayalam.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/swanalekha_malayalam.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/swanalekha_malayalam.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sxava.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sxava.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sxava.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sxava.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sxava_eo.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sxava_eo.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sxava_eo.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sxava_eo.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/sylheti_nagri.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/sylheti_nagri.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/sylheti_nagri.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/sylheti_nagri.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/syriac_arabic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/syriac_arabic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/syriac_arabic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/syriac_arabic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/syriac_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/syriac_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/syriac_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/syriac_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tagbanwa_inscript.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tagbanwa_inscript.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tagbanwa_inscript.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tagbanwa_inscript.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/taigi_poj.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/taigi_poj.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/taigi_poj.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/taigi_poj.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tainua.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tainua.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tainua.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tainua.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tangsa_lakhum.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tangsa_lakhum.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tangsa_lakhum.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tangsa_lakhum.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tawallammat_latin.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tawallammat_latin.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tawallammat_latin.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tawallammat_latin.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/teggargrent_lat.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/teggargrent_lat.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/teggargrent_lat.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/teggargrent_lat.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tem_kdh.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tem_kdh.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tem_kdh.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tem_kdh.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/thamizha_anjal_paangu.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_anjal_paangu.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/thamizha_anjal_paangu.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_anjal_paangu.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/thamizha_bamini.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_bamini.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/thamizha_bamini.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_bamini.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/thamizha_new_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_new_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/thamizha_new_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_new_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/thamizha_tamil99_ext.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_tamil99_ext.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/thamizha_tamil99_ext.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/thamizha_tamil99_ext.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tibetan_direct_input.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tibetan_direct_input.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tibetan_direct_input.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tibetan_direct_input.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tibetan_ewts.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tibetan_ewts.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tibetan_ewts.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tibetan_ewts.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tirhuta.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tirhuta.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tirhuta.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tirhuta.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tlahuica.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tlahuica.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tlahuica.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tlahuica.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tsakonian.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tsakonian.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tsakonian.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tsakonian.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/tuareg_tifinagh.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/tuareg_tifinagh.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/tuareg_tifinagh.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/tuareg_tifinagh.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/turkmen_cyrl.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/turkmen_cyrl.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/turkmen_cyrl.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/turkmen_cyrl.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/txo_toto.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/txo_toto.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/txo_toto.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/txo_toto.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/udi_keyboard.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/udi_keyboard.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/udi_keyboard.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/udi_keyboard.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/ukwuani.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/ukwuani.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/ukwuani.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/ukwuani.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/uma_graphic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/uma_graphic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/uma_graphic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/uma_graphic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/uma_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/uma_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/uma_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/uma_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/urdu_phonetic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/urdu_phonetic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/urdu_phonetic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/urdu_phonetic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/urdu_phonetic_crulp.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/urdu_phonetic_crulp.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/urdu_phonetic_crulp.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/urdu_phonetic_crulp.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/venetia_et_histria.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/venetia_et_histria.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/venetia_et_histria.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/venetia_et_histria.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/vm_tamil_modular.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/vm_tamil_modular.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/vm_tamil_modular.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/vm_tamil_modular.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/vm_tamil_typewriter.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/vm_tamil_typewriter.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/vm_tamil_typewriter.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/vm_tamil_typewriter.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/wakhi_anglicized.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_anglicized.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/wakhi_anglicized.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_anglicized.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/wakhi_cyrillic.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_cyrillic.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/wakhi_cyrillic.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_cyrillic.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/wakhi_standard.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_standard.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/wakhi_standard.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/wakhi_standard.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/wancho.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/wancho.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/wancho.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/wancho.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/warang_citi.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/warang_citi.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/warang_citi.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/warang_citi.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/wolofal.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/wolofal.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/wolofal.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/wolofal.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/xinaliq.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/xinaliq.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/xinaliq.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/xinaliq.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/yiddish_pasekh.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/yiddish_pasekh.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/yiddish_pasekh.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/yiddish_pasekh.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/yidgha.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/yidgha.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/yidgha.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/yidgha.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/younger_futhark_short_twig.kmx b/developer/src/kmcmplib/tests/fixtures/keyboards-repo/younger_futhark_short_twig.kmx similarity index 100% rename from developer/src/kmcmplib/tests/fixtures/younger_futhark_short_twig.kmx rename to developer/src/kmcmplib/tests/fixtures/keyboards-repo/younger_futhark_short_twig.kmx diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat new file mode 100644 index 0000000000000000000000000000000000000000..5e8b891ea6cf163c1e4dce6c089a835c6b153e78 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat @@ -0,0 +1,4 @@ +@echo off +echo Compiles the keyboards using the legacy kmcomp.exe +echo to use as baseline comparisons for kmcmplib +for %%d in (*.kmn) do ..\..\..\..\..\bin\kmcomp -no-compiler-version -d %%d \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn new file mode 100644 index 0000000000000000000000000000000000000000..2b3180c36ab4b473e74b1b3487fc8f0a61d623f6 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx new file mode 100644 index 0000000000000000000000000000000000000000..13ea2a686e09c16db68ff7bb355e114b9125d76d Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmn new file mode 100644 index 0000000000000000000000000000000000000000..da6d4e70873e5762dd49a41d13721abebb5ba668 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmn @@ -0,0 +1,11 @@ +c Description: Verifies that kmcmplib can compile a UTF-8 file (without BOM) + +store(&NAME) 'k002_utf8_without_bom' +store(&VERSION) '9.0' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx new file mode 100644 index 0000000000000000000000000000000000000000..0b9c9294182d9387165c98e1ffc84c8fe45452e3 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmn new file mode 100644 index 0000000000000000000000000000000000000000..b2bccb036caad4d891ef62c5bc31c02ebaaffddd --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmn @@ -0,0 +1,11 @@ +c Description: Verifies that kmcmplib can compile a UTF-8 file (with BOM) + +store(&NAME) 'k003_utf8_with_bom' +store(&VERSION) '9.0' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx new file mode 100644 index 0000000000000000000000000000000000000000..ee1cf4225502e3c7944b58f301e5be649af4ec00 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn new file mode 100644 index 0000000000000000000000000000000000000000..50a1b31ea1c7b4d369e66dc69021be22f9cdeacb --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn @@ -0,0 +1,13 @@ +c Description: Verifies that kmcmplib can compile an ANSI file +c This has some high-ascii letters in cp1252 to ensure that +c it fails to load as 'utf8 without bom' + +store(&NAME) 'k004_ansi' +store(&VERSION) '9.0' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ÀÐ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx new file mode 100644 index 0000000000000000000000000000000000000000..6c41da979cb2a3ec20aadf6de0ca778fb8ce5ec1 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp new file mode 100644 index 0000000000000000000000000000000000000000..509b89ae6df7f09a442ae60f215a6c7b3211b34f Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn new file mode 100644 index 0000000000000000000000000000000000000000..926ccdeeafbf5bf648903d0bdc530a5e0ffa9ce9 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load a bitmap without file extension + +store(&NAME) 'k005_bitmap' +store(&VERSION) '9.0' +store(&BITMAP) 'k005_bitmap' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx new file mode 100644 index 0000000000000000000000000000000000000000..bc9bc9b671724050e169e59fce976a18d54a030e Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.ico b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.ico new file mode 100644 index 0000000000000000000000000000000000000000..e1b64cf9013e6702420d97fd88b120490f33b137 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.ico differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmn new file mode 100644 index 0000000000000000000000000000000000000000..4ad3e61659683213ddc6b44a0454c87cc7576d11 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load an icon with file extension + +store(&NAME) 'k006_icon' +store(&VERSION) '9.0' +store(&BITMAP) 'k006_icon.ico' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ážáŸ’មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx new file mode 100644 index 0000000000000000000000000000000000000000..f994a1dcac9e5ae797eb88eed9cee4b0e6f85b00 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn new file mode 100644 index 0000000000000000000000000000000000000000..71d72d5120b522cd20303f3d0843e4720791230c --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load an includecodes file with \r\n line endings + +store(&NAME) 'k007_includecodes_r_n' +store(&VERSION) '9.0' +store(&includecodes) 'k007_includecodes_r_n.txt' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > $LOWER_A +$LOWER_A + [K_B] > $LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx new file mode 100644 index 0000000000000000000000000000000000000000..f8b4eac1b0d9103c7767e487b5109a06aed8e1ec Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.txt b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.txt new file mode 100644 index 0000000000000000000000000000000000000000..4cc3fe4351ddeb205cdd985d04648ec403065e35 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.txt @@ -0,0 +1,2 @@ +0061;LOWER_A +0062;LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmn new file mode 100644 index 0000000000000000000000000000000000000000..ea357b4b0ef6fc8331efb7f63aca9e3fc9291652 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load an includecodes file with \n line endings + +store(&NAME) 'k008_includecodes_n' +store(&VERSION) '9.0' +store(&includecodes) 'k008_includecodes_n.txt' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > $LOWER_A +$LOWER_A + [K_B] > $LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx new file mode 100644 index 0000000000000000000000000000000000000000..aa158cdd546cdde915e4a55c1db521e57d4abb04 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx differ diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt new file mode 100644 index 0000000000000000000000000000000000000000..4cc3fe4351ddeb205cdd985d04648ec403065e35 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt @@ -0,0 +1,2 @@ +0061;LOWER_A +0062;LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmn new file mode 100644 index 0000000000000000000000000000000000000000..a0ad9d0d9fbc820432982d13455f5a02acda4018 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmn @@ -0,0 +1,31 @@ +c Description: Verifies that kmcmplib can deal with long lines continued with backslashes, and also comments + +store(&NAME) 'k009_long_lines' +store(&VERSION) '9.0' + +begin unicode > use(main) + +group(main) using keys + +c +c ***Note*** This file has intentional whitespace at the end of line ending in [SHIFT K_N] +c in order to verify that line continuations are working correctly +c + +store(c_key) [K_K] [K_X] [SHIFT K_K] [SHIFT K_X] [K_G] \ + [K_C] [K_Q] [SHIFT K_C] [SHIFT K_Q] [SHIFT K_J] \ + [K_D] [K_Z] [SHIFT K_D] [SHIFT K_Z] [SHIFT K_N] \ + [K_T] [K_F] [SHIFT K_T] [SHIFT K_F] [K_N] \ + [K_B] [K_P] [SHIFT K_B] [SHIFT K_P] [K_M] \ + [K_Y] [K_R] [K_L] [K_V] [K_S] [K_H] [SHIFT K_L] [SHIFT K_G] \ + [RALT K_K] [RALT K_B] +store(c_out) U+1780 U+1781 U+1782 U+1783 U+1784 \ + U+1785 U+1786 U+1787 U+1788 U+1789 \ + U+178A U+178B U+178C U+178D U+178E \ + U+178F U+1790 U+1791 U+1792 U+1793 \ + U+1794 U+1795 U+1796 U+1797 U+1798 \ + U+1799 U+179A U+179B U+179C U+179F U+17A0 U+17A1 U+17A2 \ + U+179D U+179E c deprecated, but they are used in minority languages + ++ any(c_key) > index(c_out, 1) + diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmx new file mode 100644 index 0000000000000000000000000000000000000000..4acd4cc051adc10d35c6ae842b24be5cf43f0cf4 Binary files /dev/null and b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k009_long_lines.kmx differ diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 1ce542917d389df52ad136047d9b7ae09d15d9dd..5dbaee88bc9990af9513c8ffd84882250c25d92d 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -13,6 +13,8 @@ #include #include #include +#include "util_filesystem.h" +#include "util_callbacks.h" #ifdef _MSC_VER #else @@ -21,29 +23,11 @@ using namespace std; -vector < int > error_vec; - #define CERR_FATAL 0x00008000 #define CERR_ERROR 0x00004000 #define CERR_WARNING 0x00002000 #define CERR_HINT 0x00001000 -int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) -{ - error_vec.push_back(dwMsgCode); - const char*t = "unknown"; - switch(dwMsgCode & 0xF000) { - case CERR_HINT: t=" hint"; break; - case CERR_WARNING: t="warning"; break; - case CERR_ERROR: t=" error"; break; - case CERR_FATAL: t=" fatal"; break; - } - printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); - return 1; -} - -#include "../src/filesystem.h" - int main(int argc, char *argv[]) { if(argc < 4) { @@ -71,14 +55,15 @@ int main(int argc, char *argv[]) char first5[6] = "CERR_"; char* pfirst5 = first5; - KMCMP_COMPILER_OPTIONS kcopts; - kcopts.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); - kcopts.ShouldAddCompilerVersion = false; // So we can compare against existing compiled keyboards that also don't have compiler version - if(!kmcmp_SetCompilerOptions(&kcopts)) { - return __LINE__; - } + KMCMP_COMPILER_RESULT result; + KMCMP_COMPILER_OPTIONS options; + options.saveDebug = true; + options.compilerWarningsAsErrors = false; + options.warnDeprecatedCode = true; + options.shouldAddCompilerVersion = false; + options.target = CKF_KEYMAN; - if(kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr)) { + if(kmcmp_CompileKeyboard(kmn_file, options, msgproc, loadfileProc, nullptr, result)) { char* testname = strrchr( (char*) kmn_file, '/') + 1; if(strncmp(testname, pfirst5, 5) == 0){ return __LINE__; // exit code: CERR_ in Name + no Error found @@ -87,26 +72,24 @@ int main(int argc, char *argv[]) // On non-win32 platforms, we cannot get kmcmpdll.dll to build keyboards // legacy-mode, so we'll compare to a hopefully existing file that we've // been passed - FILE* fp1 = Open_File(kmx_file, "rb"); + FILE* fp1 = Open_File(kmx_file, "wb"); if(!fp1) return __LINE__; + // Write out for reference + fwrite(result.kmx, 1, result.kmxSize, fp1); + FILE* fp2 = Open_File(reference_kmx, "rb"); if(!fp2) return __LINE__; // exit code: fail if no reference kmx file in build-folder - fseek(fp1, 0, SEEK_END); - auto sz1 = ftell(fp1); - fseek(fp1, 0, SEEK_SET); fseek(fp2, 0, SEEK_END); auto sz2 = ftell(fp2); fseek(fp2, 0, SEEK_SET); - if (sz1 != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder + if (result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder - char* buf1 = new char[sz1]; - char* buf2 = new char[sz1]; - fread(buf1, 1, sz1, fp1); - fread(buf2, 1, sz1, fp2); - return memcmp(buf1, buf2, sz1) ? __LINE__ : 0; // exit code: when contents of kmx-file in build differs from contents of kmx-file in source folder + char* buf2 = new char[result.kmxSize]; + fread(buf2, 1, result.kmxSize, fp2); + return memcmp(result.kmx, buf2, result.kmxSize) ? __LINE__ : 0; // exit code: when contents of kmx-file in build differs from contents of kmx-file in source folder // success: when contents of kmx-file in build and source folder are the same } else { /*if Errors found: check number (e.g. CERR_4061_balochi_phonetic.kmn should produce Error 4061)*/ diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index b81429806e16a3d414c2626c34fba9caa4d2bc12..a048bc8236b61803f5f16ef5f91bad7b7e56eae0 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -6,16 +6,20 @@ fs = import('fs') -tests_flags = [] +tests_links = [] + +if cpp_compiler.get_id() == 'emscripten' + tests_links += ['-lnodefs.js'] +endif input_path = meson.current_source_dir() / '../../../../common/test/keyboards/baseline' output_path = meson.current_build_dir() -kmcompxtest = executable('kmcompxtest', 'kmcompxtest.cpp', - cpp_args: defns, +kmcompxtest = executable('kmcompxtest', ['kmcompxtest.cpp','util_filesystem.cpp','util_callbacks.cpp'], + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep, ) @@ -76,6 +80,26 @@ foreach kbd : tests test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(input_path, kbd) + '.kmx']) endforeach +valid_keyboard_tests = [ + 'k001_utf16', + 'k002_utf8_without_bom', + 'k003_utf8_with_bom', + # 'k004_ansi', # TODO: enable ansi test when we have the icu datafiles #8884 + 'k005_bitmap', + 'k006_icon', + 'k007_includecodes_r_n', + 'k008_includecodes_n', + 'k009_long_lines', +] + +fixtures_path = meson.current_source_dir() / 'fixtures/valid-keyboards' + +foreach kbd : valid_keyboard_tests + kbd_src = join_paths(fixtures_path, kbd) + '.kmn' + kbd_obj = join_paths(output_path, kbd) + '.kmx' + test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(fixtures_path, kbd) + '.kmx']) +endforeach + # Test fixtures that come from keyboards repo -- but only for a "full" test, # which typically we run on CI no more than once a day, because it's expensive. @@ -96,7 +120,7 @@ if get_option('full_test') # kbd is going to be an absolute path kbd_obj = output_path / test_basename + '.kmx' - reference_kmx = meson.current_source_dir() / 'fixtures' / test_basename + '.kmx' + reference_kmx = meson.current_source_dir() / 'fixtures/keyboards-repo' / test_basename + '.kmx' if fs.is_file(reference_kmx) test(test_basename, kmcompxtest, args: [kbd, kbd_obj, reference_kmx]) @@ -107,22 +131,22 @@ endif # Test the API endpoints -apitest = executable('api-test', 'api-test.cpp', - cpp_args: defns, +apitest = executable('api-test', ['api-test.cpp','util_filesystem.cpp','util_callbacks.cpp'], + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), - dependencies: icuuc_dep, + dependencies: icuuc_dep ) -test('api-test', apitest) +test('api-test', apitest, args: [output_path / 'blank_keyboard.kmx']) usetapitest = executable('uset-api-test', 'uset-api-test.cpp', - cpp_args: defns, + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep, ) diff --git a/developer/src/kmcmplib/tests/prep.sh b/developer/src/kmcmplib/tests/prep.sh index 058d4a1b9b0bfdd50fe13d29bcbdff7af508bd3c..8ad9b81171f165903fe74d4b7eb47f6f39154800 100755 --- a/developer/src/kmcmplib/tests/prep.sh +++ b/developer/src/kmcmplib/tests/prep.sh @@ -39,30 +39,30 @@ if [[ ! -d "$KEYBOARDS_ROOT/.git" ]] || [[ ! -f "$KEYBOARDS_ROOT/tools/regressio fi if builder_start_action clean; then - rm -rf "$THIS_SCRIPT_PATH/fixtures" + rm -rf "$THIS_SCRIPT_PATH/fixtures/keyboards-repo" rm -f "$THIS_SCRIPT_PATH/keyboards_commit_ref.txt" builder_finish_action success clean fi if builder_start_action build; then # yes, remove existing fixtures before build so we don't end up with stale fixtures - rm -rf "$THIS_SCRIPT_PATH/fixtures" + rm -rf "$THIS_SCRIPT_PATH/fixtures/keyboards-repo" # record the revision of the keyboards repo pushd "$KEYBOARDS_ROOT" > /dev/null git rev-parse head > "$THIS_SCRIPT_PATH/keyboards_commit_ref.txt" popd > /dev/null - "$KEYBOARDS_ROOT/tools/regression.sh" --use-legacy-compiler --local "$KEYMAN_ROOT/developer/bin/" --output "$THIS_SCRIPT_PATH/fixtures/" + "$KEYBOARDS_ROOT/tools/regression.sh" --use-legacy-compiler --local "$KEYMAN_ROOT/developer/bin/" --output "$THIS_SCRIPT_PATH/fixtures/keyboards-repo/" cd "$THIS_SCRIPT_PATH" # We don't need visual keyboards - rm ./fixtures/*.kvk + rm ./fixtures/keyboards-repo/*.kvk # Nor do we need kmw outputs - rm ./fixtures/*.js + rm ./fixtures/keyboards-repo/*.js # Finally, we remove any keyboards over 500kb, as we won't automatically test them for now # TODO: consider if we want to keep those large keyboards in the future - find ./fixtures/ -size +500k -exec rm {} \+ + find ./fixtures/keyboards-repo/ -size +500k -exec rm {} \+ builder_finish_action success build fi \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/uset-api-test.cpp b/developer/src/kmcmplib/tests/uset-api-test.cpp index 4fe599ca2e83a1af9d50d61368bafa4b337f3c82..61d263ad157cc365d885c463bc6bc200a893add7 100644 --- a/developer/src/kmcmplib/tests/uset-api-test.cpp +++ b/developer/src/kmcmplib/tests/uset-api-test.cpp @@ -19,30 +19,32 @@ #include "../src/compfile.h" #include -void test_kmcmp_ParseUnicodeSetProc(); +void test_kmcmp_parseUnicodeSet(); // std::vector error_vec; int main(int argc, char *argv[]) { - test_kmcmp_ParseUnicodeSetProc(); + test_kmcmp_parseUnicodeSet(); return 0; } -void test_kmcmp_ParseUnicodeSetProc() { +void test_kmcmp_parseUnicodeSet() { { // null test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[]", buf_, bufsiz); assert(rc == KMCMP_USET_OK); } { // basic test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[x A-C]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[x A-C]", buf_, bufsiz); assert(rc == 2); assert(buf[0] == 0x41); assert(buf[1] == 0x43); @@ -53,7 +55,8 @@ void test_kmcmp_ParseUnicodeSetProc() { // bigger test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[[🙀A-C]-[CB]]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[[🙀A-C]-[CB]]", buf_, bufsiz); assert(rc == 2); assert(buf[0] == 0x41); assert(buf[1] == 0x41); @@ -64,35 +67,40 @@ void test_kmcmp_ParseUnicodeSetProc() { // overflow test const auto bufsiz = 1; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[x A-C]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[x A-C]", buf_, bufsiz); assert(rc == KMCMP_FATAL_OUT_OF_RANGE); } { // err test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[:Adlm:]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[:Adlm:]", buf_, bufsiz); assert(rc == KMCMP_ERROR_UNSUPPORTED_PROPERTY); } { // err test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[[\\p{Mn}]&[A-Z]]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[[\\p{Mn}]&[A-Z]]", buf_, bufsiz); assert(rc == KMCMP_ERROR_UNSUPPORTED_PROPERTY); } { // err test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[abc{def}]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[abc{def}]", buf_, bufsiz); assert(rc == KMCMP_ERROR_HAS_STRINGS); } { // err test const auto bufsiz = 128; uint32_t buf[bufsiz]; - int rc = kmcmp_ParseUnicodeSet(u8"[[]", buf, bufsiz); + uintptr_t buf_ = reinterpret_cast(buf); + int rc = kmcmp_parseUnicodeSet(u8"[[]", buf_, bufsiz); assert(rc == KMCMP_ERROR_SYNTAX_ERR); } } diff --git a/developer/src/kmcmplib/tests/util_callbacks.cpp b/developer/src/kmcmplib/tests/util_callbacks.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e610a54c09e8beb6974675d23deb0cc5199140d6 --- /dev/null +++ b/developer/src/kmcmplib/tests/util_callbacks.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include "util_filesystem.h" +#include "../src/compfile.h" +#include + +std::vector error_vec; + +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { + error_vec.push_back(dwMsgCode); + const char*t = "unknown"; + switch(dwMsgCode & 0xF000) { + case CERR_HINT: t=" hint"; break; + case CERR_WARNING: t="warning"; break; + case CERR_ERROR: t=" error"; break; + case CERR_FATAL: t=" fatal"; break; + } + printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); + return 1; +} + +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { + std::string resolvedFilename = filename; + if(baseFilename && *baseFilename && IsRelativePath(filename)) { + char* p; + if ((p = strrchr_slash((char*)baseFilename)) != nullptr) { + std::string basePath = std::string(baseFilename, (int)(p - baseFilename + 1)); + resolvedFilename = basePath; + resolvedFilename.append(filename); + } + } + + FILE* fp = Open_File(resolvedFilename.c_str(), "rb"); + if(!fp) { + return false; + } + + if(!data) { + // return size + if(fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return false; + } + *size = ftell(fp); + if(*size == -1L) { + fclose(fp); + return false; + } + } else { + // return data + if(fread(data, 1, *size, fp) != *size) { + fclose(fp); + return false; + } + } + fclose(fp); + return true; +} \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/util_callbacks.h b/developer/src/kmcmplib/tests/util_callbacks.h new file mode 100644 index 0000000000000000000000000000000000000000..345de64b9f484b568f28c1652e800ee20e7307ee --- /dev/null +++ b/developer/src/kmcmplib/tests/util_callbacks.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context); +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context); + +extern std::vector error_vec; \ No newline at end of file diff --git a/developer/src/kmcmplib/src/filesystem.cpp b/developer/src/kmcmplib/tests/util_filesystem.cpp similarity index 84% rename from developer/src/kmcmplib/src/filesystem.cpp rename to developer/src/kmcmplib/tests/util_filesystem.cpp index a343641acc128ed5537e0e1835d45a2253c29f2e..7f9edd6b367a303573e165767ca1805f13bfd558 100644 --- a/developer/src/kmcmplib/src/filesystem.cpp +++ b/developer/src/kmcmplib/tests/util_filesystem.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "filesystem.h" +#include "util_filesystem.h" #ifdef _MSC_VER #include @@ -186,3 +186,42 @@ KMX_BOOL kmcmp_FileExists(const KMX_WCHAR* filename) { return FALSE; }; + + +bool IsRelativePath(KMX_CHAR const * p) { + // Relative path (returns TRUE): + // ..\...\BITMAP.BMP + // PATH\BITMAP.BMP + // BITMAP.BMP + + // Semi-absolute path (returns FALSE): + // \...\BITMAP.BMP + + // Absolute path (returns FALSE): + // C:\...\BITMAP.BMP + // \\SERVER\SHARE\...\BITMAP.BMP + + if ((*p == '\\') || (*p == '/')) return FALSE; + if (*p && *(p + 1) == ':') return FALSE; + + return TRUE; +} + +bool IsRelativePath(KMX_WCHAR const * p) { + // Relative path (returns TRUE): + // ..\...\BITMAP.BMP + // PATH\BITMAP.BMP + // BITMAP.BMP + + // Semi-absolute path (returns FALSE): + // \...\BITMAP.BMP + + // Absolute path (returns FALSE): + // C:\...\BITMAP.BMP + // \\SERVER\SHARE\...\BITMAP.BMP + + if ((*p == u'\\') || (*p == u'/'))return FALSE; + if (*p && *(p + 1) == u':') return FALSE; + + return TRUE; +} \ No newline at end of file diff --git a/developer/src/kmcmplib/src/filesystem.h b/developer/src/kmcmplib/tests/util_filesystem.h similarity index 82% rename from developer/src/kmcmplib/src/filesystem.h rename to developer/src/kmcmplib/tests/util_filesystem.h index 786b7268ac0c09bdda634296445e7e448404cc1e..4c94dae6bf4e4049d80efe8fa5d52d4b4fd7bb93 100644 --- a/developer/src/kmcmplib/src/filesystem.h +++ b/developer/src/kmcmplib/tests/util_filesystem.h @@ -1,7 +1,7 @@ #pragma once #include -#include "kmx_u16.h" +#include "../src/kmx_u16.h" // Opens files on windows and non-windows platforms. Datatypes for Filename and mode must be the same. // returns FILE* if file could be opened; FILE needs to be closed in calling function @@ -10,3 +10,6 @@ FILE* Open_File(const KMX_WCHART* Filename, const KMX_WCHART* mode); FILE* Open_File(const KMX_WCHAR* Filename, const KMX_WCHAR* mode); KMX_BOOL kmcmp_FileExists(const KMX_CHAR *filename); KMX_BOOL kmcmp_FileExists(const KMX_WCHAR *filename); + +bool IsRelativePath(KMX_CHAR const * p); +bool IsRelativePath(KMX_WCHAR const * p); diff --git a/developer/src/kmcomp/main.pas b/developer/src/kmcomp/main.pas index 5a29927a807bccd47dea671dbb6ce8d629a1f727..4aa29cf6c1ace54494285d0f9bd9c1f701d45040 100644 --- a/developer/src/kmcomp/main.pas +++ b/developer/src/kmcomp/main.pas @@ -65,7 +65,7 @@ uses UKeymanTargets; function CompileKeyboard(FInFile, FOutFile: string; FDebug, FWarnAsError: Boolean): Boolean; forward; // I4706 -function KCSetCompilerOptions(const FInFile: string; FShouldAddCompilerVersion, FUseKmcmpLib: Boolean): Boolean; forward; +function KCSetCompilerOptions(const FInFile: string; FShouldAddCompilerVersion: Boolean): Boolean; forward; //function CompilerMessage(line: Integer; msgcode: LongWord; text: PAnsiChar): Integer; stdcall; forward; procedure FixupPathSlashes(var path: string); forward; @@ -90,7 +90,6 @@ var FParamDistribution: Boolean; FMergingValidateIds: Boolean; FShouldAddCompilerVersion: Boolean; - FUseKmcmpLib: Boolean; FJsonSchemaPath: string; FParamSourcePath: string; FParamHelpLink: string; @@ -117,7 +116,6 @@ begin FColorMode := cmDefault; FShouldAddCompilerVersion := True; - FUseKmcmpLib := True; FParamInfile := ''; FParamOutfile := ''; @@ -193,8 +191,6 @@ begin FColorMode := cmForceNoColor else if s = '-no-compiler-version' then FShouldAddCompilerVersion := False - else if s = '-use-legacy-compiler' then - FUseKmcmpLib := False else if (s = '-help') or (s = '-h') then begin // Force help @@ -223,8 +219,6 @@ begin writeln(SKeymanDeveloperName + ' Compiler (32-bit)'); {$ENDIF} writeln('Version ' + CKeymanVersionInfo.VersionWithTag + ', ' + GetVersionCopyright); - if not FUseKmcmpLib then - writeln('Note: using legacy compiler'); end; if FError or (FParamInfile = '') then @@ -234,7 +228,7 @@ begin writeln(''); writeln('Usage: '+cmd+' [-s[s]] [-nologo] [-c] [-d] [-w] [-cfc] [-v[s|d]] [-source-path path] [-schema-path path] '); writeln(' '+spc+' [-m] infile [-m infile] [-t target] [outfile.kmx|outfile.js [error.log]]'); // I4699 - writeln(' '+spc+' [-add-help-link path] [-color|-no-color] [-no-compiler-version] [-use-legacy-compiler]'); + writeln(' '+spc+' [-add-help-link path] [-color|-no-color] [-no-compiler-version]'); writeln(' '+spc+' [-extract-keyboard-info field[,field...]]'); writeln(' infile can be a .kmn file (Keyboard Source, .kps file (Package Source), or .kpj (project)'); // I4699 // I4825 writeln(' if -v specified, can also be a .keyboard_info file'); @@ -258,7 +252,6 @@ begin writeln(' uses console mode to determine whether color should be used.'); writeln; writeln(' -no-compiler-version Don''t embed the compiler version stores, useful for regression tests.'); - writeln(' -use-legacy-compiler Use legacy compiler (will be removed in 18.0)'); writeln; writeln(' JSON .keyboard_info compile targets:'); writeln(' -v[s] validate infile against source schema'); @@ -298,7 +291,7 @@ begin TProjectLogConsole.Create(FSilent, FFullySilent, hOutfile, FColorMode); - KCSetCompilerOptions(FParamInfile, FShouldAddCompilerVersion, FUseKmcmpLib); + KCSetCompilerOptions(FParamInfile, FShouldAddCompilerVersion); if FValidateRepoChanges then FError := not TValidateRepoChanges.Execute(FParamInfile, FParamOutfile) @@ -324,7 +317,7 @@ begin ExitCode := 1; end; -function KCSetCompilerOptions(const FInFile: string; FShouldAddCompilerVersion, FUseKmcmpLib: Boolean): Boolean; +function KCSetCompilerOptions(const FInFile: string; FShouldAddCompilerVersion: Boolean): Boolean; var opt: TCompilerOptions; begin @@ -332,7 +325,6 @@ begin opt.dwSize := sizeof(TCompilerOptions); opt.ShouldAddCompilerVersion := FShouldAddCompilerVersion; - opt.UseKmcmpLib := FUseKmcmpLib; Result := SetCompilerOptions(@opt, @CompilerMessageW); diff --git a/developer/src/server/build.sh b/developer/src/server/build.sh index 53585f1df6b567f62a912e47239d8177c2aa8a22..c735e26fecb3f93b9cd7962f5726cc48969c656c 100755 --- a/developer/src/server/build.sh +++ b/developer/src/server/build.sh @@ -124,20 +124,18 @@ fi if (( build_keymanweb )); then pushd "$KEYMAN_ROOT/web/" - ./build.sh --no-minify + ./build.sh build --debug popd fi if (( copy_keymanweb )); then - WEB_SRC="$KEYMAN_ROOT/web/build/app/web/debug" - UI_SRC="$KEYMAN_ROOT/web/build/app/ui/debug" + WEB_SRC="$KEYMAN_ROOT/web/build/publish/debug" DST="$(dirname "$THIS_SCRIPT")/src/site/resource" rm -rf "$DST" mkdir -p "$DST/osk" mkdir -p "$DST/ui" cp "$WEB_SRC/"*.js "$WEB_SRC/"*.js.map "$DST/" - cp "$UI_SRC/"*.js "$UI_SRC/"*.js.map "$DST/" cp -R "$WEB_SRC/osk/"* "$DST/osk/" cp -R "$WEB_SRC/ui/"* "$DST/ui/" cp "$KEYMAN_ROOT/web/LICENSE" "$DST/" diff --git a/developer/src/server/src/site/chargrid.js b/developer/src/server/src/site/chargrid.js index 1ce9206bae57604d9fcf6ee7abec88a38098413b..f6c31f2df30dd4acd221dcc6cdc0004f357354d4 100644 --- a/developer/src/server/src/site/chargrid.js +++ b/developer/src/server/src/site/chargrid.js @@ -73,17 +73,10 @@ function updateLogCursor() { var i, selStart, selLength, selDirection; - if(keyman.isPositionSynthesized()) { // this is an internal function - // For touch devices, we need to ask KMW - selStart = 0; - selLength = 0; - selDirection = 'forward'; - } else { - // For desktop devices, we use the position reported by the textarea control - selStart = ta1.selectionStart; - selLength = ta1.selectionEnd - ta1.selectionStart; - selDirection = ta1.selectionDirection; - } + // We use the position reported by the textarea control + selStart = ta1.selectionStart; + selLength = ta1.selectionEnd - ta1.selectionStart; + selDirection = ta1.selectionDirection; selLength = calculateLengthByCodepoint(ta1.value, selStart, selLength); selStart = calculateLengthByCodepoint(ta1.value, 0, selStart); diff --git a/developer/src/server/src/site/test.js b/developer/src/server/src/site/test.js index 139765cc4c9df05b477351d3992e7d78d0d09fad..60a27cc484f6cb85ef1df344bafeefbbed4eec79 100644 --- a/developer/src/server/src/site/test.js +++ b/developer/src/server/src/site/test.js @@ -221,7 +221,7 @@ window.onload = function() { // Create a new on screen keyboard view and tell KeymanWeb that // we are using the targetDevice for context input. - newOSK = new com.keyman.osk.InlinedOSKView(targetDevice, keyman.util.device.coreSpec); + newOSK = new keyman.views.InlinedOSKView(keyman, { device: targetDevice }); keyman.core.contextDevice = targetDevice; keyman.osk = newOSK; @@ -238,11 +238,10 @@ window.onload = function() { keyman.addEventListener('keyboardchange', function(keyboardProperties) { if(newOSK) { keyman.osk = newOSK; - newOSK.activeKeyboard = keyman.core.activeKeyboard; + newOSK.activeKeyboard = keyman.contextManager.activeKeyboard; // Private API refs on both sides } keyboardDropdown.set(keyboardProperties.internalName); window.sessionStorage.setItem('current-keyboard', keyboardProperties.internalName); - keyman.alignInputs(); }); } @@ -285,7 +284,7 @@ function unloadKeyboardsAndModels() { const lastModel = keyman.core.activeModel; if(lastModel) { console.log('Unregistering model '+lastModel.id); - keyman.modelManager.deregister(lastModel.id); + keyman.removeModel(lastModel.id); } modelDropdown.removeAll(); @@ -360,7 +359,7 @@ function selectModel(modelId) { const lastModel = keyman.core.activeModel; if(lastModel) { - keyman.modelManager.deregister(lastModel.id); + keyman.removeModel(lastModel.id); } if(model) { diff --git a/developer/src/tike/project/Keyman.Developer.System.Project.kmnProjectFileAction.pas b/developer/src/tike/project/Keyman.Developer.System.Project.kmnProjectFileAction.pas index 703a1e4a0b6edd891950b647e5240c2e1250f02b..2b6dea94ec580d34adc0c54f471f021683ec3b44 100644 --- a/developer/src/tike/project/Keyman.Developer.System.Project.kmnProjectFileAction.pas +++ b/developer/src/tike/project/Keyman.Developer.System.Project.kmnProjectFileAction.pas @@ -115,7 +115,7 @@ begin TProject.CompilerMessageFile := Self; options.dwSize := sizeof(COMPILER_OPTIONS); options.ShouldAddCompilerVersion := addVersion; - options.UseKmcmpLib := not useLegacyCompiler; + // TODO: useLegacyCompiler means we switch to kmcmpdll vs kmc if not SetCompilerOptions(@options, ProjectCompilerMessage) then begin Log(plsFatal, 'Unable to set compiler options', CERR_FATAL, 0); diff --git a/docs/build/macos.md b/docs/build/macos.md index d8e4f61696ca7b32183d6257051d04221a166688..d670001e074e6dd38b5ee8aaea47d6a3546dbc99 100644 --- a/docs/build/macos.md +++ b/docs/build/macos.md @@ -66,7 +66,7 @@ These dependencies are also listed below if you'd prefer to install manually. PATH="$HOMEBREW_PREFIX/opt/coreutils/libexec/gnubin:$PATH" ``` -* Web: node.js, emscripten, openjdk 8 +* Web: node.js 18+, emscripten, openjdk 8 ```shell brew install node emscripten openjdk@8 diff --git a/docs/build/windows.md b/docs/build/windows.md index ef4a77e18fc890a80417f968f2d5cfc6211ed68c..fd66445b81e96cb3f3ea10c3c2df7d5f02743048 100644 --- a/docs/build/windows.md +++ b/docs/build/windows.md @@ -183,7 +183,7 @@ You can use Windows Settings to add these environment variables permanently: **Requirements**: * emscripten 3.1.32 -* node.js 18 +* node.js 18+ * [openjdk 11](https://learn.microsoft.com/en-us/java/openjdk/download#openjdk-11)+ ```ps1 diff --git a/ios/.gitignore b/ios/.gitignore index a810f0a3eff366333571a69d80fbebd46e31819d..31d4700aecc1f7dcd9bc0e338e814157fc3955dc 100644 --- a/ios/.gitignore +++ b/ios/.gitignore @@ -15,7 +15,10 @@ engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanios.js engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanweb-osk.ttf engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/kmwosk.css engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyman.js.map +engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanweb-webview.js +engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanweb-webview.js.map engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyman-sentry.js +engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/sentry.min.js Keyman 2* Carthage diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift index 27495bd51e0ce504d3e6340d6037b42704c2aa57..6ab2ea2a68134b7fd33f1023c7f9b3371bf2200e 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift @@ -336,7 +336,7 @@ extension KeymanWebViewController { } func deregisterLexicalModel(_ lexicalModel: InstallableLexicalModel) { - webView!.evaluateJavaScript("keyman.modelManager.deregister(\"\(lexicalModel.id)\")") + webView!.evaluateJavaScript("keyman.removeModel(\"\(lexicalModel.id)\")") } func registerLexicalModel(_ lexicalModel: InstallableLexicalModel) throws { @@ -397,7 +397,7 @@ extension KeymanWebViewController { webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct))") self.activeModel = predict } else { // We're registering a model in the background - don't change settings. - webView!.evaluateJavaScript("keyman.registerModel(\(stubString));", completionHandler: nil) + webView!.evaluateJavaScript("keyman.addModel(\(stubString));", completionHandler: nil) } setBannerHeight(to: Int(InputViewController.topBarHeight)) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift b/ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift index a6e3dc70aacfe725200e6b25843b3d7869e5d1a2..a344d445b956ef3f23830fa6b8317d3b6b814161 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift @@ -124,11 +124,11 @@ class Storage { return nil } } - + func keyboardURL(for keyboard: InstallableKeyboard) -> URL { return resourceDir(for: keyboard)!.appendingPathComponent("\(keyboard.id).js") } - + func lexicalModelURL(for lexicalModel: InstallableLexicalModel) -> URL { return resourceDir(for: lexicalModel)!.appendingPathComponent("\(lexicalModel.id).model.js") } @@ -154,7 +154,7 @@ class Storage { return nil } } - + func lexicalModelPackageURL(for lexicalModel: InstallableLexicalModel) -> URL { return resourceDir(for: lexicalModel)!.appendingPathComponent("\(lexicalModel.id).model.kmp") } @@ -178,11 +178,11 @@ class Storage { } return legacyKeyboardDir(forID: keyboardID).appendingPathComponent("\(keyboardID)-\(version).js") } - + func legacyLexicalModelURL(forID lexicalModelID: String, version: String) -> URL { return legacyLexicalModelDir(forID: lexicalModelID).appendingPathComponent("\(lexicalModelID)-\(version).model.js") } - + func legacyLexicalModelPackageURL(forID lexicalModelID: String, version: String, asZip: Bool = false) -> URL { // Our unzipping dependency requires a .zip extension to function, so we append that here. return legacyLexicalModelDir(forID: lexicalModelID).appendingPathComponent("\(lexicalModelID)-\(version).model.kmp\(asZip ? ".zip" : "")") @@ -225,7 +225,11 @@ extension Storage { dstDir: baseDir, excludeFromBackup: true) try Storage.copy(from: bundle, - resourceName: "keymanios.js", + resourceName: "keymanweb-webview.js", + dstDir: baseDir, + excludeFromBackup: true) + try Storage.copy(from: bundle, + resourceName: "sentry.min.js", dstDir: baseDir, excludeFromBackup: true) try Storage.copy(from: bundle, @@ -233,9 +237,9 @@ extension Storage { dstDir: baseDir, excludeFromBackup: true) // For debug compilations - IF we have a sourcemap file, copy that over too. - if bundle.url(forResource: "keyman.js.map", withExtension: nil) != nil { + if bundle.url(forResource: "keymanweb-webview.js.map", withExtension: nil) != nil { try Storage.copy(from: bundle, - resourceName: "keyman.js.map", + resourceName: "keymanweb-webview.js.map", dstDir: baseDir, excludeFromBackup: true) } @@ -286,7 +290,7 @@ extension Storage { // Perform an auto-install of the lexical model's KMP if not already installed. let defaultKMPFile = defaultLexicalModelDir.appendingPathComponent("\(Defaults.lexicalModel.id).model.kmp") let package = try ResourceFileManager.shared.prepareKMPInstall(from: defaultKMPFile) as! LexicalModelKeymanPackage - + // Install all languages for the model, not just the default-listed one. try ResourceFileManager.shared.install(resourcesWithIDs: package.installables[0].map { $0.fullID }, from: package) } catch { diff --git a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js index 63caa18869bde7ccf779876a783eef188bb3006c..6c31618fc48f141598e1daf7fbac68b872811038 100644 --- a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js +++ b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js @@ -8,8 +8,9 @@ if(_debug) { document.documentElement.appendChild(iframe); iframe.parentNode.removeChild(iframe); iframe = null; - if (typeof(window.webkit) != 'undefined') + if (typeof(window.webkit) != 'undefined') { window.webkit.messageHandlers.keyman.postMessage("ios-log:#iOS#" + log); + } }; console.debug = console.log; console.info = console.log; @@ -22,10 +23,12 @@ if(_debug) { var oskHeight = 0; var oskWidth = 0; +var bannerHeight = 0; -var sentryManager = new com.keyman.KeymanSentryManager({ +var sentryManager = new KeymanSentryManager({ hostPlatform: "ios" }); + sentryManager.init(); window.addEventListener('load', init, false); @@ -40,57 +43,60 @@ function init() { document.body.style.height = window.outerHeight; var kmw=window['keyman']; - // We could convert to relying on the promise, but the underlying input element - // tends to show a bit due to the delay when we do so. - kmw.init({'app':device,'fonts':'fonts/'});//.then(function() { - kmw['util']['setOption']('attachType','manual'); - kmw['oninserttext'] = insertText; - kmw['showKeyboardList'] = menuKeyDown; - kmw['hideKeyboard'] = hideKeyboard; - kmw['getOskHeight'] = getOskHeight; - kmw['getOskWidth'] = getOskWidth; - kmw['beepKeyboard'] = beepKeyboard; - kmw['setActiveElement']('ta'); - - //}); + kmw['showKeyboardList'] = menuKeyDown; + kmw['menuKeyUp'] = menuKeyUp; + kmw['hideKeyboard'] = hideKeyboard; + kmw['getOskHeight'] = getOskHeight; + kmw['getOskWidth'] = getOskWidth; + kmw['beepKeyboard'] = beepKeyboard; + kmw.init({ + 'embeddingApp':device, + 'fonts':'fonts/', + oninserttext: insertText + }).then(function() { + if(bannerHeight > 0) { + // The OSK is not available until initialization is complete. + kmw.osk.bannerView.activeBannerHeight = bannerHeight; + keyman.refreshOskLayout(); + } + }); } function verifyLoaded() { // During proper loads of KMW, a keyboard will be set very early on. // There's a chance that the keyboard is still loading, so we double-check // against the stub count. - if(!keyman.core.activeKeyboard && keyman.keyboardManager.keyboardStubs.length == 0) { + if(!keyman.core.activeKeyboard && !keyman.keyboardRequisitioner.cache.defaultStub) { location.reload(); } } function showBanner(flag) { console.log("Setting banner display for dictionaryless keyboards to " + flag); - keyman.osk.banner.setOptions({'alwaysShow': flag}); + keyman.osk.bannerController.setOptions({'alwaysShow': flag}); } function setBannerImage(path) { var kmw=window['keyman']; - kmw.osk.banner.setOptions({"imagePath": path}); + if(kmw.osk) { + kmw.osk.bannerController.setOptions({"imagePath": path}); + } } function setBannerHeight(h) { - var kmw = com.keyman.singleton; - var osk = kmw.osk; - - if(h >= 0) { + if(h >= 0) { // The banner itself may not be loaded yet. This will preemptively help set // its eventual display height. - com.keyman.osk.Banner.DEFAULT_HEIGHT = h; + bannerHeight = h; - if(osk.banner.activeType != 'blank') { - osk.banner.height = h; - } + if(keyman.osk) { + keyman.osk.bannerView.activeBannerHeight = h; } + } - // Refresh KMW's OSK - kmw.correctOSKTextSize(); - doResetContext(); + // Refresh KMW's OSK + keyman.refreshOskLayout(); + doResetContext(); } function setOskHeight(height) { @@ -99,8 +105,7 @@ function setOskHeight(height) { if(kmw && kmw.core && kmw.core.activeKeyboard) { kmw.core.activeKeyboard.refreshLayouts(); } - kmw.osk.show(true); - kmw.correctOSKTextSize(); + kmw.refreshOskLayout(); doResetContext(); } @@ -129,11 +134,11 @@ function getOskHeight() { var keyboardOffset = 0; function setKeymanLanguage(stub) { var kmw = window.keyman; - + KeymanWeb.registerStub(stub); - - kmw.setActiveKeyboard(stub.KP + '::' + stub.KI, stub.KLC).then(function() { - kmw.osk.show(true); + + kmw.setActiveKeyboard(stub.KI, stub.KLC).then(function() { + keyman.refreshOskLayout(); doResetContext(); }); } @@ -264,30 +269,32 @@ function doResetContext() { } function setCursorRange(pos, length) { + var context = keyman.context; + //console.log('setCursorRange('+pos+', '+length+')'); - var ta = document.getElementById('ta'); - var resetContext = (ta.selectionStart != pos || ta.selectionEnd != pos + length); - ta.selectionStart = ta._KeymanWebSelectionStart = pos; - ta.selectionEnd = ta._KeymanWebSelectionEnd = pos + length; - if(resetContext) { - //console.log(' setCursorRange: resetting context'); - doResetContext(); + + var start = pos; + var end = pos + length; + + if(context.selStart != start || context.selEnd != end) { + keyman.context.setSelection(start, end); + keyman.resetContext(); } - return ta.selectionEnd; } function setKeymanVal(text) { //console.log('setKeymanVal('+JSON.stringify(text)+')'); - if(undefined == text) text = ''; - var ta = document.getElementById('ta'); - var resetContext = ta.value != text; - ta.value = text; - if(resetContext) { - //console.log(' setKeymanVal: resetting context'); - doResetContext(); + if(text == undefined) { + text = ''; } - return ta.value; + + if(keyman.context.getText() != text) { + keyman.context.setText(text); + keyman.resetContext(); + } + + return keyman.context.getText(); } function executePopupKey(keyID, keyText) { @@ -316,16 +323,13 @@ function toHex(theString) { function enableSuggestions(model, mayPredict, mayCorrect) { // Set the options first so that KMW's ModelManager can properly handle model enablement states // the moment we actually register the new model. - keyman.osk.banner.setOptions({ - 'mayPredict': mayPredict, - 'mayCorrect': mayCorrect - }); + keyman.core.languageProcessor.mayPredict = mayPredict; + keyman.core.languageProcessor.mayCorrect = mayCorrect; - keyman.modelManager.register(model); + keyman.addModel(model); } function setSpacebarText(mode) { - keyman.options['spacebarText'] = mode; - keyman.osk.show(true); + keyman.config.spacebarText = mode; } diff --git a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyboard.html b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyboard.html index 7f03e7c2a151ce568c5e8d25ab7f4e1fc2c86463..266a9ea1b435b9e730b3e8785185df504c75c0d6 100644 --- a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyboard.html +++ b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyboard.html @@ -3,7 +3,8 @@ Keyman - + + - diff --git a/ios/kmbuild.sh b/ios/kmbuild.sh index f443f04c2589d9cf10be7c9d7ec91428482f6100..8f082eb1652309c05aa41bf987bb377b06e0c12a 100755 --- a/ios/kmbuild.sh +++ b/ios/kmbuild.sh @@ -154,12 +154,13 @@ update_bundle ( ) { if [ $DO_KMW_BUILD = true ]; then echo Building KeymanWeb from $KMW_ROOT - KMW_PRODUCT=web/build/app/embed/ + KMW_PRODUCT=web/build/app/webview/ + KMW_RESOURCES=web/build/app/resources if [ "$CONFIG" == "Debug" ]; then - KMWFLAGS="configure:embed build:embed --debug" + KMWFLAGS="configure:app/webview build:app/webview --debug" KMW_PRODUCT="$KMW_PRODUCT/debug" else - KMWFLAGS="configure:embed build:embed" + KMWFLAGS="configure:app/webview build:app/webview" KMW_PRODUCT="$KMW_PRODUCT/release" fi @@ -177,17 +178,24 @@ update_bundle ( ) { fi #Copy over the relevant resources! It's easiest to do if we navigate to the resulting folder. - cp "$KEYMAN_ROOT/$KMW_PRODUCT/osk/kmwosk.css" "$base_dir/$BUNDLE_PATH/kmwosk.css" - cp "$KEYMAN_ROOT/$KMW_PRODUCT/osk/keymanweb-osk.ttf" "$base_dir/$BUNDLE_PATH/keymanweb-osk.ttf" - cp "$KEYMAN_ROOT/$KMW_PRODUCT/keyman.js" "$base_dir/$BUNDLE_PATH/keymanios.js" + cp "$KEYMAN_ROOT/$KMW_RESOURCES/osk/kmwosk.css" "$base_dir/$BUNDLE_PATH/kmwosk.css" + cp "$KEYMAN_ROOT/$KMW_RESOURCES/osk/keymanweb-osk.ttf" "$base_dir/$BUNDLE_PATH/keymanweb-osk.ttf" + cp "$KEYMAN_ROOT/$KMW_PRODUCT/keymanweb-webview.js" "$base_dir/$BUNDLE_PATH/keymanweb-webview.js" if [ "$CONFIG" == "Debug" ]; then - cp "$KEYMAN_ROOT/$KMW_PRODUCT/keyman.js.map" "$base_dir/$BUNDLE_PATH/keyman.js.map" - elif [ -f "$base_dir/$BUNDLE_PATH/keyman.js.map" ]; then - rm "$base_dir/$BUNDLE_PATH/keyman.js.map" + cp "$KEYMAN_ROOT/$KMW_PRODUCT/keymanweb-webview.js.map" "$base_dir/$BUNDLE_PATH/keymanweb-webview.js.map" + elif [ -f "$base_dir/$BUNDLE_PATH/keymanweb-webview.js.map" ]; then + rm "$base_dir/$BUNDLE_PATH/keymanweb-webview.js.map" fi - cp "$KEYMAN_ROOT/common/web/sentry-manager/build/index.js" "$base_dir/$BUNDLE_PATH/keyman-sentry.js" + # Linked in by dependency for builder scripts... but this script isn't builder-based yet. + "${KEYMAN_ROOT}/common/web/sentry-manager/build.sh" + if [ $? -ne 0 ]; then + builder_die "ERROR: Build of KMW's error reporter for Sentry failed." + fi + + cp "$KEYMAN_ROOT/node_modules/@sentry/browser/build/bundle.min.js" "$base_dir/$BUNDLE_PATH/sentry.min.js" + cp "$KEYMAN_ROOT/common/web/sentry-manager/build/lib/index.js" "$base_dir/$BUNDLE_PATH/keyman-sentry.js" fi diff --git a/linux/debian/rules b/linux/debian/rules index ab4ad409e4e38c790121455b5e84b54834aa4af1..db1f9250174663f6c5a46a58d0c60b0c6f282306 100755 --- a/linux/debian/rules +++ b/linux/debian/rules @@ -9,7 +9,9 @@ export PYBUILD_INSTALL_ARGS=--install-scripts=/usr/share/keyman-config/ export KEYMAN_PKG_BUILD=1 +export HOME=$(shell mktemp -d) export XDG_DATA_HOME=$(shell mktemp -d) +export XDG_CONFIG_HOME=$(shell mktemp -d) # xenial needs this to be explicit export LC_ALL=C.UTF-8 @@ -32,26 +34,23 @@ override_dh_auto_configure: --wrap-mode=nodownload --prefix=/usr --sysconfdir=/etc --localstatedir=/var linux/keyman-system-service/build.sh configure -- \ --wrap-mode=nodownload --prefix=/usr --sysconfdir=/etc --localstatedir=/var - # keyman-config + linux/keyman-config/build.sh configure override_dh_auto_build: core/build.sh --no-tests build:arch linux/ibus-keyman/build.sh build linux/keyman-system-service/build.sh build - # keyman-config + linux/keyman-config/build.sh build cd linux/keyman-config && \ - make man && \ sed -i -e "s/^__pkgversion__ = \"[^\"]*\"/__pkgversion__ = \"$(DEB_VERSION)\"/g" keyman_config/version.py && \ make compile-po - dh_auto_build --sourcedir=linux/keyman-config --buildsystem=pybuild $@ override_dh_auto_test: ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) core/build.sh --no-tests test:arch linux/ibus-keyman/build.sh test linux/keyman-system-service/build.sh test - # keyman-config - dh_auto_test --sourcedir=linux/keyman-config --buildsystem=pybuild $@ + linux/keyman-config/build.sh test endif override_dh_auto_install: @@ -63,6 +62,7 @@ override_dh_auto_install: install -d $(CURDIR)/debian/keyman/usr/share/ cp -r linux/keyman-config/locale/ $(CURDIR)/debian/keyman/usr/share/ rm $(CURDIR)/debian/keyman/usr/share/locale/*.po* + # Don't call `build.sh install` - dh_auto_install does some extra smarts dh_auto_install --sourcedir=linux/keyman-config --buildsystem=pybuild $@ # Unfortunately bash-completion 2.10 (focal) doesn't yet provide dh-sequence-bash-completion, # which we could add as build-dependency, so we'll have to explicitly call diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index 41eaab30efb727be2c2e527059e32a1f2d7aa9e4..f057dcad0ce8c0f81e2e82af0014c99e429a41f8 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -1,49 +1,19 @@ #!/usr/bin/make -default: clean version man langtags - python3 setup.py build - -langtags: - cd buildtools && python3 ./build-langtags.py +default: + ./build.sh clean build install: - if [ -n "${SUDO_USER}" ]; then \ - make install-sudo; \ - else \ - make install-temp; \ - fi - -install-sudo: # run as sudo - pip3 install qrcode sentry-sdk - # eventually change this to: pip3 install . - python3 setup.py install - # install icons - mkdir -p /usr/local/share/keyman/icons - cp keyman_config/icons/* /usr/local/share/keyman/icons - # install man pages - mkdir -p /usr/local/share/man/man1 - cp ../../debian/man/*.1 /usr/local/share/man/man1 - -install-temp: - mkdir -p /tmp/keyman/$(shell python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)') - # when we no longer have to support old pip version (python > 3.6) change this to: - # pip3 install --prefix /tmp/keyman . - PYTHONUSERBASE=/tmp/keyman python3 setup.py install --user - -uninstall: clean # run as sudo - rm -rf /usr/local/share/keyman/icons - rm -f /usr/local/share/man/man1/km-*.1 - pip3 uninstall keyman_config - rm -f /usr/local/bin/km-config - rm -f /usr/local/bin/km-kvk2ldml - rm -f /usr/local/bin/km-package-get - rm -f /usr/local/bin/km-package-install - rm -f /usr/local/bin/km-package-list-installed - rm -f /usr/local/bin/km-package-uninstall + ./build.sh install + +uninstall: # run as sudo + ./build.sh uninstall clean: - -rm -rf dist make_deb build keyman_config/version.py *.egg-info __pycache__ \ - keyman_config/standards/lang_tags_map.py + ./build.sh clean + +check: + ./build.sh test devdist: version python3 setup.py egg_info -b.`TZ=UTC git log -1 --pretty=format:%cd --date=format-local:%Y%m%d%H%M` sdist @@ -93,6 +63,3 @@ $(MOFILES): %/LC_MESSAGES/keyman-config.mo: %.po msgfmt -v --check --output-file=$@ $^ compile-po: $(MOFILES) - -check: - ./run-tests.sh diff --git a/linux/keyman-config/build.sh b/linux/keyman-config/build.sh index 38cf22e917b40ab8a37c60b15a5748681bc5c66e..1cb603f091b8801ffc7fd807d02c07b461cac688 100755 --- a/linux/keyman-config/build.sh +++ b/linux/keyman-config/build.sh @@ -24,9 +24,90 @@ cd "$THIS_SCRIPT_PATH" builder_describe_outputs \ build "/linux/keyman-config/keyman_config/standards/lang_tags_map.py" -builder_run_action clean make clean +clean_action() { + rm -rf dist make_deb build keyman_config/version.py ./*.egg-info __pycache__ \ + keyman_config/standards/lang_tags_map.py +} + +build_man_pages() { + local TEMP_DATA_DIR SCHEMA_DIR + TEMP_DATA_DIR=$(mktemp -d) + SCHEMA_DIR=$TEMP_DATA_DIR/glib-2.0/schemas + export XDG_DATA_DIRS=$TEMP_DATA_DIR:${XDG_DATA_DIRS-} + export GSETTINGS_SCHEMA_DIR="${SCHEMA_DIR}" + mkdir -p "$SCHEMA_DIR" + cp ./com.keyman.gschema.xml "$SCHEMA_DIR"/ + glib-compile-schemas "$SCHEMA_DIR" + ./build-help.sh --man --no-reconf + export XDG_DATA_DIRS=${XDG_DATA_DIRS#*:} + unset GSETTINGS_SCHEMA_DIR + rm -rf "$TEMP_DATA_DIR" +} + +build_action() { + builder_echo "Create version.py" + pushd keyman_config + sed \ + -e "s/_VERSION_/${VERSION}/g" \ + -e "s/_VERSIONWITHTAG_/${VERSION_WITH_TAG}/g" \ + -e "s/_VERSIONGITTAG_/${VERSION_GIT_TAG}/g" \ + -e "s/_MAJORVERSION_/${VERSION_MAJOR}/g" \ + -e "s/_RELEASEVERSION_/${VERSION_RELEASE}/g" \ + -e "s/_TIER_/${TIER}/g" \ + -e "s/_ENVIRONMENT_/${VERSION_ENVIRONMENT}/g" \ + -e "s/_UPLOADSENTRY_/${UPLOAD_SENTRY}/g" \ + version.py.in > version.py + popd + pushd buildtools + if [ -f build-langtags.py ]; then + builder_echo "Create lang_tags_map.py" + python3 ./build-langtags.py + else + builder_echo "Skip building lang_tags_map.py during package build" + fi + popd + builder_echo "Building man pages" + build_man_pages + builder_echo "Building keyman-config" + python3 setup.py build +} + +install_action() { + if [ -n "${SUDO_USER:-}" ]; then + pip3 install qrcode sentry-sdk + # eventually change this to: pip3 install . + python3 setup.py install + # install icons + mkdir -p /usr/local/share/keyman/icons + cp keyman_config/icons/* /usr/local/share/keyman/icons + # install man pages + mkdir -p /usr/local/share/man/man1 + cp ../../debian/man/*.1 /usr/local/share/man/man1 + else + mkdir -p "/tmp/keyman/$(python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)')" + # when we no longer have to support old pip version (python > 3.6) change this to: + # pip3 install --prefix /tmp/keyman . + PYTHONUSERBASE=${DESTDIR:-/tmp/keyman} python3 setup.py install --user + fi +} + +uninstall_action() { + # run as sudo + clean_action + rm -rf /usr/local/share/keyman/icons + rm -f /usr/local/share/man/man1/km-*.1 + pip3 uninstall keyman_config + rm -f /usr/local/bin/km-config + rm -f /usr/local/bin/km-kvk2ldml + rm -f /usr/local/bin/km-package-get + rm -f /usr/local/bin/km-package-install + rm -f /usr/local/bin/km-package-list-installed + rm -f /usr/local/bin/km-package-uninstall +} + +builder_run_action clean clean_action builder_run_action configure # nothing to do -builder_run_action build make +builder_run_action build build_action builder_run_action test ./run-tests.sh -builder_run_action install make install -builder_run_action uninstall make uninstall +builder_run_action install install_action +builder_run_action uninstall uninstall_action diff --git a/linux/keyman-config/com.keyman.gschema.xml b/linux/keyman-config/com.keyman.gschema.xml index 8f05a42bf9b0c70d4bc2ee9ffdb6b3ec6716c9c3..df50060d8c44eb97e2ec4e8d4848ab23c34ef665 100644 --- a/linux/keyman-config/com.keyman.gschema.xml +++ b/linux/keyman-config/com.keyman.gschema.xml @@ -1,8 +1,12 @@ - + + true + Automatic error reporting + true to enable automatic reporting of errors to Sentry + diff --git a/linux/keyman-config/keyman_config/__init__.py b/linux/keyman-config/keyman_config/__init__.py index e5cecf4cc82d6b6ae333dab67dd5aa615454b3e2..04098ade00fcbeeed91a9f7b5ee63070780d60af 100644 --- a/linux/keyman-config/keyman_config/__init__.py +++ b/linux/keyman-config/keyman_config/__init__.py @@ -1,20 +1,18 @@ -import getpass import gettext -import importlib -import logging -import os -import platform -import sys -from .version import __version__ -from .version import __versionwithtag__ -from .version import __versiongittag__ -from .version import __majorversion__ -from .version import __releaseversion__ -from .version import __tier__ -from .version import __pkgversion__ -from .version import __environment__ -from .version import __uploadsentry__ +from keyman_config.sentry_handling import SentryErrorHandling + +from keyman_config.version import ( + __version__, + __versionwithtag__, + __versiongittag__, + __majorversion__, + __releaseversion__, + __tier__, + __pkgversion__, + __environment__, + __uploadsentry__ +) def _(txt): @@ -24,7 +22,7 @@ def _(txt): return translation -def secure_lookup(data, key1, key2 = None): +def secure_lookup(data, key1, key2=None): """ Return data[key1][key2] while dealing with data being None or key1 or key2 not existing """ @@ -38,18 +36,10 @@ def secure_lookup(data, key1, key2 = None): return None -def before_send(event, hint): - if 'exc_info' in hint: - exc_type, exc_value, tb = hint['exc_info'] - if isinstance(exc_value, KeyboardInterrupt): - # Ignore KeyboardInterrupt exception - return None - return event - - gettext.bindtextdomain('keyman-config', '/usr/share/locale') gettext.textdomain('keyman-config') + #if __tier__ == 'alpha' or __tier__ == 'beta': // #7227 disabling: # Alpha and beta versions will work against the staging server so that they # can access new APIs etc that will only be available there. The staging @@ -60,60 +50,8 @@ gettext.textdomain('keyman-config') KeymanComUrl = 'https://keyman.com' KeymanApiUrl = 'https://api.keyman.com' + # There's no staging site for downloads KeymanDownloadsUrl = 'https://downloads.keyman.com' -if 'unittest' in sys.modules.keys(): - print('Not reporting to Sentry', file=sys.stderr) -elif os.environ.get('KEYMAN_NOSENTRY'): - print('Not reporting to Sentry because KEYMAN_NOSENTRY environment variable set', file=sys.stderr) -elif not __uploadsentry__: - print('Not reporting to Sentry because UPLOAD_SENTRY is false (%s)' % __environment__, file=sys.stderr) -else: - try: - # Try new sentry-sdk first - sentry_sdk = importlib.import_module('sentry_sdk') - from sentry_sdk import configure_scope, set_user - from sentry_sdk.integrations.logging import LoggingIntegration - HaveSentryNewSdk = True - - sentry_logging = LoggingIntegration( - level=logging.INFO, # Capture info and above as breadcrumbs - event_level=logging.CRITICAL # Send critical errors as events - ) - SentryUrl = "https://1d0edbf2d0dc411b87119b6e92e2c357@o1005580.ingest.sentry.io/5983525" - sentry_sdk.init( - dsn=SentryUrl, - environment=__environment__, - release=__versiongittag__, - integrations=[sentry_logging], - before_send=before_send - ) - set_user({'id': hash(getpass.getuser())}) - with configure_scope() as scope: - scope.set_tag("app", os.path.basename(sys.argv[0])) - scope.set_tag("pkgversion", __pkgversion__) - scope.set_tag("platform", platform.platform()) - scope.set_tag("system", platform.system()) - scope.set_tag("tier", __tier__) - except ImportError: - try: - # sentry-sdk is not available, so use older raven - raven = importlib.import_module('raven') - from raven import Client - HaveSentryNewSdk = False - - # Note, legacy raven API requires secret (https://github.com/keymanapp/keyman/pull/5787#discussion_r721457909) - SentryUrl = "https://1d0edbf2d0dc411b87119b6e92e2c357:e6d5a81ee6944fc79bd9f0cbb1f2c2a4@o1005580.ingest.sentry.io/5983525" - client = Client(SentryUrl, environment=__environment__, release=__versiongittag__) - client.user_context({'id': hash(getpass.getuser())}) - client.tags_context({ - 'app': os.path.basename(sys.argv[0]), - 'pkgversion': __pkgversion__, - 'platform': platform.platform(), - 'system': platform.system(), - 'tier': __tier__, - }) - except ImportError: - # even raven is not available. This is the case on Ubuntu 16.04. Just ignore. - print(_('Neither sentry-sdk nor raven is available. Not enabling Sentry error reporting.'), file=sys.stderr) +SentryErrorHandling().initialize_sentry() diff --git a/linux/keyman-config/keyman_config/dconf_util.py b/linux/keyman-config/keyman_config/dconf_util.py index 09b95d1815a5f42021674cba68b40a6c2f7887fa..c28ec3fd19c5aa7b6a126c020caadb3412890f3b 100644 --- a/linux/keyman-config/keyman_config/dconf_util.py +++ b/linux/keyman-config/keyman_config/dconf_util.py @@ -2,25 +2,25 @@ from gi.repository import Gio -# DConf path destkop/ibus/keyman/options -DCONF_BASE = "com.keyman.options" +# GSettings path destkop/ibus/keyman/options +GSETTINGS_BASE = "com.keyman.options" -# Utilities to get and set Keyman options in DConf: +# Utilities to get and set Keyman options in GSettings: # /desktop/ibus/keyman/options/packageID/keyboardID/options def get_child_schema(info): - settings = Gio.Settings.new(DCONF_BASE) + settings = Gio.Settings.new(GSETTINGS_BASE) path = settings.get_property('path') if not path.endswith('/'): path += '/' path += info['packageID'] + '/' + info['keyboardID'] + '/' - return Gio.Settings(DCONF_BASE + '.child', path) + return Gio.Settings(GSETTINGS_BASE + '.child', path) def get_option(info): """ - Get the Keyman keyboard options from DConf + Get the Keyman keyboard options from GSettings Convert from list of comma-separated strings into dictionary Args: @@ -41,7 +41,7 @@ def get_option(info): def set_option(info, options): """ - Store the Keyman keyboard options in DConf as a list of strings + Store the Keyman keyboard options in GSettings as a list of strings Args: info: dictionary diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index af7b34063d3336fd2c451c4ddddbdcf4073a1570..ed3f7ec8a444fdc178238f495f9ef29ca05f41f9 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -9,6 +9,7 @@ import requests import requests_cache from gi.repository import GObject +from keyman_config import _ from keyman_config import KeymanApiUrl, KeymanDownloadsUrl from keyman_config.deprecated_decorator import deprecated @@ -20,6 +21,16 @@ class InstallLocation(GObject.GEnum): Unknown = 99 +def get_install_area_string(area): + if area == InstallLocation.OS: + return _('System') + elif area == InstallLocation.Shared: + return _('Shared') + elif area == InstallLocation.User: + return _('User') + return _('Unknown') + + def get_package_download_data(packageID, weekCache=False): """ Get package download data from keyboards download api. diff --git a/linux/keyman-config/keyman_config/sentry_handling.py b/linux/keyman-config/keyman_config/sentry_handling.py new file mode 100644 index 0000000000000000000000000000000000000000..f2b2f8adc3441b83ced5dc75879f06cbea06310f --- /dev/null +++ b/linux/keyman-config/keyman_config/sentry_handling.py @@ -0,0 +1,152 @@ +#!/usr/bin/python3 +import getpass +import importlib +import logging +import os +import platform +import sys +from keyman_config.gsettings import GSettings +from keyman_config.version import ( + __version__, + __versionwithtag__, + __versiongittag__, + __majorversion__, + __releaseversion__, + __tier__, + __pkgversion__, + __environment__, + __uploadsentry__ +) + +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gio, Gtk + + +class SentryErrorHandling: + def __init__(self) -> None: + self.settings = Gio.Settings.new('com.keyman.options') + + def initialize_sentry(self): + (enabled, reason) = self.is_sentry_enabled() + if not enabled: + print(reason, file=sys.stderr) + logging.info(reason) + return (enabled, reason) + else: + try: + self._sentry_sdk_initialize() + except ImportError: + try: + self._raven_initialize() + except ImportError: + # even raven is not available. This is the case on Ubuntu 16.04. Just ignore. + print(_('Neither sentry-sdk nor raven is available. Not enabling Sentry error reporting.'), + file=sys.stderr) + logging.info('Neither sentry-sdk nor raven is available. Not enabling Sentry error reporting.') + return (False, _('Neither sentry-sdk nor raven is available. Not enabling Sentry error reporting.')) + return (True, '') + + def is_sentry_enabled(self): + if 'unittest' in sys.modules.keys(): + return (False, 'Running unit tests, not reporting to Sentry') + elif self._get_environ_nosentry(): + return (False, 'Not reporting to Sentry because KEYMAN_NOSENTRY environment variable set') + elif not __uploadsentry__: + return (False, f'Not reporting to Sentry because UPLOAD_SENTRY is false ({__environment__})') + elif not self._get_setting(): + return (False, 'Not reporting to Sentry because disabled in GSettings') + return (True, 'Reporting to Sentry') + + def is_sentry_disabled_by_variable(self): + return self._get_environ_nosentry() or not __uploadsentry__ + + def bind_checkbutton(self, button: Gtk.CheckButton): + self.settings.bind("error-reporting", button, "active", Gio.SettingsBindFlags.NO_SENSITIVITY) + self.settings.connect("changed::error-reporting", self._on_sentry_reporting_toggled) + + def set_enabled(self, enabled): + assert not self.is_sentry_disabled_by_variable() + was_enabled = self.is_sentry_enabled() + self._save_setting(enabled) + if enabled != was_enabled: + self._handle_enabled(enabled) + + def _get_environ_nosentry(self): + keyman_nosentry = os.environ.get('KEYMAN_NOSENTRY') + return keyman_nosentry and (int(keyman_nosentry) == 1) + + def _handle_enabled(self, enabled): + if enabled: + self.initialize_sentry() + else: + self._close_sentry() + + def _save_setting(self, enabled: bool): + self.settings.set_boolean('error-reporting', enabled) + + def _get_setting(self) -> bool: + return self.settings.get_boolean('error-reporting') + + def _on_sentry_reporting_toggled(self, settings, key): + self._handle_enabled(self.settings.get_boolean('error-reporting')) + + def _close_sentry(self): + from sentry_sdk import Hub + logging.info("Shutting down Sentry error reporting") + client = Hub.current.client + if client is not None: + client.close(timeout=2.0) + + def _sentry_sdk_initialize(self): + # Try new sentry-sdk first + sentry_sdk = importlib.import_module('sentry_sdk') + from sentry_sdk import configure_scope, set_user + from sentry_sdk.integrations.logging import LoggingIntegration + + sentry_logging = LoggingIntegration( + level=logging.INFO, # Capture info and above as breadcrumbs + event_level=logging.CRITICAL # Send critical errors as events + ) + SentryUrl = "https://1d0edbf2d0dc411b87119b6e92e2c357@o1005580.ingest.sentry.io/5983525" + sentry_sdk.init( + dsn=SentryUrl, + environment=__environment__, + release=__versiongittag__, + integrations=[sentry_logging], + before_send=self._before_send + ) + set_user({'id': hash(getpass.getuser())}) + with configure_scope() as scope: + scope.set_tag("app", os.path.basename(sys.argv[0])) + scope.set_tag("pkgversion", __pkgversion__) + scope.set_tag("platform", platform.platform()) + scope.set_tag("system", platform.system()) + scope.set_tag("tier", __tier__) + logging.info("Initialized Sentry error reporting") + + def _raven_initialize(self): + # sentry-sdk is not available, so use older raven + raven = importlib.import_module('raven') + from raven import Client + + # Note, legacy raven API requires secret (https://github.com/keymanapp/keyman/pull/5787#discussion_r721457909) + SentryUrl = "https://1d0edbf2d0dc411b87119b6e92e2c357:e6d5a81ee6944fc79bd9f0cbb1f2c2a4@o1005580.ingest.sentry.io/5983525" + client = Client(SentryUrl, environment=__environment__, release=__versiongittag__) + client.user_context({'id': hash(getpass.getuser())}) + client.tags_context({ + 'app': os.path.basename(sys.argv[0]), + 'pkgversion': __pkgversion__, + 'platform': platform.platform(), + 'system': platform.system(), + 'tier': __tier__, + }) + logging.info("Initialized Sentry error reporting (raven)") + + def _before_send(self, event, hint): + if 'exc_info' in hint: + exc_type, exc_value, tb = hint['exc_info'] + if isinstance(exc_value, KeyboardInterrupt): + # Ignore KeyboardInterrupt exception + return None + return event diff --git a/linux/keyman-config/keyman_config/view_installed.py b/linux/keyman-config/keyman_config/view_installed.py index 01b1a59b89b3e413619c87abd56c479667abeab2..263d665d25f8ec89ef1694185015fad5e7f2d9b4 100755 --- a/linux/keyman-config/keyman_config/view_installed.py +++ b/linux/keyman-config/keyman_config/view_installed.py @@ -19,12 +19,13 @@ from keyman_config.accelerators import bind_accelerator, init_accel from keyman_config.dbus_util import get_keyman_config_service from keyman_config.downloadkeyboard import DownloadKmpWindow from keyman_config.get_kmp import (InstallLocation, get_keyboard_dir, - get_keyman_dir) + get_keyman_dir, get_install_area_string) from keyman_config.install_window import InstallKmpWindow, find_keyman_image from keyman_config.keyboard_details import KeyboardDetailsView from keyman_config.kmpmetadata import get_fonts, parsemetadata from keyman_config.list_installed_kmp import get_installed_kmp from keyman_config.options import OptionsView +from keyman_config.sentry_handling import SentryErrorHandling from keyman_config.uninstall_kmp import uninstall_kmp from keyman_config.welcome import WelcomeView @@ -94,7 +95,7 @@ class ViewInstalledWindowBase(Gtk.Window): self.close() def run(self): - self.resize(576, 324) + self.resize(776, 424) self.connect("destroy", Gtk.main_quit) self.show_all() Gtk.main() @@ -104,26 +105,60 @@ class ViewInstalledWindow(ViewInstalledWindowBase): def __init__(self): ViewInstalledWindowBase.__init__(self) -# window is split left/right hbox -# right is ButtonBox -# possibly 2 ButtonBox in a vbox -# top one with _Remove, _About, ?_Welcome? or ?Read_Me?, _Options -# bottom one with _Download, _Install, Re_fresh, _Close -# left is GtkTreeView - does it need to be inside anything else apart from the hbox? -# with liststore which defines columns -# GdkPixbuf icon -# gchararray name -# gchararray version -# gchararray packageID (hidden) -# enum? area (user, shared, system) (icon or hidden?) -# gchararray welcomefile (hidden) (or just use area and packageID?) -# changing selected item in treeview changes what buttons are activated -# on selected_item_changed signal set the data that the buttons will use in their callbacks -# see https://developer.gnome.org/gtk3/stable/TreeWidget.html#TreeWidget + self.sentry = SentryErrorHandling() + outerHbox = Gtk.HBox() + sidebar = Gtk.StackSidebar() + stack = Gtk.Stack() + + outerHbox.pack_start(sidebar, False, False, 0) + outerHbox.pack_end(Gtk.Separator(), False, False, 0) + outerHbox.pack_end(stack, True, True, 0) + stack.set_hexpand(True) + stack.set_vexpand(True) + sidebar.set_stack(stack) + + stack.add_titled(self.add_keyboard_layouts_widget(), "KeyboardLayouts", _("Keyboard Layouts")) + stack.add_titled(self.add_options_widget(), "Options", _("Options")) + + outmostVBox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL) + outmostVBox.pack_start(outerHbox, True, True, 0) + + bbox_bottom = Gtk.ButtonBox(spacing=12, orientation=Gtk.Orientation.HORIZONTAL) + bbox_bottom.set_layout(Gtk.ButtonBoxStyle.START) + + button = Gtk.Button.new_with_mnemonic(_("_Install keyboard...")) + button.set_tooltip_text(_("Install a keyboard from a file")) + button.connect("clicked", self.on_installfile_clicked) + bbox_bottom.add(button) + + button = Gtk.Button.new_with_mnemonic(_("_Download keyboard...")) + button.set_tooltip_text(_("Download and install a keyboard from the Keyman website")) + button.connect("clicked", self.on_download_clicked) + bbox_bottom.add(button) + + button = Gtk.Button.new_with_mnemonic(_("_Refresh")) + button.set_tooltip_text(_("Refresh keyboard list")) + button.connect("clicked", self.on_refresh_clicked) + bbox_bottom.add(button) + + button = Gtk.Button.new_with_mnemonic(_("_Close")) + button.set_tooltip_text(_("Close window")) + button.connect("clicked", self.on_close_clicked) + bind_accelerator(self.accelerators, button, 'q') + bind_accelerator(self.accelerators, button, 'w') + bbox_bottom.add(button) + + bbox_hbox = Gtk.HBox(spacing=12) + bbox_hbox.pack_start(bbox_bottom, True, True, 12) + outmostVBox.pack_end(bbox_hbox, False, True, 12) + self.add(outmostVBox) + + def add_keyboard_layouts_widget(self): hbox = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL) - s = Gtk.ScrolledWindow() - hbox.pack_start(s, True, True, 0) + + scrolledWindow = Gtk.ScrolledWindow() + hbox.pack_start(scrolledWindow, True, True, 0) self.store = Gtk.ListStore( GdkPixbuf.Pixbuf, # icon @@ -131,15 +166,11 @@ class ViewInstalledWindow(ViewInstalledWindowBase): str, # version str, # packageID int, # enum InstallLocation (KmpArea is GObject version) + str, # InstallLocation area + str, # Tooltip with InstallLocation path str, # path to welcome file if it exists or None str) # path to options file if it exists or None - # add installed keyboards to the the store e.g. - # treeiter = store.append([GdkPixbuf.Pixbuf.new_from_file_at_size( - # "/usr/local/share/keyman/libtralo/libtralo.ico.png", 16, 16), \ - # "LIBTRALO", "1.6.1", \ - # "libtralo", KmpArea.SHARED, True]) - self.refresh_installed_kmp() self.tree = Gtk.TreeView(self.store) @@ -155,11 +186,16 @@ class ViewInstalledWindow(ViewInstalledWindowBase): # i18n: column header in table displaying installed keyboards column = Gtk.TreeViewColumn(_("Version"), renderer, text=2) self.tree.append_column(column) + # i18n: column header in table displaying installed keyboards + column = Gtk.TreeViewColumn(_("Area"), renderer, text=5) + self.tree.append_column(column) + + self.tree.set_tooltip_column(column=6) select = self.tree.get_selection() select.connect("changed", self.on_tree_selection_changed) - s.add(self.tree) + scrolledWindow.add(self.tree) vbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12) @@ -191,36 +227,34 @@ class ViewInstalledWindow(ViewInstalledWindowBase): bbox_top.add(self.options_button) vbox.pack_start(bbox_top, False, False, 12) + hbox.pack_start(vbox, False, False, 12) - bbox_bottom = Gtk.ButtonBox(spacing=12, orientation=Gtk.Orientation.VERTICAL) - bbox_bottom.set_layout(Gtk.ButtonBoxStyle.END) + return hbox - button = Gtk.Button.new_with_mnemonic(_("_Refresh")) - button.set_tooltip_text(_("Refresh keyboard list")) - button.connect("clicked", self.on_refresh_clicked) - bbox_bottom.add(button) + def add_options_widget(self): + vbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL) + label = Gtk.Label(_("General")) + label.set_padding(5, 5) + label.set_halign(Gtk.Align.START) + vbox.pack_start(label, False, False, 10) - button = Gtk.Button.new_with_mnemonic(_("_Download")) - button.set_tooltip_text(_("Download and install a keyboard from the Keyman website")) - button.connect("clicked", self.on_download_clicked) - bbox_bottom.add(button) - - button = Gtk.Button.new_with_mnemonic(_("_Install")) - button.set_tooltip_text(_("Install a keyboard from a file")) - button.connect("clicked", self.on_installfile_clicked) - bbox_bottom.add(button) + (enabled, reason) = self.sentry.is_sentry_enabled() + disabledByVariable = self.sentry.is_sentry_disabled_by_variable() - button = Gtk.Button.new_with_mnemonic(_("_Close")) - button.set_tooltip_text(_("Close window")) - button.connect("clicked", self.on_close_clicked) - bind_accelerator(self.accelerators, button, 'q') - bind_accelerator(self.accelerators, button, 'w') - bbox_bottom.add(button) + self.errorReportingButton = Gtk.CheckButton(_("Automatically report errors to keyman.com")) + self.errorReportingButton.set_active(enabled) + self.errorReportingButton.set_sensitive(not disabledByVariable) + self.sentry.bind_checkbutton(self.errorReportingButton) + vbox.pack_start(self.errorReportingButton, False, False, 0) - vbox.pack_end(bbox_bottom, False, False, 12) + if disabledByVariable: + label = Gtk.Label(reason) + label.set_halign(Gtk.Align.START) + label.set_padding(25, 0) + label.set_sensitive(False) + vbox.pack_start(label, False, False, 0) - hbox.pack_start(vbox, False, False, 12) - self.add(hbox) + return vbox def addlistitems(self, installed_kmp, store, install_area): for kmp in sorted(installed_kmp): @@ -255,6 +289,8 @@ class ViewInstalledWindow(ViewInstalledWindowBase): kmpdata['version'], kmpdata['packageID'], install_area, + get_install_area_string(install_area), + path.replace(os.path.expanduser('~'), '~'), welcome_file, options_file]) diff --git a/linux/keyman-config/run-tests.sh b/linux/keyman-config/run-tests.sh index 3b8664512adc639aa376c6e6ed4b5da8e479713c..b390ce613bc589d926da09a8ca39c28ecd7c3a19 100755 --- a/linux/keyman-config/run-tests.sh +++ b/linux/keyman-config/run-tests.sh @@ -1,6 +1,9 @@ #!/bin/bash PYTHONPATH=.:$PYTHONPATH +XDG_CONFIG_HOME=$(mktemp --directory) +export XDG_CONFIG_HOME + if [ -n "$TEAMCITY_VERSION" ]; then if ! pip3 list --format=columns | grep -q teamcity-messages; then pip3 install teamcity-messages @@ -9,3 +12,5 @@ if [ -n "$TEAMCITY_VERSION" ]; then else python3 -m unittest discover -v -s tests -p test_*.py fi + +rm -rf "$XDG_CONFIG_HOME" diff --git a/linux/keyman-config/tests/test_gnome_keyboards_util.py b/linux/keyman-config/tests/test_gnome_keyboards_util.py index b5a0cac842fff324e018f25672c1b93817ef376b..a53908cdd3296aaee7dfa26b838ac0fe29042f67 100644 --- a/linux/keyman-config/tests/test_gnome_keyboards_util.py +++ b/linux/keyman-config/tests/test_gnome_keyboards_util.py @@ -9,14 +9,14 @@ class GnomeKeyboardsUtilTests(unittest.TestCase): def setUp(self): _reset_gnome_shell() - @patch('keyman_config.os.system') + @patch('os.system') def test_IsGnomeShell_RunningGnomeShell(self, mockSystem): # Setup mockSystem.return_value = 0 # Execute/Verify self.assertEqual(is_gnome_shell(), True) - @patch('keyman_config.os.system') + @patch('os.system') def test_IsGnomeShell_NotRunningGnomeShell(self, mockSystem): # Setup mockSystem.return_value = 1 diff --git a/linux/keyman-config/tests/test_install_kmp.py b/linux/keyman-config/tests/test_install_kmp.py index 2f250550cd3ecef1ea5f1f1a88debe4a292c6178..d1e10bd645b6d9aaa450bc9fed3cc64883b92b31 100644 --- a/linux/keyman-config/tests/test_install_kmp.py +++ b/linux/keyman-config/tests/test_install_kmp.py @@ -262,13 +262,11 @@ class InstallKmpTests(unittest.TestCase): def test_InstallKmp_FutureKeymanVersion(self): # Setup - workdir = tempfile.TemporaryDirectory().name - os.makedirs(workdir) - packagedir = tempfile.TemporaryDirectory().name - os.makedirs(packagedir) - self.mockGetKeyboardDir.return_value = packagedir - kmpfile = self._createEmptyKmp(workdir) - self._createKmpJson(packagedir, ', "fileVersion": "99.0"') + workdir = tempfile.TemporaryDirectory() + packagedir = tempfile.TemporaryDirectory() + self.mockGetKeyboardDir.return_value = packagedir.name + kmpfile = self._createEmptyKmp(workdir.name) + self._createKmpJson(packagedir.name, ', "fileVersion": "99.0"') # Execute with self.assertRaises(InstallError) as context: @@ -278,22 +276,24 @@ class InstallKmpTests(unittest.TestCase): self.assertTrue('foo.kmp requires Keyman 99.0 or higher' in context.exception.message) self.mockInstallToIbus.assert_not_called() + # Teardown + workdir.cleanup() + packagedir.cleanup() + def test_InstallKmp(self): for testcase in [ {'name': 'PreviousKeymanVersion', 'fileVersion': ', "fileVersion": "7.0"'}, - {'name': 'SameKeymanVersion', 'fileVersion': ', "fileVersion": "' + __version__ + '"'}, + {'name': 'SameKeymanVersion', 'fileVersion': f', "fileVersion": "{__version__}"'}, {'name': 'NoFileVersion', 'fileVersion': ''} ]: with self.subTest(msg=testcase['name'], data=testcase['fileVersion']): # Setup self.mockInstallToIbus.reset_mock() - workdir = tempfile.TemporaryDirectory().name - os.makedirs(workdir) - packagedir = tempfile.TemporaryDirectory().name - os.makedirs(packagedir) - self.mockGetKeyboardDir.return_value = packagedir - kmpfile = self._createEmptyKmp(workdir) - self._createKmpJson(packagedir, testcase['fileVersion']) + workdir = tempfile.TemporaryDirectory() + packagedir = tempfile.TemporaryDirectory() + self.mockGetKeyboardDir.return_value = packagedir.name + kmpfile = self._createEmptyKmp(workdir.name) + self._createKmpJson(packagedir.name, testcase['fileVersion']) # Execute InstallKmp()._install_kmp(kmpfile, 'km', InstallLocation.User) @@ -301,6 +301,10 @@ class InstallKmpTests(unittest.TestCase): # Verify self.mockInstallToIbus.assert_called_once() + # Teardown + workdir.cleanup() + packagedir.cleanup() + if __name__ == '__main__': unittest.main() diff --git a/linux/keyman-config/tests/test_package_install_completion.py b/linux/keyman-config/tests/test_package_install_completion.py index 97238e3dc2e57051c63d085d1692018267841f71..61374a6c66cbc5756e22b0c42e42810adf378b03 100644 --- a/linux/keyman-config/tests/test_package_install_completion.py +++ b/linux/keyman-config/tests/test_package_install_completion.py @@ -2,7 +2,7 @@ import os import tempfile import unittest -from unittest.mock import patch, ANY +from unittest.mock import patch from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader @@ -10,17 +10,11 @@ from importlib.util import module_from_spec, spec_from_loader class PackageInstallCompletionTests(unittest.TestCase): def setUp(self): - patcher1 = patch('keyman_config.install_kmp.extract_kmp') - self.mockExtractKmp = patcher1.start() - self.addCleanup(patcher1.stop) - patcher2 = patch('keyman_config.kmpmetadata.get_metadata') - self.mockGetMetadata = patcher2.start() - self.addCleanup(patcher2.stop) - + self.mockExtractKmp = self._setupMock('keyman_config.install_kmp.extract_kmp') + self.mockGetMetadata = self._setupMock('keyman_config.kmpmetadata.get_metadata') loader = SourceFileLoader('km_package_install', os.path.join(os.path.dirname( os.path.abspath(__file__)), '../km-package-install')) - spec = spec_from_loader(loader.name, loader) - if spec: + if spec := spec_from_loader(loader.name, loader): self.mod = module_from_spec(spec) loader.exec_module(self.mod) @@ -29,32 +23,43 @@ class PackageInstallCompletionTests(unittest.TestCase): self.cacheDir = os.path.join(self.tempDir.name, 'keyman') os.makedirs(self.cacheDir) + def tearDown(self): + self.tempDir.cleanup() + + def _setupMock(self, arg0): + patcher = patch(arg0) + result = patcher.start() + self.addCleanup(patcher.stop) + return result + def _list_languages_for_keyboard_impl(self, packageId): return self.mod._list_languages_for_keyboard_impl(packageId, 'someDir') def test_PackageCompletionNoLanguage(self): - open(os.path.join(self.cacheDir, 'foo'), 'w') - self.mockGetMetadata.return_value = (None, None, None, [{}], None) - result = self._list_languages_for_keyboard_impl('foo') - self.assertEqual(result, "") + with open(os.path.join(self.cacheDir, 'foo'), 'w'): + self.mockGetMetadata.return_value = (None, None, None, [{}], None) + result = self._list_languages_for_keyboard_impl('foo') + self.assertEqual(result, "") def test_PackageCompletionOneLanguage(self): - open(os.path.join(self.cacheDir, 'khmer_angkor'), 'w') - self.mockGetMetadata.return_value = ( - None, None, None, - [{'languages': [{'name': 'Central Khmer (Khmer, Cambodia)', 'id': 'km'}]}], - None) - result = self._list_languages_for_keyboard_impl('khmer_angkor') - self.assertEqual(result, "km") + with open(os.path.join(self.cacheDir, 'khmer_angkor'), 'w'): + self.mockGetMetadata.return_value = ( + None, None, None, + [{'languages': [{'name': 'Central Khmer (Khmer, Cambodia)', 'id': 'km'}]}], + None + ) + result = self._list_languages_for_keyboard_impl('khmer_angkor') + self.assertEqual(result, "km") def test_PackageCompletionMultipleLanguages(self): - open(os.path.join(self.cacheDir, 'sil_euro_latin'), 'w') - self.mockGetMetadata.return_value = ( - None, None, None, - [{'languages': [ - {'name': 'English', 'id': 'en'}, - {'name': 'French', 'id': 'fr'}, - {'name': 'German', 'id': 'de'}]}], - None) - result = self._list_languages_for_keyboard_impl('sil_euro_latin') - self.assertEqual(result, 'en\nfr\nde') + with open(os.path.join(self.cacheDir, 'sil_euro_latin'), 'w'): + self.mockGetMetadata.return_value = ( + None, None, None, + [{'languages': [ + {'name': 'English', 'id': 'en'}, + {'name': 'French', 'id': 'fr'}, + {'name': 'German', 'id': 'de'}]}], + None + ) + result = self._list_languages_for_keyboard_impl('sil_euro_latin') + self.assertEqual(result, 'en\nfr\nde') diff --git a/linux/scripts/install.sh b/linux/scripts/install.sh index 40f2eb8f5b09bcd7f6571fa272d660391349ebaa..55074f48481915a713d630926f56b3161d76b955 100755 --- a/linux/scripts/install.sh +++ b/linux/scripts/install.sh @@ -15,7 +15,7 @@ INSTALLDIR=${INSTALLDIR:-"/usr/local"} if [[ "${SUDOINSTALL}" != "no" ]]; then if [ "$EUID" -ne 0 ] then - echo "Please run 'make un/install' with sudo" + echo "Please run 'build.sh un/install' with sudo" exit fi fi @@ -24,7 +24,7 @@ if [ -f "/usr/share/ibus/component/keyman.xml" ] && [ "${SUDOINSTALL}" == "yes" if grep -Fq "/usr/lib/ibus" /usr/share/ibus/component/keyman.xml then echo "component file is in ibus-keyman package version so move it" - echo "run 'sudo make uninstall' to put it back" + echo "run 'sudo build.sh uninstall' to put it back" mv /usr/share/ibus/component/keyman.xml /usr/share/doc/ibus-keyman/ else echo "component file is local one so overwrite it" @@ -37,10 +37,10 @@ cd "$BASEDIR" cd ibus-keyman if [[ "${SUDOINSTALL}" == "uninstall" ]]; then - echo "doing make uninstall of ibus-keyman" + echo "doing build.sh uninstall of ibus-keyman" ./build.sh uninstall else - echo "doing make install of ibus-keyman" + echo "doing build.sh install of ibus-keyman" ./build.sh install fi cd "$BASEDIR" @@ -49,20 +49,20 @@ cd keyman-config echo "SUDOINSTALL: ${SUDOINSTALL}" if [[ "${SUDOINSTALL}" == "yes" ]]; then if [ ! -d build ]; then - echo "keyman-config must be built before it is installed. Run 'make configure' if needed then 'make'" + echo "keyman-config must be built before it is installed. Run 'build.sh configure build'." exit 1 fi echo "doing sudo glib-compile-schemas for keyman-config" cp com.keyman.gschema.xml /usr/share/glib-2.0/schemas/ glib-compile-schemas /usr/share/glib-2.0/schemas/ echo "doing sudo install of keyman-config" - make install + ./build.sh install elif [[ "${SUDOINSTALL}" == "uninstall" ]]; then echo "doing sudo uninstall of keyman-config" - make uninstall + ./build.sh uninstall else echo "doing /tmp install of keyman-config" - make install-temp + ./build.sh install fi cd "$BASEDIR" diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index 39ea70f7d1719859c9bba2a3de0240b013628b52..6c249891c1633a3bcd5da6279f6fc13a316cc29a 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -28,10 +28,7 @@ function downloadSource() { if [ "${proj:=}" == "keyman" ]; then cd "${BASEDIR}" || exit - fi - - if [ "${proj}" == "keyman" ]; then - make clean + ./build.sh clean fi # Update tier in Debian watch files (replacing any previously set tier) and remove comment diff --git a/linux/scripts/reconf.sh b/linux/scripts/reconf.sh index cd74571a4a1dc696934de83f04d395d4850cda05..4e4f5c1b115cbc37e02905dfcdc262ccb9941024 100755 --- a/linux/scripts/reconf.sh +++ b/linux/scripts/reconf.sh @@ -21,7 +21,7 @@ cd "$BASEDIR/ibus-keyman" ./build.sh clean configure cd "$BASEDIR/keyman-config" -make clean +./build.sh clean cd keyman_config sed \ diff --git a/oem/firstvoices/ios/exportAppStore.plist b/oem/firstvoices/ios/exportAppStore.plist index 123af31a0c437e0927eddef23f03b3a850374642..c4ac972730f0715629ebd05914f11ab9f1026ca6 100644 --- a/oem/firstvoices/ios/exportAppStore.plist +++ b/oem/firstvoices/ios/exportAppStore.plist @@ -7,9 +7,9 @@ teamID D7TR486TEH signingCertificate - EFA9DE793B2A25F38270468AE62060CFF55CD634 + C4726D4C8BD4C2FE38A551B84CEDB8E4B3FD12D6 installerSigningCertificate - EFA9DE793B2A25F38270468AE62060CFF55CD634 + C4726D4C8BD4C2FE38A551B84CEDB8E4B3FD12D6 provisioningProfiles com.firstvoices.keyboards diff --git a/package-lock.json b/package-lock.json index 5a6663b179eb3e80b883f4b208695a4cdd023e97..d78384e30fc3d19b8a5008e283abf8a4eb4dc4d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "developer/src/kmc", "developer/src/server", "common/models/*", + "common/test/resources", "common/tools/*", "common/web/*", "common/predictive-text", @@ -34,6 +35,7 @@ "devDependencies": { "@typescript-eslint/eslint-plugin": "^5.59.1", "chai": "^4.3.4", + "esbuild": "^0.15.16", "eslint": "^8.39.0", "eslint-config-standard-with-typescript": "^34.0.1", "eslint-plugin-import": "^2.27.5", @@ -42,6 +44,7 @@ "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^10.9.1", + "tslib": "^2.5.2", "typescript": "^4.9.5" } }, @@ -49,7 +52,10 @@ "name": "@keymanapp/models-templates", "license": "MIT", "dependencies": { - "@keymanapp/models-wordbreakers": "*" + "@keymanapp/keyman-version": "*", + "@keymanapp/models-types": "*", + "@keymanapp/models-wordbreakers": "*", + "@keymanapp/resources-gosh": "*" }, "devDependencies": { "@keymanapp/models-types": "*", @@ -59,6 +65,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "typescript": "^4.9.5" } }, @@ -82,10 +89,12 @@ }, "devDependencies": { "@keymanapp/models-types": "*", + "@keymanapp/resources-gosh": "*", "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^10.9.1", "typescript": "^4.9.5" } @@ -136,6 +145,22 @@ "dev": true, "license": "MIT" }, + "common/test/resources": { + "name": "@keymanapp/common-test-resources", + "license": "MIT", + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "@types/node": "^10.17.21", + "chai": "^4.3.4", + "typescript": "^4.9.5" + } + }, + "common/test/resources/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", + "dev": true + }, "common/tools/hextobin": { "name": "@keymanapp/hextobin", "license": "MIT", @@ -316,43 +341,8 @@ "mocha-teamcity-reporter": "^4.0.0", "promise-status-async": "^1.2.10", "sinon": "^14.0.0", - "ts-node": "^9.1.1", - "typescript": "^4.5.4" - } - }, - "common/web/gesture-recognizer/node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, - "common/web/gesture-recognizer/node_modules/ts-node": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", - "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", - "dev": true, - "dependencies": { - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.17", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "typescript": ">=2.7" + "ts-node": "^10.9.1", + "typescript": "^4.9.5" } }, "common/web/input-processor": { @@ -389,6 +379,12 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", + "karma": "^6.4.1", + "karma-chai": "^0.1.0", + "karma-chrome-launcher": "^2.2.0", + "karma-mocha": "^2.0.1", + "karma-mocha-reporter": "^2.2.5", + "karma-teamcity-reporter": "^1.1.0", "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^10.9.1", @@ -422,16 +418,19 @@ "@keymanapp/keyman-version": "*", "@keymanapp/models-templates": "*", "@keymanapp/models-wordbreakers": "*", + "@keymanapp/sourcemap-path-remapper": "*", "@keymanapp/web-utils": "*", "es6-shim": "^0.35.5", "string.prototype.codepointat": "^0.2.1", "string.prototype.startswith": "^0.2.0" }, "devDependencies": { + "@keymanapp/common-test-resources": "*", "@keymanapp/models-types": "*", "@keymanapp/resources-gosh": "*", "@types/node": "^10.17.21", "chai": "^4.3.4", + "combine-source-map": "^0.8.0", "karma": "^6.4.1", "karma-browserstack-launcher": "^1.6.0", "karma-chai": "^0.1.0", @@ -483,6 +482,17 @@ "typescript": "^4.9.5" } }, + "common/web/tslib": { + "name": "@keymanapp/tslib", + "dependencies": { + "tslib": "^2.5.2", + "typescript": "^4.9.5" + }, + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "esbuild": "^0.15.16" + } + }, "common/web/types": { "name": "@keymanapp/common-types", "license": "MIT", @@ -731,6 +741,9 @@ "@keymanapp/keyman-version": "*", "@keymanapp/resources-gosh": "*", "@types/node": "^14.0.5", + "chai": "^4.3.4", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", "typescript": "^4.9.5" } }, @@ -891,26 +904,6 @@ "type-detect": "4.0.8" } }, - "developer/src/kmc-kmn/node_modules/@sinonjs/samsam": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-7.0.1.tgz", - "integrity": "sha512-zsAk2Jkiq89mhZovB2LLOdTCxJF4hqqTToGP0ASWlhp4I1hqOjcfmZGafXntCN7MDC6yySH0mFHrYtHceOeLmw==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^2.0.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - }, - "developer/src/kmc-kmn/node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, "developer/src/kmc-kmn/node_modules/@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", @@ -964,12 +957,6 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, - "developer/src/kmc-kmn/node_modules/isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", - "dev": true - }, "developer/src/kmc-kmn/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -1071,37 +1058,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, - "developer/src/kmc-kmn/node_modules/nise": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.4.tgz", - "integrity": "sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^2.0.0", - "@sinonjs/fake-timers": "^10.0.2", - "@sinonjs/text-encoding": "^0.7.1", - "just-extend": "^4.0.2", - "path-to-regexp": "^1.7.0" - } - }, - "developer/src/kmc-kmn/node_modules/nise/node_modules/@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "developer/src/kmc-kmn/node_modules/path-to-regexp": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", - "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", - "dev": true, - "dependencies": { - "isarray": "0.0.1" - } - }, "developer/src/kmc-kmn/node_modules/sinon": { "version": "15.0.2", "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.0.2.tgz", @@ -1198,6 +1154,7 @@ } }, "developer/src/kmc-ldml": { + "name": "@keymanapp/kmc-ldml", "license": "MIT", "dependencies": { "@keymanapp/keyman-version": "*", @@ -2307,6 +2264,38 @@ "node": ">=12" } }, + "node_modules/@esbuild/android-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.18.tgz", + "integrity": "sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.18.tgz", + "integrity": "sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -2453,6 +2442,10 @@ "resolved": "resources/build/version", "link": true }, + "node_modules/@keymanapp/common-test-resources": { + "resolved": "common/test/resources", + "link": true + }, "node_modules/@keymanapp/common-types": { "resolved": "common/web/types", "link": true @@ -2461,14 +2454,14 @@ "resolved": "developer/src/server", "link": true }, - "node_modules/@keymanapp/gesture-recognizer": { - "resolved": "common/web/gesture-recognizer", - "link": true - }, "node_modules/@keymanapp/developer-test-helpers": { "resolved": "developer/src/common/web/test-helpers", "link": true }, + "node_modules/@keymanapp/gesture-recognizer": { + "resolved": "common/web/gesture-recognizer", + "link": true + }, "node_modules/@keymanapp/hextobin": { "resolved": "common/tools/hextobin", "link": true @@ -2549,6 +2542,10 @@ "resolved": "common/tools/sourcemap-path-remapper", "link": true }, + "node_modules/@keymanapp/tslib": { + "resolved": "common/web/tslib", + "link": true + }, "node_modules/@keymanapp/web-sentry-manager": { "resolved": "common/web/sentry-manager", "link": true @@ -2796,6 +2793,11 @@ "node": ">=6" } }, + "node_modules/@sentry/browser/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sentry/cli": { "version": "2.2.0", "dev": true, @@ -2830,6 +2832,11 @@ "node": ">=6" } }, + "node_modules/@sentry/core/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sentry/hub": { "version": "5.30.0", "license": "BSD-3-Clause", @@ -2842,6 +2849,11 @@ "node": ">=6" } }, + "node_modules/@sentry/hub/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sentry/minimal": { "version": "5.30.0", "license": "BSD-3-Clause", @@ -2854,6 +2866,11 @@ "node": ">=6" } }, + "node_modules/@sentry/minimal/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sentry/node": { "version": "6.19.6", "license": "BSD-3-Clause", @@ -2927,6 +2944,11 @@ "node": ">=6" } }, + "node_modules/@sentry/node/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sentry/types": { "version": "5.30.0", "license": "BSD-3-Clause", @@ -2945,6 +2967,11 @@ "node": ">=6" } }, + "node_modules/@sentry/utils/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "license": "MIT", @@ -4366,6 +4393,33 @@ "color-support": "bin.js" } }, + "node_modules/combine-source-map": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz", + "integrity": "sha512-UlxQ9Vw0b/Bt/KYwCFqdEwsQ1eL8d1gibiFb7lxQJFdvTgc2hIZi6ugsg+kyhzhPV+QEpUiEIwInIAIrgoEkrg==", + "dev": true, + "dependencies": { + "convert-source-map": "~1.1.0", + "inline-source-map": "~0.6.0", + "lodash.memoize": "~3.0.3", + "source-map": "~0.5.3" + } + }, + "node_modules/combine-source-map/node_modules/convert-source-map": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz", + "integrity": "sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg==", + "dev": true + }, + "node_modules/combine-source-map/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/component-emitter": { "version": "1.3.0", "dev": true, @@ -4937,9 +4991,10 @@ }, "node_modules/esbuild": { "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.18.tgz", + "integrity": "sha512-x/R72SmW3sSFRm5zrrIjAhCeQSAWoni3CmHEqfQrZIQTM3lVCdehdwuIqaOtfC2slvpdlLa62GYoN8SxT23m6Q==", "dev": true, "hasInstallScript": true, - "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, @@ -4971,13 +5026,46 @@ "esbuild-windows-arm64": "0.15.18" } }, + "node_modules/esbuild-android-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.18.tgz", + "integrity": "sha512-wnpt3OXRhcjfIDSZu9bnzT4/TNTDsOUvip0foZOUBG7QbSt//w3QV4FInVJxNhKc/ErhUxc5z4QjHtMi7/TbgA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-android-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.18.tgz", + "integrity": "sha512-G4xu89B8FCzav9XU8EjsXacCKSG2FT7wW9J6hOc18soEHJdtWu03L3TQDGf0geNxfLTtxENKBzMSq9LlbjS8OQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/esbuild-darwin-64": { "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.18.tgz", + "integrity": "sha512-2WAvs95uPnVJPuYKP0Eqx+Dl/jaYseZEUUT1sjg97TJa4oBtbAKnPnl3b5M9l51/nbx7+QAEtuummJZW0sBEmg==", "cpu": [ "x64" ], "dev": true, - "license": "MIT", "optional": true, "os": [ "darwin" @@ -4986,6 +5074,278 @@ "node": ">=12" } }, + "node_modules/esbuild-darwin-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.18.tgz", + "integrity": "sha512-tKPSxcTJ5OmNb1btVikATJ8NftlyNlc8BVNtyT/UAr62JFOhwHlnoPrhYWz09akBLHI9nElFVfWSTSRsrZiDUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.18.tgz", + "integrity": "sha512-TT3uBUxkteAjR1QbsmvSsjpKjOX6UkCstr8nMr+q7zi3NuZ1oIpa8U41Y8I8dJH2fJgdC3Dj3CXO5biLQpfdZA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.18.tgz", + "integrity": "sha512-R/oVr+X3Tkh+S0+tL41wRMbdWtpWB8hEAMsOXDumSSa6qJR89U0S/PpLXrGF7Wk/JykfpWNokERUpCeHDl47wA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.18.tgz", + "integrity": "sha512-lphF3HiCSYtaa9p1DtXndiQEeQDKPl9eN/XNoBf2amEghugNuqXNZA/ZovthNE2aa4EN43WroO0B85xVSjYkbg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.18.tgz", + "integrity": "sha512-hNSeP97IviD7oxLKFuii5sDPJ+QHeiFTFLoLm7NZQligur8poNOWGIgpQ7Qf8Balb69hptMZzyOBIPtY09GZYw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.18.tgz", + "integrity": "sha512-UH779gstRblS4aoS2qpMl3wjg7U0j+ygu3GjIeTonCcN79ZvpPee12Qun3vcdxX+37O5LFxz39XeW2I9bybMVA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.18.tgz", + "integrity": "sha512-54qr8kg/6ilcxd+0V3h9rjT4qmjc0CccMVWrjOEM/pEcUzt8X62HfBSeZfT2ECpM7104mk4yfQXkosY8Quptug==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-mips64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.18.tgz", + "integrity": "sha512-Mk6Ppwzzz3YbMl/ZZL2P0q1tnYqh/trYZ1VfNP47C31yT0K8t9s7Z077QrDA/guU60tGNp2GOwCQnp+DYv7bxQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-ppc64le": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.18.tgz", + "integrity": "sha512-b0XkN4pL9WUulPTa/VKHx2wLCgvIAbgwABGnKMY19WhKZPT+8BxhZdqz6EgkqCLld7X5qiCY2F/bfpUUlnFZ9w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-riscv64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.18.tgz", + "integrity": "sha512-ba2COaoF5wL6VLZWn04k+ACZjZ6NYniMSQStodFKH/Pu6RxzQqzsmjR1t9QC89VYJxBeyVPTaHuBMCejl3O/xg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-s390x": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.18.tgz", + "integrity": "sha512-VbpGuXEl5FCs1wDVp93O8UIzl3ZrglgnSQ+Hu79g7hZu6te6/YHgVJxCM2SqfIila0J3k0csfnf8VD2W7u2kzQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-netbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.18.tgz", + "integrity": "sha512-98ukeCdvdX7wr1vUYQzKo4kQ0N2p27H7I11maINv73fVEXt2kyh4K4m9f35U1K43Xc2QGXlzAw0K9yoU7JUjOg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-openbsd-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.18.tgz", + "integrity": "sha512-yK5NCcH31Uae076AyQAXeJzt/vxIo9+omZRKj1pauhk3ITuADzuOx5N2fdHrAKPxN+zH3w96uFKlY7yIn490xQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-sunos-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.18.tgz", + "integrity": "sha512-On22LLFlBeLNj/YF3FT+cXcyKPEI263nflYlAhz5crxtp3yRG1Ugfr7ITyxmCmjm4vbN/dGrb/B7w7U8yJR9yw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-32": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.18.tgz", + "integrity": "sha512-o+eyLu2MjVny/nt+E0uPnBxYuJHBvho8vWsC2lV61A7wwTWC3jkN2w36jtA+yv1UgYkHRihPuQsL23hsCYGcOQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.18.tgz", + "integrity": "sha512-qinug1iTTaIIrCorAUjR0fcBk24fjzEedFYhhispP8Oc7SFvs+XeW3YpAKiKp8dRpizl4YYAhxMjlftAMJiaUw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-arm64": { + "version": "0.15.18", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.18.tgz", + "integrity": "sha512-q9bsYzegpZcLziq0zgUi5KqGVtfhjxGbnksaBFYmWLxeV/S1fK4OLdq2DFYnXcLMjlZw2L0jLsk1eGoB522WXQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/escalade": { "version": "3.1.1", "dev": true, @@ -6263,11 +6623,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/google-closure-compiler-java": { - "version": "20200224.0.0", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -6596,6 +6951,24 @@ "version": "2.0.4", "license": "ISC" }, + "node_modules/inline-source-map": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz", + "integrity": "sha512-0mVWSSbNDvedDWIN4wxLsdPM4a7cIPcpyMxj3QZ406QRwQ6ePGB1YIHxVPjqpcUGbWQ5C+nHTwGNWAGvt7ggVA==", + "dev": true, + "dependencies": { + "source-map": "~0.5.3" + } + }, + "node_modules/inline-source-map/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/internal-slot": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", @@ -7411,6 +7784,12 @@ "version": "4.4.2", "license": "MIT" }, + "node_modules/lodash.memoize": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz", + "integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A==", + "dev": true + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -8285,15 +8664,6 @@ "type-detect": "4.0.8" } }, - "node_modules/nise/node_modules/@sinonjs/fake-timers": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.0.2.tgz", - "integrity": "sha512-SwUDyjWnah1AaNl7kxsa7cfLhlTYoiyhDAIgyh+El30YvXs/o7OLXpYH88Zdhyx9JExKrmHDJ+10bwIcY80Jmw==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^2.0.0" - } - }, "node_modules/nise/node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -9377,34 +9747,6 @@ "sinon": ">=4.0.0" } }, - "node_modules/sinon/node_modules/diff": { - "version": "3.5.0", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/sinon/node_modules/has-flag": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/sinon/node_modules/supports-color": { - "version": "5.5.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/sinon" - } - }, "node_modules/sinon/node_modules/@sinonjs/commons": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", @@ -9986,8 +10328,9 @@ } }, "node_modules/tslib": { - "version": "1.14.1", - "license": "0BSD" + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.2.tgz", + "integrity": "sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -10004,6 +10347,12 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, + "node_modules/tsutils/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true + }, "node_modules/tunnel": { "version": "0.0.6", "license": "MIT", @@ -10779,16 +11128,17 @@ "@keymanapp/lexical-model-layer": "*", "@keymanapp/models-types": "*", "@keymanapp/recorder-core": "*", + "@keymanapp/tslib": "*", "@keymanapp/web-utils": "*", "@types/node": "^11.9.4", - "eventemitter3": "^4.0.0" + "eventemitter3": "^4.0.0", + "tslib": "^2.5.2" }, "devDependencies": { "@keymanapp/resources-gosh": "*", "@keymanapp/web-sentry-manager": "*", "@sentry/cli": "2.2.0", "chai": "^4.3.4", - "google-closure-compiler-java": "^20200224.0.0", "karma": "^6.4.1", "karma-browserstack-launcher": "^1.6.0", "karma-chai": "^0.1.0", diff --git a/package.json b/package.json index 7a2b53d50d53a777f786760652e187d9e6d5da02..787fe68e5a2b5c08988f0cae658d88de9f0b3fa3 100644 --- a/package.json +++ b/package.json @@ -2,17 +2,19 @@ "name": "root", "private": true, "devDependencies": { - "chai": "^4.3.4", - "mocha": "^10.0.0", - "mocha-teamcity-reporter": "^4.0.0", - "ts-node": "^10.9.1", - "typescript": "^4.9.5", "@typescript-eslint/eslint-plugin": "^5.59.1", + "chai": "^4.3.4", + "esbuild": "^0.15.16", "eslint": "^8.39.0", "eslint-config-standard-with-typescript": "^34.0.1", "eslint-plugin-import": "^2.27.5", "eslint-plugin-n": "^15.7.0", - "eslint-plugin-promise": "^6.1.1" + "eslint-plugin-promise": "^6.1.1", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", + "ts-node": "^10.9.1", + "tslib": "^2.5.2", + "typescript": "^4.9.5" }, "scripts": {}, "workspaces": [ @@ -28,6 +30,7 @@ "developer/src/kmc", "developer/src/server", "common/models/*", + "common/test/resources", "common/tools/*", "common/web/*", "common/predictive-text", @@ -35,10 +38,10 @@ "web" ], "dependencies": { + "@keymanapp/common-types": "file:common/web/types", + "@keymanapp/developer-test-helpers": "file:developer/src/common/web/test-helpers", "@keymanapp/hextobin": "file:common/tools/hextobin", "@keymanapp/keyman-version": "file:common/web/keyman-version", - "@keymanapp/common-types": "file:common/web/types", - "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml", - "@keymanapp/developer-test-helpers": "file:developer/src/common/web/test-helpers" + "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml" } } diff --git a/resources/build/build-utils-ci.inc.sh b/resources/build/build-utils-ci.inc.sh index de36ae9e9b606556869c22f2f7492240825070f7..cab449a4636d59e5f9a388bfad84601d165e0956 100644 --- a/resources/build/build-utils-ci.inc.sh +++ b/resources/build/build-utils-ci.inc.sh @@ -100,18 +100,7 @@ function _builder_publish_npm_package() { dry_run=--dry-run fi - # We use --no-git-tag-version because our CI system controls version numbering - # and already tags releases. We also want to have the version of this match - # the release of Keyman Developer -- these two versions should be in sync. - # Because this is a large repo with multiple projects and build systems, it's - # better for us that individual build systems don't take too much ownership of - # git tagging. :) - npm version --allow-same-version --no-git-tag-version --no-commit-hooks "$VERSION_WITH_TAG" - - # Update all @keymanapp/* [*]dependencies in package.json to the current - # version-with-tag, so that the published version has precise dependencies, - # and we don't accidentally end up with either older or newer deps here - _builder_npm_set_dependency_version + _builder_write_npm_version # Note: In either case, npm publish MUST be given --access public to publish a # package in the @keymanapp scope on the public npm package index. @@ -128,18 +117,33 @@ function _builder_publish_npm_package() { fi } -# Updates all @keymanapp/* [*]dependencies in package.json to the current -# version-with-tag, so that the published version has precise dependencies, and -# we don't accidentally end up with either older or newer deps. This overwrites -# the local package.json, so it does need to be restored afterwards -function _builder_npm_set_dependency_version() { - cat package.json | "$JQ" --arg VERSION_WITH_TAG "$VERSION_WITH_TAG" \ - ' - . + - (try { dependencies: (.dependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + - (try { devDependencies: (.devDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + - (try { bundleDependencies: (.bundleDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + - (try { optionalDependencies: (.optionalDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) - ' > package1.json - mv -f package1.json package.json +function _builder_write_npm_version() { + # We use --no-git-tag-version because our CI system controls version numbering + # and already tags releases. We also want to have the version of this match + # the release of Keyman Developer -- these two versions should be in sync. + # Because this is a large repo with multiple projects and build systems, it's + # better for us that individual build systems don't take too much ownership of + # git tagging. :) + if ! "$JQ" -e '.version' package.json > /dev/null; then + pushd "$KEYMAN_ROOT" > /dev/null + npm version --allow-same-version --no-git-tag-version --no-commit-hooks --workspaces "$VERSION_WITH_TAG" + popd > /dev/null + fi + + # Updates all @keymanapp/* [*]dependencies in all package.jsons to the current + # version-with-tag, so that the published version has precise dependencies, and + # we don't accidentally end up with either older or newer deps. This overwrites + # the local package.json files, so they do need to be restored afterwards + find "$KEYMAN_ROOT" -name "package.json" -not -path '*/node_modules/*' -print0 | \ + while IFS= read -r -d '' line; do + cat "$line" | "$JQ" --arg VERSION_WITH_TAG "$VERSION_WITH_TAG" \ + ' + . + + (try { dependencies: (.dependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + + (try { devDependencies: (.devDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + + (try { bundleDependencies: (.bundleDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + + (try { optionalDependencies: (.optionalDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + ' > "${line}_" + mv -f "${line}_" "$line" + done } \ No newline at end of file diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index f9440ae83eb36f3f293a56efaee5de96bccce8ce..6a47cbd5295aeeda7f5f0c0a7622fad5bf7b24f9 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -346,12 +346,12 @@ _builder_failure_trap() { fi builder_finish_action failure $action$target - - # Make 100% sure that the exit code chains fully. - # Without this, nested scripts have failed to chain errors from npm calls past the script - # that directly executed the failed npm command. - exit $trappedExitCode fi + + # Make 100% sure that the exit code chains fully. + # Without this, nested scripts have failed to chain errors from npm calls past the script + # that directly executed the failed npm command. + exit $trappedExitCode } # @@ -711,6 +711,30 @@ _builder_expand_action_targets() { fi } +_builder_child_base= +# +# Describes the path from the build script's working directory to the common subfolder +# containing child scripts / projects without defined custom paths. +# +# This function must be called to set the child base path before builder_describe is +# called in order to work correctly. Furthermore, note that this setting will be +# ignored by targets with custom paths. +# +# ### Usage +# +# ```bash +# builder_set_child_base path +# ``` +# +# ### Parameters +# +# * `path` The relative path from the directory containing the calling script to +# the base folder to use for child-project detection and resolution +# +builder_set_child_base() { + _builder_child_base="$1/" +} + # # Describes a build script, defines available parameters and their meanings. Use # together with `builder_parse` to process input parameters. @@ -827,8 +851,8 @@ builder_describe() { else # If the target name matches a folder name, implicitly # make it available as a child project - if [[ -d "$THIS_SCRIPT_PATH/${value:1}" ]]; then - target_path="${value:1}" + if [[ -d "$THIS_SCRIPT_PATH/$_builder_child_base${value:1}" ]]; then + target_path="$_builder_child_base${value:1}" fi fi _builder_targets+=($value) @@ -945,8 +969,8 @@ builder_describe() { # # ```bash # builder_describe_outputs \ -# "configure" "/node_modules" \ -# "build" "build/index.js" +# configure /node_modules \ +# build build/index.js # ``` # function builder_describe_outputs() { @@ -1337,6 +1361,7 @@ _builder_parse_expanded_parameters() { _builder_report_dependencies ;; *) + # script does not recognize anything of action or target form at this point. _builder_parameter_error "$0" parameter "$key" esac fi diff --git a/tsconfig-base.json b/tsconfig-base.json index 8fb21424110afb5cf5691249d94a1921863764f9..8e3eafb6cf49d7b0638c8ac2c58ab25d49d743f5 100644 --- a/tsconfig-base.json +++ b/tsconfig-base.json @@ -12,10 +12,11 @@ "paths": { "@keymanapp/input-processor": ["./common/web/input-processor/src"], "@keymanapp/keyboard-processor": ["./common/web/keyboard-processor/src"], + "@keymanapp/keyman": ["./web" ], "@keymanapp/models-types": ["./common/models/types"], "@keymanapp/models-templates": ["./common/models/templates"], "@keymanapp/models-wordbreakers": ["./common/models/wordbreakers"], - "@keymanapp/utils": ["./common/web/utils"], + "@keymanapp/web-utils": ["./common/web/utils"], "@keymanapp/lm-message-types": ["./common/web/lm-message-types"], "@keymanapp/keyman-version": ["./common/web/keyman-version"], } diff --git a/tsconfig.cjs.json b/tsconfig.cjs.json index 0c5f0092885579fe92b935c40f771fa6d998650d..4695672e90831e10d897d2f540feed5943d3689d 100644 --- a/tsconfig.cjs.json +++ b/tsconfig.cjs.json @@ -7,35 +7,14 @@ "references": [ { "path": "./common/web/gesture-recognizer/tsconfig.json" }, { "path": "./common/web/gesture-recognizer/src/tools/unit-test-resources/tsconfig.json" }, - { "path": "./common/web/input-processor/src/tsconfig.json" }, - { "path": "./common/web/keyboard-processor/src/tsconfig.json" }, - { "path": "./common/web/recorder/src/nodeProctor.tsconfig.json" }, - { "path": "./common/web/recorder/src/tsconfig.json" }, - { "path": "./common/web/sentry-manager/src/tsconfig.json" }, - { "path": "./common/web/utils/tsconfig.json" }, - - { "path": "./common/models/templates/tsconfig.json" }, - { "path": "./common/models/types/tsconfig.json" }, - { "path": "./common/models/wordbreakers/tsconfig.json" }, - - { "path": "./common/predictive-text/browser.tsconfig.json" }, { "path": "./common/predictive-text/testing/one-stage-embedded-webworker/tsconfig.json" }, { "path": "./common/predictive-text/testing/two-stage-embedded-webworker/tsconfig.json" }, { "path": "./common/predictive-text/testing/two-stage-embedded-webworker/worker/tsconfig.json" }, - { "path": "./common/predictive-text/tsconfig.json" }, { "path": "./developer/src/server/tsconfig.json" }, { "path": "./resources/build/version/tsconfig.json" }, { "path": "./resources/build/version/tsconfig.production.json" }, - - { "path": "./web/bulk_rendering/tsconfig.json" }, - { "path": "./web/source/tsconfig.json" }, - { "path": "./web/tools/recorder/tsconfig.json" }, - { "path": "./web/tools/sourcemap-root/tsconfig.json" }, - - { "path": "./common/web/lm-message-types/" }, - { "path": "./common/web/lm-worker/" }, - { "path": "./common/web/keyman-version/" }, + // { "path": "./web/bulk_rendering/tsconfig.json" }, ] } \ No newline at end of file diff --git a/tsconfig.esm.json b/tsconfig.esm.json index 44bf4011df97fe970e73e01794dc2fc562be0bb8..de47c84fdeafa7cfbf90c721ac86bd3a0abedd2c 100644 --- a/tsconfig.esm.json +++ b/tsconfig.esm.json @@ -21,7 +21,24 @@ { "path": "./developer/src/kmc-package/test/tsconfig.json" }, { "path": "./developer/src/kmc-package/tsconfig.json" }, - { "path": "./common/web/keyman-version/tsconfig.esm.json" }, + { "path": "./common/web/keyman-version" }, { "path": "./common/web/types/" }, + + { "path": "./common/web/input-processor/tsconfig.json" }, + { "path": "./common/web/keyboard-processor/tsconfig.json" }, + { "path": "./common/web/recorder/tsconfig.json" }, + { "path": "./common/web/sentry-manager/src/tsconfig.json" }, + { "path": "./common/web/utils/tsconfig.json" }, + + { "path": "./common/models/templates/tsconfig.json" }, + { "path": "./common/models/types/tsconfig.json" }, + { "path": "./common/models/wordbreakers/tsconfig.json" }, + { "path": "./common/predictive-text/tsconfig.json" }, + + { "path": "./web/src/tsconfig.all.json" }, + // { "path": "./web/tools/recorder/tsconfig.json" }, + // { "path": "./web/tools/sourcemap-root/tsconfig.json" }, + { "path": "./common/web/lm-message-types/" }, + { "path": "./common/web/lm-worker/" }, ] } \ No newline at end of file diff --git a/web/README.md b/web/README.md index 4f49dbdf085e2b2c3dec49bb70cbb5562314698c..7558c2a2c2067d858b3a9a519f522bbf130c2b98 100644 --- a/web/README.md +++ b/web/README.md @@ -9,19 +9,18 @@ configure your build environment. The following folders contain the distribution for Keyman Engine for Web: - source Source code - source/resources/osk OSK resources for inclusion in mobile app builds; - keymanweb-osk.ttf is maintained at https://github.com/silnrsi/font-keymanweb-osk - unit_tests Automated testing resources + src Source code + build/app/resources OSK + UI resources for inclusion in all build types; + keymanweb-osk.ttf is maintained at https://github.com/silnrsi/font-keymanweb-osk - build/web/release Fully-compiled KeymanWeb modules for release - build/embed/release Fully-compiled KMEA/KMEI modules for inclusion in mobile app builds - build/web/debug Fully-compiled but non-minified KeymanWeb modules - build/embed/debug Fully-compiled but non-minified KMEA/KMEI modules + build/app/browser/release Fully-compiled KeymanWeb modules for release + build/app/webview/release Fully-compiled KMEA/KMEI modules for inclusion in mobile app builds + build/app/browser/debug Fully-compiled but non-minified KeymanWeb modules + build/app/webview/debug Fully-compiled but non-minified KMEA/KMEI modules - samples Sample pages demonstrating ways to link with KeymanWeb - testing Test-case web-pages for various aspects of KeymanWeb functionality - unit_tests A Node-driven test suite for automated testing of KeymanWeb + src/samples Sample pages demonstrating ways to link with KeymanWeb + src/test/manual Test-case web-pages for various aspects of KeymanWeb functionality + src/test/auto A Node-driven test suite for automated testing of KeymanWeb ********************************************************************** @@ -32,7 +31,7 @@ compile Keyman Engine for Web before viewing the pages. Refer to the samples for usage details. To view pages using compiled Keyman Engine for Web, -1. cd to **keyman/web/source** +1. cd to **keyman/web/** 2. Run `./build.sh` - Use `./build.sh --help` for the script's documentation. @@ -42,18 +41,18 @@ Before running unit tests on Keyman Engine for Web, first run `./build.sh` according to the instructions above. Once the build is complete, running `npm test` will run the unit testing suite -on your local machine in-browser. Alternatively, see `unit_tests/test.sh`, which +on your local machine in-browser. Alternatively, see `test.sh`, which the former command executes. ### Debugging Unit Tests 1. During development, to run a specific unit test, change the `it` to `it.only`. You can also run all tests under a specific group with `describe.only`. -2. From this directory, run `./unit_tests/test.sh -debug`. Alternatively, from +2. From this directory, run `./test.sh --debug`. Alternatively, from `web/` or any `web/` subdirectory, ``` - npm run test -- -debug + npm run test -- --debug ``` The `--` part tells `npm` to funnel anything to the script as the script's @@ -65,3 +64,73 @@ the former command executes. tab. 4. In the Dev console, you can set a breakpoint in your test and refresh the page to debug + +### Approximate Overall Design + +```mermaid +graph TD; + OSK[web/src/engine/osk]; + KP["common/web/keyboard-processor"]; + IP["common/web/input-processor"]; + OSK-->KP; + IP-->KP; + Utils["common/web/utils"]; + KP---->Utils; + Wordbreakers["common/models/wordbreakers"]; + Models["common/models/templates"]; + Models-->Utils; + LMWorker["common/web/lm-worker"]; + LMWorker-->Models; + LMWorker-->Wordbreakers; + LMLayer["common/predictive-text"]; + LMLayer-->LMWorker; + IP-->LMLayer; + + subgraph PredText["WebWorker + its interface"] + LMLayer; + LMWorker; + Models; + Wordbreakers; + end + + subgraph Headless["Fully headless components"] + direction LR + KP; + IP; + Utils; + PredText; + end + + subgraph ClassicWeb["Previously unmodularized components"] + Device[web/src/engine/device-detect]; + Device----->Utils; + Elements[web/src/engine/element-wrappers]; + Elements-->KP; + KeyboardCache[web/src/engine/package-cache]; + KeyboardCache-->IP; + DomUtils[web/src/engine/dom-utils]; + DomUtils-->Utils; + OSK-->DomUtils; + OSK---->IP; + Configuration[web/src/engine/paths]; + Configuration-->OSK; + CommonEngine[web/src/engine/main]; + CommonEngine-->Configuration; + CommonEngine-->Device; + CommonEngine-->KeyboardCache; + CommonEngine-->OSK; + Attachment[web/src/engine/attachment]; + Attachment-->DomUtils; + Attachment-->Elements; + end + + subgraph WebEngine["Keyman Engine for Web (top-level libraries)"] + Browser[web/src/app/browser]; + WebView[web/src/app/webview]; + + WebView--->CommonEngine; + + Browser--->CommonEngine; + Browser-->Attachment; + end +``` \ No newline at end of file diff --git a/web/build.sh b/web/build.sh index d9ccdb3067d35e8d61c1a2d6d833544e612b2c44..35ee1b5ebbe943773eacd1c85205715db3390fe1 100755 --- a/web/build.sh +++ b/web/build.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash # -# Compiles the Keyman Engine for Web and its various end-products -# - -set -eu +# Compile keymanweb and copy compiled javascript and resources to output/embedded folder ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary @@ -18,475 +15,112 @@ cd "$THIS_SCRIPT_PATH" # ################################ Main script ################################ -# Definition of global compile constants - -GESTURE_RECOGNIZER_BUILD="$KEYMAN_ROOT/common/web/gesture-recognizer/build/." -GESTURE_RECOGNIZER_TARGET="../build/engine/gesture-recognizer/" - # Ensures that we rely first upon the local npm-based install of Typescript. # (Facilitates automated setup for build agents.) # TODO: this should be removeable given set_keyman_standard_build_path does this in build-utils.sh (and relative paths are dodgy in $PATH!) PATH="../node_modules/.bin:$PATH" -PREDICTIVE_TEXT_SOURCE="../common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js" -PREDICTIVE_TEXT_OUTPUT="src/test/manual/web/prediction-ui/simple-en-trie.js" - -builder_describe "Builds Keyman Engine for Web (KMW)." \ - "@/common/web/keyman-version build" \ - "@/common/web/input-processor build" \ - "@/web/src/tools/building/sourcemap-root build" \ +builder_set_child_base src +builder_describe "Builds engine modules for Keyman Engine for Web (KMW)." \ + "@/common/web/gesture-recognizer build:engine/gestures" \ "clean" \ "configure" \ "build" \ - "test Runs unit tests. Only $(builder_term test:web) is currently defined" \ - ":embed Builds the configuration of KMW used within the Keyman mobile apps" \ - ":engine Builds all common code used by other targets" \ - ":web Builds the website-oriented configuration of Keyman Engine for Web" \ - ":ui Builds the desktop UI modules used by the $(builder_term :web) build product" \ - ":samples Builds only sample & test pages found under src/samples and src/test" \ - ":tools Builds related development + unit-test resources" \ - "--no-minify Skips any minification steps in the build" \ - "--all Sets action to run on KMW's submodules as well if appropriate (actions - $(builder_term test))" + "test" \ + ":app/browser The form of Keyman Engine for Web for use on websites" \ + ":app/webview A puppetable version of KMW designed for use in a host app's WebView" \ + ":app/ui Builds KMW's desktop form-factor keyboard-selection UI modules" \ + ":engine/attachment Subset used for detecting valid page contexts for use in text editing " \ + ":engine/device-detect Subset used for device-detection " \ + ":engine/dom-utils A common subset of function used for DOM calculations, layout, etc" \ + ":engine/events Specialized classes utilized to support KMW API events" \ + ":engine/element-wrappers Subset used to integrate with website elements" \ + ":engine/gestures The gesture-recognition engine used by KMW's OSK" \ + ":engine/main Builds all common code used by KMW's app/-level targets" \ + ":engine/osk Builds the Web OSK module" \ + ":engine/package-cache Subset used to collate keyboards and request them from the cloud" \ + ":engine/paths Subset used to configure KMW" \ + ":samples Builds all needed resources for the KMW sample-page set" \ + "--ci+ Set to utilize CI-based test configurations & reporting." # Possible TODO? -# "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $(builder_term build:embed) and $(builder_term build:web)" \ +# "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $DOC_BUILD_EMBED_WEB" \ builder_describe_outputs \ - configure /node_modules \ - configure:embed /node_modules \ - configure:engine /node_modules \ - configure:web /node_modules \ - configure:ui /node_modules \ - configure:samples /node_modules \ - configure:tools /node_modules \ - build:embed /web/build/app/embed/release/keyman.js \ - build:engine /web/build/engine/main/obj/keymanweb.js \ - build:web /web/build/app/web/release/keymanweb.js \ - build:ui /web/build/app/ui/release/kmwuibutton.js \ - build:samples /web/$PREDICTIVE_TEXT_OUTPUT \ - build:tools /web/build/tools/building/sourcemap-root/index.js + configure /node_modules builder_parse "$@" -#### Build utility methods + definitions #### - -# Build products for each main target. -EMBED_TARGETS=( "keyman.js" ) -WEB_TARGETS=( "keymanweb.js" ) -UI_TARGETS=( "kmwuibutton.js" "kmwuifloat.js" "kmwuitoggle.js" "kmwuitoolbar.js" ) - -: ${CLOSURECOMPILERPATH:=../node_modules/google-closure-compiler-java} -: ${JAVA:=java} - -minifier="$CLOSURECOMPILERPATH/compiler.jar" - -# We'd love to add the argument --source_map_include_content for distribution in the future, -# but Closure doesn't include the TS sources properly at this time. -# -# `checkTypes` is blocked b/c TypeScript can perform our type checking... and it causes an error -# with TypeScript's `extend` implementation (it doesn't recognize a constructor without manual edits). -# We also get a global `this` warning from the same. -# -# `checkVars` is blocked b/c Closure will otherwise fail on TypeScript namespacing, as each original TS -# source file will redeclare the namespace variable, despite being merged into a single file post-compilation. -# -# `jsDocMissingType` prevents errors on type documentation Closure thinks is missing. TypeScript may not -# have the same requirements, and we trust TypeScript over Closure. -minifier_warnings="--jscomp_error=* --jscomp_off=lintChecks --jscomp_off=unusedLocalVariables --jscomp_off=globalThis --jscomp_off=checkTypes --jscomp_off=checkVars --jscomp_off=jsdocMissingType --jscomp_off=uselessCode --jscomp_off=missingRequire --jscomp_off=strictMissingRequire" - -# We use these to prevent Closure from auto-inserting its own polyfills. Turns out, they can break in the -# WebView used by Android API 19, which our app still supported when written. Unsure if it can still occur -# in Android API 21, our current minimum. -# -# Also, we currently apply all needed polyfills either manually or during TS compilation; we don't need the extra, -# excess code. -minifier_lang_specs="--language_in ECMASCRIPT5 --language_out ECMASCRIPT5" -minifycmd="$JAVA -jar $minifier --compilation_level WHITESPACE_ONLY $minifier_warnings --generate_exports $minifier_lang_specs" - -readonly minifier -readonly minifycmd - -minified_sourcemap_cleaner="build/tools/building/sourcemap-root/index.mjs" - -# Fails the build if a specified file does not exist. -assert_exists ( ) { - if [[ ! -f $1 ]]; then - builder_die "Build failed: expected file ${COLOR_GREY}$1${COLOR_RESET} is missing." - fi -} - -# $1 - base file name -# $2 - output path -# $3 - optimization level -# $4 - extra path info to add to minified sourcemap "sourceRoot" property. -# $5 - additional output wrapper -minify ( ) { - if [ $# -ge 4 ]; then - cleanerOptions="--clean --sourceRoot $4" - else - cleanerOptions="--clean" - fi - - if [ $# -ge 5 ]; then - wrapper=$5 - else - wrapper="%output%" - fi - - local INPUT="$1" - local INPUT_FILE="$(basename $1)" - local INPUT_DIR="$(dirname $1)" - local INPUT_SOURCEMAP="$INPUT_DIR/$INPUT_FILE.map" - local OUTPUT="$2" - local OUTPUT_FILE="$(basename $2)" - local OUTPUT_SOURCEMAP="$(dirname $2)/$OUTPUT_FILE.map" - - # --source_map_location_mapping - maps paths on INPUT source maps for consumption by Closure. - # ../../.. => keymanapp, ../.. => keymanapp/keyman. We have TS root sources on 'keyman'. - $minifycmd --source_map_input "$INPUT|$INPUT_SOURCEMAP" \ - --create_source_map "$OUTPUT_SOURCEMAP" --source_map_include_content \ - --source_map_location_mapping "$INPUT_DIR|../../.." \ - --js "$INPUT" --compilation_level $3 \ - --js_output_file "$OUTPUT" --warning_level VERBOSE --output_wrapper "$wrapper -//# sourceMappingURL=$INPUT_FILE.map" - - # Now to clean the source map. - assert_exists "$OUTPUT" - assert_exists "$OUTPUT_SOURCEMAP" - - # "Clean" the minified output sourcemaps. - node $minified_sourcemap_cleaner "$INPUT_SOURCEMAP" "$OUTPUT_SOURCEMAP" $cleanerOptions -} - -# Copies specified engine resources to the specified target's build output directories. -# -# ### Parameters -# -# * 1: `product` the product's source path under src/ -# * 2: `outputs` an array of resource types to copy over -# -# ### Example -# -# ```bash -# copy_resources app/web osk ui -# ``` -copy_resources ( ) { - local COMPILE_TARGET=$1 - shift - - local RESOURCES_TO_COPY=("$@") - - # We leave out obj here, as it's not a 'release' of any sort and - # thus doesn't need to publish sources or resources. - local CONFIGS=(debug) - - if ! builder_has_option --no-minify; then - CONFIGS+=(release) - fi - - builder_echo - - for CONFIG in "${CONFIGS[@]}"; - do - local CONFIG_OUT_PATH=build/$COMPILE_TARGET/$CONFIG - - builder_echo "Copying resources to $CONFIG_OUT_PATH/src" - - for RESOURCE in "${RESOURCES_TO_COPY[@]}"; - do - mkdir -p "$CONFIG_OUT_PATH/$RESOURCE" - mkdir -p "$CONFIG_OUT_PATH/src/resources/$RESOURCE" - - builder_echo "- src/resources/$RESOURCE/ => $CONFIG_OUT_PATH/$RESOURCE" - cp -Rf "src/resources/$RESOURCE" "$CONFIG_OUT_PATH/" >/dev/null - done - - builder_echo - done -} - -# Copies specified source folders corresponding to the specified target's build -# output directories. -# -# ### Parameters -# -# * 1: `product` the product's source path under src/ -# * 2: `outputs` an array of src/ subfolders to copy over -# -# ### Example -# -# ```bash -# copy_sources app/web engine resources/osk -# ``` -copy_sources ( ) { - local COMPILE_TARGET=$1 - shift - - local SOURCES_TO_COPY=("$@") - - # We leave out obj here, as it's not a 'release' of any sort and - # thus doesn't need to publish sources or resources. - CONFIGS=(debug) - - if ! builder_has_option --no-minify; then - CONFIGS+=(release) - fi - - for CONFIG in "${CONFIGS[@]}"; - do - local CONFIG_OUT_PATH=build/$COMPILE_TARGET/$CONFIG - builder_echo "Copying $COMPILE_TARGET sources to $CONFIG_OUT_PATH/src" - - rm -rf "$CONFIG_OUT_PATH/src" - mkdir -p "$CONFIG_OUT_PATH/src" - echo $VERSION_PATCH > "$CONFIG_OUT_PATH/src/version.txt" - - for SOURCE_FOLDER in "${SOURCES_TO_COPY[@]}"; - do - builder_echo "- src/$SOURCE_FOLDER/ => $CONFIG_OUT_PATH/src/$SOURCE_FOLDER/" - mkdir -p "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER" - cp -Rf "src/$SOURCE_FOLDER/"* "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER/" - done - - builder_echo - done -} - -# Compiles compiled scripts from the first folder specified to the second folder specified. -# -# ### Parameters -# -# * 1: `src` the folder containing scripts to be copied -# * 2: `dst` the destination folder for the copy operation -# * 3: `scripts` an array of filenames for expected compiled scripts -# -# ### Example -# -# ```bash -# compile_and_minify build/app/web/obj build/app/web/debug keymanweb.js -# ``` -copy_outputs ( ) { - local src="$1" - local dst="$2" - - shift - shift - - local BASE_SCRIPTS=("$@") - - mkdir -p "$dst" - - for SCRIPTJS in "${BASE_SCRIPTS[@]}"; - do - cp -Rf "$src/$SCRIPTJS" "$dst/" - cp -Rf "$src/$SCRIPTJS.map" "$dst/" - done -} - -# Compiles all build products corresponding to the specified target. -# -# ### Parameters -# -# * 1: `product` the product's source path under src/ -# -# ### Example -# -# ```bash -# compile app/embed -# ``` -compile ( ) { - local COMPILE_TARGET=$1 - tsc -b src/$COMPILE_TARGET -v || builder_die "Build command tsc -b src/$COMPILE_TARGET -v failed with exit code $?" - builder_echo "$COMPILE_TARGET TypeScript compiled under build/$COMPILE_TARGET/obj" -} - -# Finalizes all build products corresponding to the specified target. -# This should be called after `compile` for all `app/` targets. -# -# ### Parameters -# -# * 1: `product` the product's source path under src/ -# * 2: `outputs` an array of expected output script files for the build -# -# ### Example -# -# ```bash -# compile app/embed -# finalize app/embed keyman.js -# ``` -finalize ( ) { - if [ $# -lt 2 ]; then - builder_die "Scripting error: insufficient argument count!" - fi - - local COMPILE_TARGET=$1 - local COMPILED_INTERMEDIATE_PATH=build/$COMPILE_TARGET/obj - local DEBUG_OUT_PATH=build/$COMPILE_TARGET/debug - local RELEASE_OUT_PATH=build/$COMPILE_TARGET/release - - shift - - local OUTPUT_SCRIPTS=("$@") - - # START: debug output - - mkdir -p "$DEBUG_OUT_PATH" - copy_outputs "$COMPILED_INTERMEDIATE_PATH" "$DEBUG_OUT_PATH" "${OUTPUT_SCRIPTS[@]}" - - builder_echo "Compiled $COMPILE_TARGET debug version saved under $DEBUG_OUT_PATH: ${OUTPUT_SCRIPTS[*]}" - - # START: release output - if ! builder_has_option --no-minify; then - for SCRIPT in "${OUTPUT_SCRIPTS[@]}"; - do - minify "$COMPILED_INTERMEDIATE_PATH/$SCRIPT" "$RELEASE_OUT_PATH/$SCRIPT" SIMPLE_OPTIMIZATIONS - done - - builder_echo "Compiled $COMPILE_TARGET release version saved under $RELEASE_OUT_PATH: ${OUTPUT_SCRIPTS[*]}" - else - # The prior 'release' is now outdated: delete it. - rm -rf "$RELEASE_OUT_PATH" - fi -} - #### Build action definitions #### -if builder_start_action configure; then - verify_npm_setup +##################### TODO: call child action, verify things work as expected! - if ! builder_has_option --no-minify; then - # NPM install is required for the file to be present. - if ! [ -f $minifier ]; - then - builder_die "File $minifier does not exist: have you set the environment variable \$CLOSURECOMPILERPATH?" - fi - fi +# We can run all clean & configure actions at once without much issue. - builder_finish_action success configure -fi +builder_run_child_actions clean ## Clean actions -# Possible issue: there's no clear rule to `clean` the engine, which is auto-built -# by build:embed and build:web. -# -# Some sort of command to run ONLY for a general `clean` (no target specified) would -# be perfect for that, I think. - -if builder_start_action clean:engine; then - src/engine/build.sh clean - builder_finish_action success clean:engine -fi - -if builder_start_action clean:embed; then - rm -rf build/app/embed - builder_finish_action success clean:embed -fi - -if builder_start_action clean:web; then - rm -rf build/app/web - builder_finish_action success clean:web -fi +###--- Future tie-in: if #8831 gets accepted, uncomment the next two lines. ---### +# # If a full-on general clean was requested, we can nuke the entire build folder. +# builder_run_action clean:project rm -rf ./build -if builder_start_action clean:ui; then - rm -rf build/app/ui - builder_finish_action success clean:ui -fi - -if builder_start_action clean:samples; then - rm -f $PREDICTIVE_TEXT_OUTPUT - - builder_finish_action success clean:samples -fi - -if builder_start_action clean:tools; then - src/tools/build.sh clean - - builder_finish_action success clean:tools -fi +builder_run_child_actions configure ## Build actions -# Adds a simple version 'header' when there's a main engine build product. -if builder_has_action build:embed || \ - builder_has_action build:web || \ - builder_has_action build:ui; then +builder_run_child_actions build:engine/device-detect +builder_run_child_actions build:engine/dom-utils +builder_run_child_actions build:engine/element-wrappers +builder_run_child_actions build:engine/events - builder_echo "" - builder_echo purple "Compiling version ${VERSION}" -fi +# Uses engine/dom-utils +builder_run_child_actions build:engine/osk -builder_echo +# Uses engine/element-wrappers +builder_run_child_actions build:engine/attachment +# Uses engine/osk (due to resource-path config interface) +builder_run_child_actions build:engine/paths -if builder_start_action build:engine; then - src/engine/build.sh build +# Uses engine/config (also due to resource-path config interface, but for the +# more complete version of that interface) +builder_run_child_actions build:engine/package-cache - builder_finish_action success build:engine -fi +# Uses engine/paths, engine/device-detect, engine/package-cache, & engine/osk +builder_run_child_actions build:engine/main -if builder_start_action build:embed; then - compile app/embed - finalize app/embed ${EMBED_TARGETS[@]} - - # The embedded version doesn't use UI modules. - copy_resources app/embed osk - copy_sources app/embed app/embed engine resources/osk - - builder_finish_action success build:embed - - # TODO: handle this block somehow. - - # if [ $UPLOAD_EMBED_SENTRY = true ]; then # upload-symbols:embed - # if [ $BUILD_DEBUG_EMBED = true ]; then - # ARTIFACT_FOLDER="release/unminified/embedded" - # pushd $EMBED_OUTPUT_NO_MINI - # else - # ARTIFACT_FOLDER="release/embedded" - # pushd $EMBED_OUTPUTs - # fi - # echo "Uploading to Sentry..." - # npm run sentry-cli -- releases files "$VERSION_GIT_TAG" upload-sourcemaps --strip-common-prefix $ARTIFACT_FOLDER --rewrite --ext js --ext map --ext ts || builder_die "Sentry upload failed." - # echo "Upload successful." - # popd - # fi -fi +# Uses all but engine/element-wrappers and engine/attachment +builder_run_child_actions build:app/webview -### -embed section complete. +# Uses literally everything `engine/` above +builder_run_child_actions build:app/browser -if builder_start_action build:web; then - compile app/web - finalize app/web ${WEB_TARGETS[@]} +# Uses app/browser above for engine type-defs +builder_run_child_actions build:app/ui - # The testing pages need both osk & ui resources in the same place. - copy_resources app/web osk ui - copy_sources app/web app/web engine resources/osk +# Needs both app/browser and app/ui. +builder_run_child_actions build:samples - builder_finish_action success build:web -fi +builder_run_child_actions test -if builder_start_action build:ui; then - compile app/ui - finalize app/ui ${UI_TARGETS[@]} - - copy_resources app/ui ui - copy_sources app/ui app/ui resources/ui +if builder_start_action test; then + TEST_OPTS= + if builder_has_option --ci; then + TEST_OPTS=--ci + fi + ./test.sh $TEST_OPTS - builder_finish_action success build:ui + builder_finish_action success test fi -if builder_start_action build:tools; then - src/tools/build.sh - builder_finish_action success build:tools -fi +### Temporary tie-in for pre-modularization version of the gesture-recognizer -if builder_start_action build:samples; then - # Some test pages actually have build scripts. - ./src/test/manual/embed/android-harness/build.sh # is not yet builder-based. +if builder_start_action build:engine/gestures; then + # Definition of global compile constants - builder_echo "Copying samples & test page resources..." - # Should probably be changed into a build script for the `prediction-ui` test page. - cp "${PREDICTIVE_TEXT_SOURCE}" "${PREDICTIVE_TEXT_OUTPUT}" + GESTURE_RECOGNIZER_BUILD="$KEYMAN_ROOT/common/web/gesture-recognizer/build/." + GESTURE_RECOGNIZER_TARGET="$KEYMAN_ROOT/web/build/engine/gesture-recognizer/" # Copy gesture-recognizer build artifacts into web-space for CI testing if [ -f $GESTURE_RECOGNIZER_TARGET ]; then @@ -500,26 +134,5 @@ if builder_start_action build:samples; then # Which could then have a parallel script for `prediction-mtnt` that downloads + extracts # the current MTNT model. - builder_finish_action success build:samples; + builder_finish_action success build:engine/gestures; fi - -if builder_start_action test:web; then - if builder_has_option --all; then - ./test.sh - else - ./test.sh :engine - fi - - builder_finish_action success test:web -fi - -# TODO: handle the block below somehow. - -# # We can only upload 'web' / 'native' artifacts after ALL are done compiling. -# if [ $UPLOAD_WEB_SENTRY = true ]; then -# pushd $WEB_OUTPUT -# echo "Uploading to Sentry..." -# npm run sentry-cli -- releases files "$VERSION_GIT_TAG" upload-sourcemaps --strip-common-prefix release/web/ --rewrite --ext js --ext map --ext ts || builder_die "Sentry upload failed." -# echo "Upload successful." -# popd -# fi diff --git a/web/ci.sh b/web/ci.sh index 4d9670a9be2fb98d2c4feb500709470d60e68cd6..2f038b90b2ca00e650261424096b7a0e50329101 100755 --- a/web/ci.sh +++ b/web/ci.sh @@ -25,6 +25,7 @@ cd "$THIS_SCRIPT_PATH" S_KEYMAN_COM= builder_describe "Defines and implements the CI build steps for Keyman Engine for Web (KMW)." \ + "@/web/src/tools/building/sourcemap-root prepare:s.keyman.com" \ "build" \ "test Runs all unit tests." \ "post-test Runs post-test cleanup. Should be run even if a prior step fails." \ @@ -38,6 +39,9 @@ builder_parse "$@" #### +TIER=`cat ../TIER.md` +BUILD_NUMBER=`cat ../VERSION.md` + if builder_start_action build; then # Build step: since CI builds start (and should start) from scratch, run the following # three actions: @@ -45,7 +49,9 @@ if builder_start_action build; then # - clean: make extra-sure that no prior build products exist. # - also useful when validating this script on a local dev machine! # - build: then do the ACTUAL build. - ./build.sh configure clean build + # one option: + # - --ci: For app/browser, outputs 'release' config filesize profiling logs + ./build.sh configure clean build --ci builder_finish_action success build fi @@ -53,15 +59,17 @@ fi if builder_start_action test; then # Testing step: run ALL unit tests, including those of the submodules. - # For only top-level build-product tests, specify the :engine target - # For all others, specify only the :libraries target - FLAGS= - + OPTIONS= if ! builder_is_debug_build; then - FLAGS=--ci + OPTIONS=--ci fi - ./test.sh $FLAGS + # No --reporter option exists yet for the headless modules. + + $KEYMAN_ROOT/common/web/keyboard-processor/build.sh test $OPTIONS + $KEYMAN_ROOT/common/web/input-processor/build.sh test $OPTIONS + + ./build.sh test $OPTIONS builder_finish_action success test fi @@ -109,15 +117,17 @@ if builder_start_action prepare:s.keyman.com; then # The main build products are expected to reside at the root of this folder. BASE_PUBLISH_FOLDER="$S_KEYMAN_COM/kmw/engine/$VERSION" + echo "FOLDER: $BASE_PUBLISH_FOLDER" mkdir -p "$BASE_PUBLISH_FOLDER" - cp -Rf build/app/web/release/* "$BASE_PUBLISH_FOLDER" - cp -Rf build/app/ui/release/* "$BASE_PUBLISH_FOLDER" + # s.keyman.com - release-config only. It's notably smaller, thus far more favorable + # for distribution via cloud service. + cp -Rf build/publish/release/* "$BASE_PUBLISH_FOLDER" # Third phase: tweak the sourcemaps # We can use an alt-mode of Web's sourcemap-root tool for this. for sourcemap in "$BASE_PUBLISH_FOLDER/"*.map; do - node build/tools/building/sourcemap-root/index.mjs null "$sourcemap" --sourceRoot "https://s.keyman.com/kmw/engine/$VERSION/src" + node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.mjs" null "$sourcemap" --sourceRoot "https://s.keyman.com/kmw/engine/$VERSION/src" done # Actual construction of the PR will be left to CI-config scripting for now. @@ -156,16 +166,9 @@ if builder_start_action prepare:downloads.keyman.com; then fi fi - pushd build/app/web/release - "${COMPRESS_CMD}" $COMPRESS_ADD ../../../../$ZIP * - cd .. - "${COMPRESS_CMD}" $COMPRESS_ADD ../../../$ZIP debug - popd - - pushd build/app/ui/release + pushd build/publish + # Zip both the 'debug' and 'release' configurations together. "${COMPRESS_CMD}" $COMPRESS_ADD ../../../../$ZIP * - cd .. - "${COMPRESS_CMD}" $COMPRESS_ADD ../../../$ZIP debug popd # --- Second action artifact - the 'static' folder (hosted user testing on downloads.keyman.com) --- diff --git a/web/common.inc.sh b/web/common.inc.sh new file mode 100644 index 0000000000000000000000000000000000000000..96a3c61bbd8eccf782d9330bbfdd2130fa9fa072 --- /dev/null +++ b/web/common.inc.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# + +# Compiles all build products corresponding to the specified target. +# This should be called from the working directory of a child project's +# build script. +# +# ### Parameters +# +# * 1: `product` the product's source path under src/ +# +# ### Example +# +# ```bash +# compile engine/main +# ``` +compile ( ) { + if [ $# -lt 1 ]; then + builder_die "Scripting error: insufficient argument count!" + fi + + local COMPILE_TARGET="$1" + local BUNDLE_FLAG="${2:-}" + + tsc -b "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET" -v + + if [ -f "./build-bundler.js" ]; then + node "./build-bundler.js" "$BUNDLE_FLAG" + + # So... tsc does declaration-bundling on its own pretty well, at least for local development. + tsc --emitDeclarationOnly --outFile "${KEYMAN_ROOT}/web/build/$COMPILE_TARGET/lib/index.d.ts" -p "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET" + fi +} + +_copy_dir_if_exists ( ) { + local SRC=$1 + local DST=$2 + + if [ -d "$SRC" ]; then + cp -rf "$SRC/." "$DST" + fi +} + +# Copies top-level build artifacts into common 'debug' and 'release' config folders +# for use in publishing. +prepare ( ) { + local CHILD_BUILD_ROOT="$KEYMAN_ROOT/web/build/app" + local PUBLISH_BUILD_ROOT="$KEYMAN_ROOT/web/build/publish" + + mkdir -p "$PUBLISH_BUILD_ROOT/debug" + mkdir -p "$PUBLISH_BUILD_ROOT/release" + + _copy_dir_if_exists "$CHILD_BUILD_ROOT/browser/debug" "$PUBLISH_BUILD_ROOT/debug" + _copy_dir_if_exists "$CHILD_BUILD_ROOT/browser/release" "$PUBLISH_BUILD_ROOT/release" + + _copy_dir_if_exists "$CHILD_BUILD_ROOT/resources" "$PUBLISH_BUILD_ROOT/debug" + _copy_dir_if_exists "$CHILD_BUILD_ROOT/resources" "$PUBLISH_BUILD_ROOT/release" + + _copy_dir_if_exists "$CHILD_BUILD_ROOT/ui/debug" "$PUBLISH_BUILD_ROOT/debug" + _copy_dir_if_exists "$CHILD_BUILD_ROOT/ui/release" "$PUBLISH_BUILD_ROOT/release" +} + +# Runs all headless tests corresponding to the specified target. +# This should be called from the working directory of a child project's +# build script. +# +# ### Parameters +# +# * 1: `product` the folder under src/test/auto/headless containing the +# child project's tests +# +# ### Example +# +# ```bash +# # from engine/osk +# test-headless osk +# ``` +test-headless ( ) { + TEST_OPTS= + if builder_has_option --ci; then + TEST_OPTS="--reporter mocha-teamcity-reporter" + fi + + mocha --recursive "${KEYMAN_ROOT}/web/src/test/auto/headless/$1" $TEST_OPTS +} \ No newline at end of file diff --git a/web/namespaced-build.sh b/web/namespaced-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a426e84d72a596baef69034325cc1e2ad44462d --- /dev/null +++ b/web/namespaced-build.sh @@ -0,0 +1,579 @@ +#!/usr/bin/env bash +# +# Compiled the Keyman Engine for Web's old namespaced version +# Will be removed by the end of the Web modularization work. +# + +# set -x +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# This script runs from its own folder +cd "$THIS_SCRIPT_PATH" + +# ################################ Main script ################################ + +# Definition of global compile constants + +UI="app/ui" +WEB="app/web" +EMBEDDED="app/embed" + +BUILD_BASE="build" + +DEBUG="debug" +RELEASE="release" +INTERMEDIATE="obj" + +# Composites and outputs the output path corresponding to the build configuration +# specified by the parameters. +# +# ### Parameters +# +# * 1: - build product (app/embed, app/web, app/ui, engine) +# * 2: (optional) - build stage / config (obj, debug, release) +# +# ### Example +# +# ```bash +# cp index.js "$(output_path app/web debug)/index.js" +# ``` +# +# The block above would copy index.js into the build output folder for app/web's debug +# product. +# +# ``` bash +# rm -rf "$(output_path app/web)" +# ``` +# +# The block above is useful for deleting all app/web build products as part of a `clean` +# action. +# +# ### Other Notes +# +# In the future, we may opt to move $INTERMEDIATE stuff underneath both $DEBUG and $RELEASE, +# making it a third param. This is currently unclear, but if so, we'd do +# $DEBUG/$INTERMEDIATE and $RELEASE/$INTERMEDIATE via a third argument. +output_path ( ) { + if [ $# -lt 1 ]; then + builder_die "Insufficient argument count!" + elif [ $# -eq 1 ]; then + # Used by clean: actions + echo "$BUILD_BASE/$1" + else + echo "$BUILD_BASE/$1/$2" + fi +} + +SOURCE="src" + +SENTRY_RELEASE_VERSION="release@$VERSION_WITH_TAG" + +# Ensures that we rely first upon the local npm-based install of Typescript. +# (Facilitates automated setup for build agents.) +PATH="../node_modules/.bin:$PATH" + +compiler="npm run tsc --" +compilecmd="$compiler" + +PREDICTIVE_TEXT_SOURCE="../common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js" +PREDICTIVE_TEXT_OUTPUT="src/test/manual/web/prediction-ui/simple-en-trie.js" + +builder_describe "Builds Keyman Engine for Web (KMW)." \ + "@../common/web/keyman-version build" \ + "@../common/web/input-processor build" \ + "@src/tools/building/sourcemap-root build" \ + "clean" \ + "configure" \ + "build" \ + "test Runs unit tests. Only $(builder_term test:web) is currently defined" \ + ":embed Builds the configuration of KMW used within the Keyman mobile apps" \ + ":engine Builds all common code used by other targets" \ + ":web Builds the website-oriented configuration of Keyman Engine for Web" \ + ":ui Builds the desktop UI modules used by the $(builder_term :web) build product" \ + ":samples Builds only sample & test pages found under src/samples and src/test" \ + ":tools Builds related development + unit-test resources" \ + "--no-minify Skips any minification steps in the build" \ + "--all Sets action to run on KMW's submodules as well if appropriate (actions - $(builder_term test))" + +# Possible TODO? +# "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $(builder_term build:embed) and $(builder_term build:web)" \ + +builder_describe_outputs \ + configure ../node_modules \ + configure:embed ../node_modules \ + configure:engine ../node_modules \ + configure:web ../node_modules \ + configure:ui ../node_modules \ + configure:samples ../node_modules \ + configure:tools ../node_modules \ + build:embed $(output_path $EMBEDDED $RELEASE)/keyman.js \ + build:web $(output_path $WEB $RELEASE)/keymanweb.js \ + build:ui $(output_path $UI $RELEASE)/kmwuibutton.js \ + build:samples $PREDICTIVE_TEXT_OUTPUT +# Deliberately excluding build:tools b/c its script provides the definitions. + +builder_parse "$@" + +#### Build utility methods + definitions #### + +# Build products for each main target. +EMBED_TARGETS=( "keyman.js" ) +WEB_TARGETS=( "keymanweb.js" ) +UI_TARGETS=( "kmwuibutton.js" "kmwuifloat.js" "kmwuitoggle.js" "kmwuitoolbar.js" ) + +: ${CLOSURECOMPILERPATH:=../node_modules/google-closure-compiler-java} +: ${JAVA:=java} + +minifier="$CLOSURECOMPILERPATH/compiler.jar" + +# We'd love to add the argument --source_map_include_content for distribution in the future, +# but Closure doesn't include the TS sources properly at this time. +# +# `checkTypes` is blocked b/c TypeScript can perform our type checking... and it causes an error +# with TypeScript's `extend` implementation (it doesn't recognize a constructor without manual edits). +# We also get a global `this` warning from the same. +# +# `checkVars` is blocked b/c Closure will otherwise fail on TypeScript namespacing, as each original TS +# source file will redeclare the namespace variable, despite being merged into a single file post-compilation. +# +# `jsDocMissingType` prevents errors on type documentation Closure thinks is missing. TypeScript may not +# have the same requirements, and we trust TypeScript over Closure. +minifier_warnings="--jscomp_error=* --jscomp_off=lintChecks --jscomp_off=unusedLocalVariables --jscomp_off=globalThis --jscomp_off=checkTypes --jscomp_off=checkVars --jscomp_off=jsdocMissingType --jscomp_off=uselessCode --jscomp_off=missingRequire --jscomp_off=strictMissingRequire" + +# We use these to prevent Closure from auto-inserting its own polyfills. Turns out, they can break in the +# WebView used by Android API 19, which our app still supported when written. Unsure if it can still occur +# in Android API 21, our current minimum. +# +# Also, we currently apply all needed polyfills either manually or during TS compilation; we don't need the extra, +# excess code. +minifier_lang_specs="--language_in ECMASCRIPT5 --language_out ECMASCRIPT5" +minifycmd="$JAVA -jar $minifier --compilation_level WHITESPACE_ONLY $minifier_warnings --generate_exports $minifier_lang_specs" + +readonly minifier +readonly minifycmd + +minified_sourcemap_cleaner="build/tools/building/sourcemap-root/index.mjs" + +# Fails the build if a specified file does not exist. +assert_exists ( ) { + if ! [ -f $1 ]; then + echo "Build failed: expected file ${COLOR_GREY}$1${COLOR_RESET} is missing." + exit 1 + fi +} + +# $1 - base file name +# $2 - output path +# $3 - optimization level +# $4 - extra path info to add to minified sourcemap "sourceRoot" property. +# $5 - additional output wrapper +minify ( ) { + if [ $# -ge 4 ]; then + cleanerOptions="--clean --sourceRoot $4" + else + cleanerOptions="--clean" + fi + + if [ $# -ge 5 ]; then + wrapper=$5 + else + wrapper="%output%" + fi + + local INPUT="$1" + local INPUT_FILE="$(basename $1)" + local INPUT_DIR="$(dirname $1)" + local INPUT_SOURCEMAP="$INPUT_DIR/$INPUT_FILE.map" + local OUTPUT="$2" + local OUTPUT_FILE="$(basename $2)" + local OUTPUT_SOURCEMAP="$(dirname $2)/$OUTPUT_FILE.map" + + # --source_map_location_mapping - maps paths on INPUT source maps for consumption by Closure. + # ../../.. => keymanapp, ../.. => keymanapp/keyman. We have TS root sources on 'keyman'. + $minifycmd --source_map_input "$INPUT|$INPUT_SOURCEMAP" \ + --create_source_map "$OUTPUT_SOURCEMAP" --source_map_include_content \ + --source_map_location_mapping "$INPUT_DIR|../../.." \ + --js "$INPUT" --compilation_level $3 \ + --js_output_file "$OUTPUT" --warning_level VERBOSE --output_wrapper "$wrapper +//# sourceMappingURL=$INPUT_FILE.map" + + # Now to clean the source map. + assert_exists "$OUTPUT" + assert_exists "$OUTPUT_SOURCEMAP" + + # "Clean" the minified output sourcemaps. + node $minified_sourcemap_cleaner "$INPUT_SOURCEMAP" "$OUTPUT_SOURCEMAP" $cleanerOptions +} + +# Copies specified engine resources to the specified target's build output directories. +# +# ### Parameters +# +# * 1: `product` the product's source path under src/ +# * 2: `outputs` an array of resource types to copy over +# +# ### Example +# +# ```bash +# copy_resources app/web osk ui +# ``` +copy_resources ( ) { + local COMPILE_TARGET=$1 + shift + + local RESOURCES_TO_COPY=("$@") + + # We leave out $INTERMEDIATE here, as it's not a 'release' of any sort and + # thus doesn't need to publish sources or resources. + local CONFIGS=($DEBUG) + + if ! builder_has_option --skip-minify; then + CONFIGS+=($RELEASE) + fi + + echo + + for CONFIG in "${CONFIGS[@]}"; + do + local CONFIG_OUT_PATH="$(output_path $COMPILE_TARGET $CONFIG)" + + echo Copying resources to $CONFIG_OUT_PATH/src + + for RESOURCE in "${RESOURCES_TO_COPY[@]}"; + do + mkdir -p "$CONFIG_OUT_PATH/$RESOURCE" + mkdir -p "$CONFIG_OUT_PATH/src/resources/$RESOURCE" + + echo "- $SOURCE/resources/$RESOURCE/ => $CONFIG_OUT_PATH/$RESOURCE" + cp -Rf "$SOURCE/resources/$RESOURCE" "$CONFIG_OUT_PATH/" >/dev/null + done + + echo + done +} + +# Copies specified source folders corresponding to the specified target's build +# output directories. +# +# ### Parameters +# +# * 1: `product` the product's source path under src/ +# * 2: `outputs` an array of src/ subfolders to copy over +# +# ### Example +# +# ```bash +# copy_sources app/web engine resources/osk +# ``` +copy_sources ( ) { + local COMPILE_TARGET=$1 + shift + + local SOURCES_TO_COPY=("$@") + + # We leave out $INTERMEDIATE here, as it's not a 'release' of any sort and + # thus doesn't need to publish sources or resources. + CONFIGS=($DEBUG) + + if ! builder_has_option --skip-minify; then + CONFIGS+=($RELEASE) + fi + + for CONFIG in "${CONFIGS[@]}"; + do + CONFIG_OUT_PATH="$(output_path $COMPILE_TARGET $CONFIG)" + echo Copying $COMPILE_TARGET sources to $CONFIG_OUT_PATH/src + + rm -rf "$CONFIG_OUT_PATH/src" + mkdir -p "$CONFIG_OUT_PATH/src" + echo $VERSION_PATCH > "$CONFIG_OUT_PATH/src/version.txt" + + for SOURCE_FOLDER in "${SOURCES_TO_COPY[@]}"; + do + echo "- $SOURCE/$SOURCE_FOLDER/ => $CONFIG_OUT_PATH/src/$SOURCE_FOLDER/" + mkdir -p "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER" + cp -Rf "$SOURCE/$SOURCE_FOLDER/"* "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER/" + done + + echo + done +} + +# Compiles compiled scripts from the first folder specified to the second folder specified. +# +# ### Parameters +# +# * 1: `src` the folder containing scripts to be copied +# * 2: `dst` the destination folder for the copy operation +# * 3: `scripts` an array of filenames for expected compiled scripts +# +# ### Example +# +# ```bash +# compile_and_minify build/app/web/obj build/app/web/debug keymanweb.js +# ``` +copy_outputs ( ) { + local src="$1" + local dst="$2" + + shift + shift + + local BASE_SCRIPTS=("$@") + + mkdir -p "$dst" + + for SCRIPTJS in "${BASE_SCRIPTS[@]}"; + do + cp -Rf "$src/$SCRIPTJS" "$dst/" + cp -Rf "$src/$SCRIPTJS.map" "$dst/" + done +} + +# Compiles all build products corresponding to the specified target. +# +# ### Parameters +# +# * 1: `product` the product's source path under src/ +# +# ### Example +# +# ```bash +# compile app/embed +# ``` +compile ( ) { + if [ $# -lt 1 ]; then + fail "Scripting error: insufficient argument count!" + fi + + local COMPILE_TARGET=$1 + local COMPILED_INTERMEDIATE_PATH="$(output_path $COMPILE_TARGET $INTERMEDIATE)" + + $compilecmd -b src/$COMPILE_TARGET -v + + echo $COMPILE_TARGET TypeScript compiled under $COMPILED_INTERMEDIATE_PATH +} + +# Finalizes all build products corresponding to the specified target. +# This should be called after `compile` for all `app/` targets. +# +# ### Parameters +# +# * 1: `product` the product's source path under src/ +# * 2: `outputs` an array of expected output script files for the build +# +# ### Example +# +# ```bash +# compile app/embed +# finalize app/embed keyman.js +# ``` +finalize ( ) { + if [ $# -lt 2 ]; then + fail "Scripting error: insufficient argument count!" + fi + + local COMPILE_TARGET=$1 + local COMPILED_INTERMEDIATE_PATH="$(output_path $COMPILE_TARGET $INTERMEDIATE)" + local DEBUG_OUT_PATH="$(output_path $COMPILE_TARGET $DEBUG)" + local RELEASE_OUT_PATH="$(output_path $COMPILE_TARGET $RELEASE)" + + shift + + local OUTPUT_SCRIPTS=("$@") + + # START: debug output + + mkdir -p "$DEBUG_OUT_PATH" + copy_outputs "$COMPILED_INTERMEDIATE_PATH" "$DEBUG_OUT_PATH" "${OUTPUT_SCRIPTS[@]}" + + echo Compiled $COMPILE_TARGET debug version saved under $DEBUG_OUT_PATH: ${OUTPUT_SCRIPTS[*]} + + # START: release output + if ! builder_has_option --skip-minify; then + for SCRIPT in "${OUTPUT_SCRIPTS[@]}"; + do + minify "$COMPILED_INTERMEDIATE_PATH/$SCRIPT" "$RELEASE_OUT_PATH/$SCRIPT" SIMPLE_OPTIMIZATIONS + done + + echo Compiled $COMPILE_TARGET release version saved under $RELEASE_OUT_PATH: ${OUTPUT_SCRIPTS[*]} + else + # The prior 'release' is now outdated: delete it. + rm -rf "$RELEASE_OUT_PATH" + fi +} + +#### Build action definitions #### + +if builder_start_action configure; then + verify_npm_setup + + if ! builder_has_option --no-minify; then + # NPM install is required for the file to be present. + if ! [ -f $minifier ]; + then + echo File $minifier does not exist: have you set the environment variable \$CLOSURECOMPILERPATH? + exit 1 + fi + fi + + builder_finish_action success configure +fi + +## Clean actions + +# Possible issue: there's no clear rule to `clean` the engine, which is auto-built +# by build:embed and build:web. +# +# Some sort of command to run ONLY for a general `clean` (no target specified) would +# be perfect for that, I think. + +if builder_start_action clean:engine; then + src/engine/build.sh clean + builder_finish_action success clean:engine +fi + +if builder_start_action clean:embed; then + rm -rf "$(output_path $EMBEDDED)" + builder_finish_action success clean:embed +fi + +if builder_start_action clean:web; then + rm -rf "$(output_path $WEB)" + builder_finish_action success clean:web +fi + +if builder_start_action clean:ui; then + rm -rf "$(output_path $UI)" + builder_finish_action success clean:ui +fi + +if builder_start_action clean:samples; then + rm -f $PREDICTIVE_TEXT_OUTPUT + + builder_finish_action success clean:samples +fi + +if builder_start_action clean:tools; then + src/tools/build.sh clean + + builder_finish_action success clean:tools +fi + +## Build actions + +# Adds a simple version 'header' when there's a main engine build product. +if builder_has_action build:embed || \ + builder_has_action build:web || \ + builder_has_action build:ui; then + + echo "" + echo "${COLOR_PURPLE}Compiling version ${VERSION}${COLOR_RESET}" +fi + +echo "" + + +if builder_start_action build:engine; then + src/engine/build.sh build + + builder_finish_action success build:engine +fi + +if builder_start_action build:embed; then + compile $EMBEDDED + finalize $EMBEDDED ${EMBED_TARGETS[@]} + + # The embedded version doesn't use UI modules. + copy_resources $EMBEDDED osk + copy_sources $EMBEDDED app/embed engine resources/osk + + builder_finish_action success build:embed + + # TODO: handle this block somehow. + + # if [ $UPLOAD_EMBED_SENTRY = true ]; then # upload-symbols:embed + # if [ $BUILD_DEBUG_EMBED = true ]; then + # ARTIFACT_FOLDER="release/unminified/embedded" + # pushd $EMBED_OUTPUT_NO_MINI + # else + # ARTIFACT_FOLDER="release/embedded" + # pushd $EMBED_OUTPUTs + # fi + # echo "Uploading to Sentry..." + # npm run sentry-cli -- releases files "$SENTRY_RELEASE_VERSION" upload-sourcemaps --strip-common-prefix $ARTIFACT_FOLDER --rewrite --ext js --ext map --ext ts || fail "Sentry upload failed." + # echo "Upload successful." + # popd + # fi +fi + +### -embed section complete. + +if builder_start_action build:web; then + compile $WEB + finalize $WEB ${WEB_TARGETS[@]} + + # The testing pages need both osk & ui resources in the same place. + copy_resources $WEB osk ui + copy_sources $WEB app/web engine resources/osk + + builder_finish_action success build:web +fi + +if builder_start_action build:ui; then + compile $UI + finalize $UI ${UI_TARGETS[@]} + + copy_resources $UI ui + copy_sources $UI app/ui resources/ui + + builder_finish_action success build:ui +fi + +if builder_start_action build:tools; then + src/tools/build.sh + builder_finish_action success build:tools +fi + +if builder_start_action build:samples; then + # Some test pages actually have build scripts. + ./src/test/manual/embed/android-harness/build.sh # is not yet builder-based. + + echo "Copying samples & test page resources..." + # Should probably be changed into a build script for the `prediction-ui` test page. + cp "${PREDICTIVE_TEXT_SOURCE}" "${PREDICTIVE_TEXT_OUTPUT}" + + # Which could then have a parallel script for `prediction-mtnt` that downloads + extracts + # the current MTNT model. + + builder_finish_action success build:samples; +fi + +if builder_start_action test:web; then + if builder_has_option --all; then + ./test.sh + else + ./test.sh :engine + fi + + builder_finish_action success test:web +fi + +# TODO: handle the block below somehow. + +# # We can only upload 'web' / 'native' artifacts after ALL are done compiling. +# if [ $UPLOAD_WEB_SENTRY = true ]; then +# pushd $WEB_OUTPUT +# echo "Uploading to Sentry..." +# npm run sentry-cli -- releases files "$SENTRY_RELEASE_VERSION" upload-sourcemaps --strip-common-prefix release/web/ --rewrite --ext js --ext map --ext ts || fail "Sentry upload failed." +# echo "Upload successful." +# popd +# fi diff --git a/web/package.json b/web/package.json index e21450c22dac8db94f1fc84d23e4a59bd03d5fe5..21b17525a6701bd4ba8336dbfa613a0af00b08f3 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,57 @@ { "name": "keyman", "description": "Facilitates text input in any language.", + "exports": { + "./app/browser": { + "types": "./build/app/browser/obj/test-index.d.js", + "import": "./build/app/browser/obj/test-index.js" + }, + "./engine/attachment": { + "types": "./build/engine/attachment/obj/index.d.ts", + "import": "./build/engine/attachment/obj/index.js" + }, + "./engine/paths": { + "types": "./build/engine/paths/obj/index.d.ts", + "import": "./build/engine/paths/obj/index.js" + }, + "./engine/device-detect": { + "types": "./build/engine/device-detect/obj/index.d.ts", + "import": "./build/engine/device-detect/obj/index.js" + }, + "./engine/dom-utils": { + "types": "./build/engine/dom-utils/obj/index.d.ts", + "import": "./build/engine/dom-utils/obj/index.js" + }, + "./engine/element-wrappers": { + "types": "./build/engine/element-wrappers/obj/index.d.ts", + "import": "./build/engine/element-wrappers/obj/index.js" + }, + "./engine/events": { + "types": "./build/engine/events/obj/index.d.ts", + "import": "./build/engine/events/obj/index.js" + }, + "./engine/package-cache": { + "types": "./build/engine/package-cache/obj/index.d.ts", + "import": "./build/engine/package-cache/obj/index.js", + "require": "./build/engine/package-cache/obj/index.js" + }, + "./engine/package-cache/dom-requester": { + "types": "./build/engine/package-cache/obj/domCloudRequester.d.ts", + "import": "./build/engine/package-cache/obj/domCloudRequester.js" + }, + "./engine/package-cache/node-requester": { + "types": "./build/engine/package-cache/obj/nodeCloudRequester.d.ts", + "import": "./build/engine/package-cache/obj/nodeCloudRequester.js" + }, + "./engine/main": { + "types": "./build/engine/main/obj/index.d.ts", + "import": "./build/engine/main/obj/index.js" + }, + "./engine/osk": { + "types": "./build/engine/osk/obj/index.d.ts", + "import": "./build/engine/osk/obj/index.js" + } + }, "repository": { "type": "git", "url": "git+https://github.com/keymanapp/keyman.git" @@ -21,7 +72,6 @@ "@keymanapp/web-sentry-manager": "*", "@sentry/cli": "2.2.0", "chai": "^4.3.4", - "google-closure-compiler-java": "^20200224.0.0", "karma": "^6.4.1", "karma-browserstack-launcher": "^1.6.0", "karma-chai": "^0.1.0", @@ -56,8 +106,11 @@ "@keymanapp/lexical-model-layer": "*", "@keymanapp/models-types": "*", "@keymanapp/recorder-core": "*", + "@keymanapp/tslib": "*", "@keymanapp/web-utils": "*", "@types/node": "^11.9.4", - "eventemitter3": "^4.0.0" - } + "eventemitter3": "^4.0.0", + "tslib": "^2.5.2" + }, + "type": "module" } diff --git a/web/src/app/browser/README.md b/web/src/app/browser/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a911dc6736c7f2c3ac4404a314b4a9dbfebeea60 --- /dev/null +++ b/web/src/app/browser/README.md @@ -0,0 +1 @@ +This subproject holds the new, modularized form of the old namespaced `app/web` subproject. \ No newline at end of file diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..cc81f7c402c301ae3739f5c6e928fca8d7603d7b --- /dev/null +++ b/web/src/app/browser/build-bundler.js @@ -0,0 +1,111 @@ +/* + * Note: while this file is not meant to exist long-term, it provides a nice + * low-level proof-of-concept for esbuild bundling of the various Web submodules. + * + * Add some extra code at the end of src/index.ts and run it to verify successful bundling! + */ + +import esbuild from 'esbuild'; +import fs from 'fs'; +import { determineNeededDowncompileHelpers, buildTslibTreeshaker } from '@keymanapp/tslib/esbuild-tools'; + +let EMIT_FILESIZE_PROFILE = false; + +if(process.argv.length > 2) { + for(let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + + switch(arg) { + case '': + break; + case '--ci': + EMIT_FILESIZE_PROFILE=true + break; + // May add other options if desired in the future. + default: + console.error("Invalid command-line option set for script; only --ci is permitted."); + process.exit(1); + } + } +} + +/* + * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - + * the `@class` emit comment-annotation is designed to facilitate tree-shaking for ES5-targeted + * down-level emits. `esbuild` doesn't look for it by default... but we can override that with + * this plugin. + */ +let es5ClassAnnotationAsPurePlugin = { + name: '@class -> __PURE__', + setup(build) { + build.onLoad({filter: /\.js$/ }, async (args) => { + let source = await fs.promises.readFile(args.path, 'utf8'); + return { + // Marks any classes compiled by TS (as per the /** @class */ annotation) + // as __PURE__ in order to facilitate tree-shaking. + contents: source.replace('/** @class */', '/* @__PURE__ */ /** @class */'), + loader: 'js' + } + }); + } +} + +const commonConfig = { + alias: { + 'tslib': '@keymanapp/tslib' + }, + bundle: true, + sourcemap: true, + format: "iife", + nodePaths: ['../../../../node_modules'], + entryPoints: { + 'index': '../../../build/app/browser/obj/debug-main.js', + }, + outfile: '../../../build/app/browser/debug/keymanweb.js', + plugins: [ es5ClassAnnotationAsPurePlugin ], + target: "es5", + treeShaking: true, + tsconfig: './tsconfig.json' +}; + +// Prepare the needed setup for `tslib` treeshaking. +const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig, /worker-main\.wrapped(?:\.min)?\.js/); +commonConfig.plugins = [buildTslibTreeshaker(unusedHelpers), ...commonConfig.plugins]; + +// And now... do the actual builds. +await esbuild.build(commonConfig); + +let result = await esbuild.build({ + ...commonConfig, + minifyWhitespace: true, + minifySyntax: true, + minifyIdentifiers: true, + format: "iife", + nodePaths: ['../../../../node_modules'], + entryPoints: { + 'index': '../../../build/app/browser/obj/release-main.js', + }, + outfile: '../../../build/app/browser/release/keymanweb.js', + // Enables source-file output size profiling! + metafile: true +}); + +let filesizeProfile = await esbuild.analyzeMetafile(result.metafile, { verbose: true }); +fs.writeFileSync('../../../build/app/browser/filesize-profile.log', ` +// Minified Keyman Engine for Web ('app/browser' target), filesize profile +${filesizeProfile} +`); +if(EMIT_FILESIZE_PROFILE) { + // Profiles the sourcecode! + console.log(filesizeProfile); +} + +await esbuild.build({ + ...commonConfig, + format: "esm", + entryPoints: { + 'index': '../../../build/app/browser/obj/test-index.js', + }, + outfile: '../../../build/app/browser/lib/index.mjs', + tsconfig: './tsconfig.json' +}); \ No newline at end of file diff --git a/web/src/app/browser/build.sh b/web/src/app/browser/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..2cbc51fae5a070028fe3eeba113b5ca88756068d --- /dev/null +++ b/web/src/app/browser/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +SUBPROJECT_NAME=app/browser +. "$KEYMAN_ROOT/web/common.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# This script runs from its own folder +cd "$THIS_SCRIPT_PATH" + +# ################################ Main script ################################ + +builder_describe "Builds the Keyman Engine for Web's website-integrating version for use in non-puppeted browsers." \ + "@/web/src/engine/attachment build" \ + "@/web/src/engine/main build" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + "--ci+ Set to utilize CI-based test configurations & reporting." + +# Possible TODO?s +# "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $DOC_BUILD_EMBED_WEB" \ + +builder_parse "$@" + +config=release +if builder_is_debug_build; then + config=debug +fi + +builder_describe_outputs \ + configure /node_modules \ + build /web/build/$SUBPROJECT_NAME/$config/keymanweb.js + +#### Build action definitions #### + +compile_and_copy() { + local COMPILE_FLAGS= + if builder_has_option --ci; then + COMPILE_FLAGS=--ci + fi + compile $SUBPROJECT_NAME $COMPILE_FLAGS + + mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk" + cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/" + + # Update the build/publish copy of our build artifacts + prepare + + local PROFILE_DEST="$KEYMAN_ROOT/web/build/profiling/" + mkdir -p "$PROFILE_DEST" + cp "$KEYMAN_ROOT/web/build/app/browser/filesize-profile.log" "$PROFILE_DEST/web-engine-filesize.log" + cp "$KEYMAN_ROOT/common/web/lm-worker/build/filesize-profile.log" "$PROFILE_DEST/lm-worker-filesize.log" +} + +builder_run_action configure verify_npm_setup +builder_run_action clean rm -rf "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME" +builder_run_action build compile_and_copy + +# No headless tests for this child project. Currently, DOM-based unit & +# integrated tests are run solely by the top-level $KEYMAN_ROOT/web project. \ No newline at end of file diff --git a/web/src/app/browser/src/beepHandler.ts b/web/src/app/browser/src/beepHandler.ts new file mode 100644 index 0000000000000000000000000000000000000000..5fae87425777442d7faee91df9b092672d65222d --- /dev/null +++ b/web/src/app/browser/src/beepHandler.ts @@ -0,0 +1,87 @@ +import { type KeyboardInterface } from '@keymanapp/keyboard-processor'; +import { DesignIFrame, OutputTarget } from 'keyman/engine/element-wrappers'; + +// Utility object used to handle beep (keyboard error response) operations. +class BeepData { + e: HTMLElement; + c: string; + + constructor(e: HTMLElement) { + this.e = e; + this.c = e.style.backgroundColor; + } + + reset(): void { + this.e.style.backgroundColor = this.c; + } +} + +export class BeepHandler { + readonly keyboardInterface: KeyboardInterface; + + constructor(keyboardInterface: KeyboardInterface) { + this.keyboardInterface = keyboardInterface; + } + + _BeepObjects: BeepData[] = []; // BeepObjects - maintains a list of active 'beep' visual feedback elements + _BeepTimeout: number = 0; // BeepTimeout - a flag indicating if there is an active 'beep'. + // Set to 1 if there is an active 'beep', otherwise leave as '0'. + /** + * Function beep KB (DOM-side implementation) + * Scope Public + * @param {Object} Pelem element to flash + * Description Flash body as substitute for audible beep; notify embedded device to vibrate + */ + beep(outputTarget: OutputTarget) { + if(!(outputTarget instanceof OutputTarget)) { + return; + } + + // All code after this point is DOM-based, triggered by the beep. + var Pelem: HTMLElement = outputTarget.getElement(); + if(outputTarget instanceof DesignIFrame) { + Pelem = outputTarget.docRoot; // I1446 - beep sometimes fails to flash when using OSK and rich control + } + + if(!Pelem) { + return; // There's no way to signal a 'beep' to null, so just cut everything short. + } + + if(!Pelem.style || typeof(Pelem.style.backgroundColor)=='undefined') { + return; + } + + for(var Lbo=0; Lbo { + this.keyboardInterface.resetContextCache(); + + var Lbo; + this._BeepTimeout = 0; + for(Lbo=0;Lbo; + + initialize(options: Required) { + if(this._options) { + // Preserve old options, but replace with any newly-set ones if specified. + // If specified, even as 'undefined' or 'null', it will still override. + this._options = {...this._options, ...options}; + } else { + this._options = {...options}; + } + super.initialize(options); + this._options = options; + + this._ui = options.ui; + this._attachType = options.attachType; + + whenDocumentReady().then(() => { + if(options.useAlerts && !this.alertHost) { + this._alertHost = new AlertHost(); + } else if(!options.useAlerts && this.alertHost) { + this._alertHost?.shutdown(); + this._alertHost = null; + } + }); + } + + get options() { + return this._options; + } + + get attachType() { + return this._attachType; + } + + get alertHost(): AlertHost | undefined { + return this._alertHost; + } + + set signalUser(host: AlertHost) { + if(!host || host != this.alertHost) { + this.alertHost.shutdown(); + } + + this._alertHost = host; + } + + debugReport(): Record { + const baseReport = super.debugReport(); + baseReport.attachType = this.attachType; + baseReport.ui = this._ui; + baseReport.keymanEngine = 'app/browser'; + + return baseReport; + } + + onRuleFinalization(ruleBehavior: RuleBehavior, outputTarget: OutputTarget) { + // TODO: Patch up to modularized form. But that doesn't exist yet for some of these... + + // If the transform isn't empty, we've changed text - which should produce a 'changed' event in the DOM. + const ruleTransform = ruleBehavior.transcription.transform; + if(!isEmptyTransform(ruleTransform)) { + if(outputTarget instanceof DOMOutputTarget) { + outputTarget.changed = true; + } + } + } +} + +export interface BrowserInitOptionSpec extends InitOptionSpec { + ui?: string; + attachType?: 'auto' | 'manual' | ''; // If blank or undefined, attachType will be assigned to "auto" or "manual" + useAlerts?: boolean; +} + +export const BrowserInitOptionDefaults: Required = { + ui: '', + attachType: '', + useAlerts: true, + ...InitOptionDefaults +} \ No newline at end of file diff --git a/web/src/app/browser/src/context/focusAssistant.ts b/web/src/app/browser/src/context/focusAssistant.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0ae8a6bdb1b7caa8ead25b222cf655fbc5bd470 --- /dev/null +++ b/web/src/app/browser/src/context/focusAssistant.ts @@ -0,0 +1,162 @@ +import EventEmitter from "eventemitter3"; + +/** + * The return object documented for + * https://help.keyman.com/developer/engine/web/16.0/reference/core/getUIState. + * + * As it has long been documented in this format, property names should not be adjusted! + */ +export class FocusStateAPIObject { + /** + * Indicates that KMW is in a state of actively restoring focus to a previously-active element. + */ + activated: boolean; + + /** + * Indicates that KMW is actively maintaining focus on the currently active OutputTarget control + * while some UI element (the OSK, a keyboard-change UI) is the current focus of user-interaction. + */ + activationPending: boolean; + + constructor(pending: boolean, activated: boolean) { + this.activationPending = pending; + this.activated = activated; + } +} + +interface EventMap { + /** + * Called immediately after the `maintainingFocus` flag is cleared. + * @returns + */ + 'maintainingfocusend': () => void; +} + +// Formerly handled under "UIManager". +/** + * This class provides fields and methods useful for assisting context management. Control focus (and + * thus, activation of the corresponding OutputTarget) should not be lost to non-context components of + * KMW, such as the OSK or a keyboard selector. + */ +export class FocusAssistant extends EventEmitter { + private _maintainingFocus: boolean = false; // ActivatingKeymanWebUI - Does the OSK have active focus / an active interaction? + + /** + * Returns `true` only when the active target has an active `forceScroll` method/state, which deliberately + * blurs and then refocuses the same element in order to force a browser-default page scroll to keep the + * element and text-caret visible. + */ + readonly isTargetForcingScroll: () => boolean; + + constructor(isTargetForcingScroll: () => boolean) { + super(); + this.isTargetForcingScroll = isTargetForcingScroll; + } + + /* + * Long-term idea here: about all of the relevant OSK events that would interact with this have "enter" and + * "leave" variants - we could take a stack of `Promise`s. On a `Promise` fulfillment, remove it from the + * stack. When the last one is removed, the focus-maintenance state would end, allowing further events + * to deactivate the active OutputTarget. + */ + + /** + * Indicates that KMW is actively maintaining focus on the currently active OutputTarget control, rather + * than losing focus while some UI element (the OSK, a keyboard-change UI) is the most direct recipient + * of browser focus due to user-interaction - generally, with non-context engine components. + * + * While the flag is active, the context-management system should not deactivate an OutputTarget upon + * its element's loss of focus within the page unless setting a different OutputTarget as active. + * + * TODO: (potential) Future enhancement - this should not be possible to set if there is no currently-active + * context target to maintain. + */ + // Formerly `isActivating`. + public get maintainingFocus(): boolean { + return this._maintainingFocus; + } + + public set maintainingFocus(value: boolean) { + const priorValue = this._maintainingFocus; + this._maintainingFocus = value; + + // Needed to properly update .activeTarget upon loss of maintaining-focus state. + if(priorValue && !value) { + this.emit('maintainingfocusend'); + } + } + + /* + * Long-term idea here: as (aside from OSK title/resize bar interactions) it's always used to actively + * RESTORE focus, taking in a closure to perform during the 'focus restoration state' would make a nice + * design. Something like .performRefocus(closure: () => void): + * - would set this field before calling the closure + * - would unset this field after calling the closure + * - would try-catch to guarantee the 'unset'. + * + * OSK drag handlers should utilize the other field, anyway. + */ + + /** + * Indicates that KMW is in a state of actively restoring focus to a previously-active element. + * This is most commonly utilized whenever a keyboard is newly activated, generally due to + * user interaction to select the new keyboard. + */ + // Formerly `justActivated`. + restoringFocus: boolean = false; // JustActivatedKeymanWebUI - focussing back to control after KeymanWeb UI interaction + + /** + * JH (2023-04-24): given how it's used within the KMW engine, this seems extremely similar in purpose to + * `restoringFocus` - it's set before calling an element's focus method to prevent focus-handlers from causing + * unwanted side-effects. The ONE critical detail: KSF / `saveFocus` will block a single check, not waiting for + * control flow restoration before clearing, where the other matching cases will block 100 (maybe to prevent + * some sort of event softlock?). + * + * So, it's like the `saveFocus` variant immediately clears the flag once checked, while others are intended + * to only clear the flag once control returns to the method that triggered a focus op. + * + * A future refactor should be able to merge the two, though it's worth noting that there are early checks for + * this, but _not_ `restoringFocus`, in the context-management control-blur event handler. So, it's not 100% + * super-straightforward, but a refactor should be manageable all the same. + */ + _IgnoreNextSelChange = 0; + + /** + * Is used as a time-delayed async `restoringFocus` or `maintainingFocus` - could be modeled decently as a Promise. + * Probably more the latter, as it's a touch-OSK interaction like the other `maintainingFocus` cases. + */ + focusing: boolean; + /** + * Manages the time-delay aspect of `focusing` state. + */ + focusTimer: number; + + /** + * Function getUIState + * Scope Public + * @return {Object.} + * Description Return object with activation state of UI: + * activationPending (bool): KMW being activated + * activated (bool): KMW active + */ + getUIState(): FocusStateAPIObject { + return new FocusStateAPIObject(this.maintainingFocus, this.restoringFocus); + } + + /** + * Set or clear the IsActivatingKeymanWebUI flag (exposed function) + * + * @param {(boolean|number)} state Activate (true,false) + */ + setMaintainingFocus(state: boolean) { + this.maintainingFocus = state ? true : false; + } + + setFocusTimer(): void { + this.focusing=true; + + this.focusTimer = window.setTimeout(function() { + this.focusing=false; + }.bind(this), 50) + } +} \ No newline at end of file diff --git a/web/src/app/browser/src/context/pageIntegrationHandlers.ts b/web/src/app/browser/src/context/pageIntegrationHandlers.ts new file mode 100644 index 0000000000000000000000000000000000000000..30b91cb102e654dd46da183407f2176fb389efe8 --- /dev/null +++ b/web/src/app/browser/src/context/pageIntegrationHandlers.ts @@ -0,0 +1,236 @@ +import { DomEventTracker } from 'keyman/engine/events'; + +import KeymanEngine from "../keymanEngine.js"; +import { FocusAssistant } from './focusAssistant.js'; +import { RotationProcessor } from '../utils/rotationProcessor.js'; + +// Note: in the future, it'd probably be best to have an instance per iframe window as +// well as the top-level window. This was not done in or before KMW 16.0 though, so +// we'll leave that out for now within the initial modular form of app/browser KMW in 17.0. +export class PageIntegrationHandlers { + private readonly window: Window; + private readonly engine: KeymanEngine; + private readonly domEventTracker = new DomEventTracker(); + + /** + * Used together with `deactivateOnRelease` to determine the distance of vertical scrolls; + * if sufficiently far at any point, we avoid deactivating the current context when it ends. + */ + private touchY: number; + + /** + * Used together with `touchY` to determine the distance of vertical scrolls; + * if sufficiently far at any point, we avoid deactivating the current context when it ends. + */ + private deactivateOnRelease: boolean; + + /** + * Used on certain browser/OS combinations (e.g. Chrome on Android) to prevent odd behaviors + * that arise when URL bars scroll into view during an ongoing scroll, as this can impede + * proper / smooth positioning of the OSK. (Deactivating the active target also hides the OSK.) + */ + private deactivateOnScroll: boolean; + + /** + * This component should only ever be applied to a base page: we need the ability to add 'scroll + * space' on touch devices so that the OSK doesn't block the bottom. + */ + private mobilePageTrailer: HTMLDivElement; + + private rotationProcessor: RotationProcessor; + + constructor(window: Window, engine: KeymanEngine) { + this.window = window; + this.engine = engine; + + this.attachHandlers(); + + if(engine.config.hostDevice.touchable) { + this.buildPageTrailer(); + + this.rotationProcessor = new RotationProcessor(this.engine); + this.rotationProcessor.init(); + } + } + + private buildPageTrailer() { + // Add a blank DIV to the bottom of the page to allow the bottom of the page to be shown + const dTrailer = this.mobilePageTrailer = document.createElement('div'); + const ds=dTrailer.style; + ds.width='100%'; + ds.height=(screen.width/2)+'px'; // ... interesting choice, but okay. + document.body.appendChild(dTrailer); + } + + private get focusAssistant(): FocusAssistant { + return this.engine.contextManager.focusAssistant; + } + + private suppressFocusCheck: (e: FocusEvent) => boolean = (e) => { + if(this.focusAssistant.isTargetForcingScroll()) { + // Prevent triggering other blur-handling events (as possible) - this blur + // is programmatic in order to force a browser scroll-position update. + // All focus changes should be prevented at this time. + e.stopPropagation(); + e.cancelBubble = true; + } + // But DO perform default event behavior (actually blurring & focusing the affected element) + return true; + } + + /** + * Reset context when entering or exiting the active element. + * Will also trigger OSK shift state / layer reset. + **/ + private pageFocusHandler: (e: FocusEvent) => boolean = () => { + if(!this.focusAssistant.maintainingFocus && this.engine.osk?.vkbd) { + this.engine.contextManager.deactivateCurrentTarget(); + this.engine.contextManager.resetContext(); + } + return false; + } + + // Sets up page-default touch-based handling for activation-state management. + // These always trigger for the page, wherever a touch may occur. Does not + // prevent element-specific or OSK-key-specific handling from triggering. + + private touchStartActivationHandler: (e: TouchEvent) => boolean = (e) => { + const osk = this.engine.osk; + if(!osk) { + return false; + } + const device = this.engine.config.hostDevice; + + this.deactivateOnRelease=true; + this.touchY=e.touches[0].screenY; + + // On Chrome, scrolling up or down causes the URL bar to be shown or hidden + // according to whether or not the document is at the top of the screen. + // But when doing that, each OSK row top and height gets modified by Chrome + // looking very ugly. It would be best to hide the OSK then show it again + // when the user scroll finishes, but Chrome has no way to reliably report + // the touch end event after a move. c.f. http://code.google.com/p/chromium/issues/detail?id=152913 + // The best compromise behaviour is simply to hide the OSK whenever any + // non-input and non-OSK element is touched. + this.deactivateOnScroll=false; + if(device.OS == 'android' && device.browser == 'chrome') { + // this.deactivateOnScroll has the inverse of the 'true' default, + // but that fact actually facilitates the following conditional logic. + if(typeof(osk._Box) == 'undefined') return false; + if(typeof(osk._Box.style) == 'undefined') return false; + + // The following tests are needed to prevent the OSK from being hidden during normal input! + let p=(e.target as HTMLElement).parentElement; + if(typeof(p) != 'undefined' && p != null) { + if(p.className.indexOf('kmw-key-') >= 0) return false; + if(typeof(p.parentElement) != 'undefined' && p.parentElement != null) { + p=p.parentElement; + if(p.className.indexOf('kmw-key-') >= 0) return false; + } + } + + this.deactivateOnScroll = true; + } + return false; + }; + + private touchMoveActivationHandler: (e: TouchEvent) => boolean = (e) => { + if(this.deactivateOnScroll) { // Android / Chrone case. + this.focusAssistant.focusing = false; + this.engine.contextManager.deactivateCurrentTarget(); + } + + const y = e.touches[0].screenY; + const y0 = this.touchY; + if(y-y0 > 5 || y0-y < 5) { + this.deactivateOnRelease = false; + } + return false; + }; + + private touchEndActivationHandler: (e: TouchEvent) => boolean = (e) => { + // Should not hide OSK if simply closing the language menu (30/4/15) + // or if the focusing timer (focusAssistant.setFocusTimer) is still active. + if(this.deactivateOnRelease && !this.engine.touchLanguageMenu && !this.focusAssistant.focusing) { + this.engine.contextManager.deactivateCurrentTarget(); + } + this.deactivateOnRelease=false; + return false; + }; + + + private _WindowLoad: (e: Event) => void = () => { + // Always return to top of page after a page reload + document.body.scrollTop=0; + if(typeof document.documentElement != 'undefined') { + document.documentElement.scrollTop=0; + } + } + + /** + * Function _WindowUnload + * Scope Private + * Description Remove handlers before detaching KMW window + */ + private _WindowUnload: () => void = () => { + // Future note: should restrict this to anything for the corresponding document if on a + // child iframe, not the whole engine. + this.engine.shutdown(); + } + + private attachHandlers() { + const eventTracker = this.domEventTracker; + const device = this.engine.config.hostDevice; + const docBody = this.window.document.body; + + eventTracker.attachDOMEvent(this.window, 'focus', this.pageFocusHandler, false); + eventTracker.attachDOMEvent(this.window, 'blur', this.pageFocusHandler, false); + + /* + * To prevent propagation of focus & blur events from the input-scroll workaround, + * we attach top-level capturing listeners to the focus & blur events. They prevent propagation + * but NOT default behavior, allowing the scroll to complete while preventing nearly all + * possible event 'noise' that could result from the workaround. + */ + eventTracker.attachDOMEvent(docBody, 'focus', this.suppressFocusCheck, true); + eventTracker.attachDOMEvent(docBody, 'blur', this.suppressFocusCheck, true); + + if(device.touchable) { + eventTracker.attachDOMEvent(docBody, 'touchstart', this.touchStartActivationHandler,false); + eventTracker.attachDOMEvent(docBody, 'touchmove', this.touchMoveActivationHandler, false); + eventTracker.attachDOMEvent(docBody, 'touchend', this.touchEndActivationHandler, false); + } + + eventTracker.attachDOMEvent(window, 'load', this._WindowLoad, false); + eventTracker.attachDOMEvent(window, 'unload', this._WindowUnload,false); + + eventTracker.attachDOMEvent(document, 'keyup', this.engine.hotkeyManager._Process, false); + } + + public shutdown() { + const eventTracker = this.domEventTracker; + const device = this.engine.config.hostDevice; + const docBody = this.window.document.body; + + // See `attachHandlers` for the purpose behind all handlers listed here. + + eventTracker.detachDOMEvent(this.window, 'focus', this.pageFocusHandler, false); + eventTracker.detachDOMEvent(this.window, 'blur', this.pageFocusHandler, false); + + eventTracker.detachDOMEvent(docBody, 'focus', this.suppressFocusCheck, true); + eventTracker.detachDOMEvent(docBody, 'blur', this.suppressFocusCheck, true); + + if(device.touchable) { + eventTracker.detachDOMEvent(docBody, 'touchstart', this.touchStartActivationHandler,false); + eventTracker.detachDOMEvent(docBody, 'touchmove', this.touchMoveActivationHandler, false); + eventTracker.detachDOMEvent(docBody, 'touchend', this.touchEndActivationHandler, false); + + this.mobilePageTrailer?.parentElement.removeChild(this.mobilePageTrailer); + } + + eventTracker.detachDOMEvent(window, 'load', this._WindowLoad, false); + eventTracker.detachDOMEvent(window, 'unload', this._WindowUnload,false); + + eventTracker.detachDOMEvent(document, 'keyup', this.engine.hotkeyManager._Process, false); + } +} \ No newline at end of file diff --git a/web/src/app/browser/src/contextManager.ts b/web/src/app/browser/src/contextManager.ts new file mode 100644 index 0000000000000000000000000000000000000000..42e7edd729f291c3bbef37a8e6ec00dee3d0b6e3 --- /dev/null +++ b/web/src/app/browser/src/contextManager.ts @@ -0,0 +1,795 @@ +import { type Keyboard, KeyboardScriptError } from '@keymanapp/keyboard-processor'; +import { type KeyboardStub } from 'keyman/engine/package-cache'; +import { CookieSerializer } from 'keyman/engine/dom-utils'; +import { eventOutputTarget, PageContextAttachment } from 'keyman/engine/attachment'; +import { DomEventTracker, LegacyEventEmitter } from 'keyman/engine/events'; +import { DesignIFrame, OutputTarget, nestedInstanceOf } from 'keyman/engine/element-wrappers'; +import { + ContextManagerBase, + type KeyboardInterface, + LegacyAPIEvents +} from 'keyman/engine/main'; +import { BrowserConfiguration } from './configuration.js'; +import { FocusAssistant } from './context/focusAssistant.js'; + +export interface KeyboardCookie { + current: string; +} + +/** + * Set target element text direction (LTR or RTL), but only if the element is empty + * + * If the element base directionality is changed after it contains content, unless all the text + * has the same directionality, text runs will be re-ordered which is confusing and causes + * incorrect caret positioning + * + * @param {Object} Ptarg Target element + */ +function _SetTargDir(Ptarg: HTMLElement, activeKeyboard: Keyboard) { + const elDir = activeKeyboard?.isRTL ? 'rtl' : 'ltr'; + + if(Ptarg) { + if(Ptarg instanceof Ptarg.ownerDocument.defaultView.HTMLInputElement + || Ptarg instanceof Ptarg.ownerDocument.defaultView.HTMLTextAreaElement) { + if((Ptarg as HTMLInputElement|HTMLTextAreaElement).value.length == 0) { + Ptarg.dir=elDir; + } + } else if(typeof Ptarg.textContent == "string" && Ptarg.textContent.length == 0) { // As with contenteditable DIVs, for example. + Ptarg.dir=elDir; + } + } +} + +export default class ContextManager extends ContextManagerBase { + private _activeKeyboard: {keyboard: Keyboard, metadata: KeyboardStub}; + private cookieManager = new CookieSerializer('KeymanWeb_Keyboard'); + readonly focusAssistant = new FocusAssistant(() => this.activeTarget?.isForcingScroll()); + readonly page: PageContextAttachment; + private mostRecentTarget: OutputTarget; + private currentTarget: OutputTarget; + + private globalKeyboard: {keyboard: Keyboard, metadata: KeyboardStub}; + + private _eventsObj: () => LegacyEventEmitter; + private domEventTracker = new DomEventTracker(); + + constructor(engineConfig: BrowserConfiguration, eventsClosure: () => LegacyEventEmitter) { + super(engineConfig); + + this._eventsObj = eventsClosure; + + this.page = new PageContextAttachment(window.document, { + hostDevice: this.engineConfig.hostDevice + }); + + this.focusAssistant.on('maintainingfocusend', () => { + // Basically, if the maintaining state were the reason we still had an `activeTarget`... + if(!this.activeTarget && this.mostRecentTarget) { + this.emit('targetchange', this.activeTarget); + } + }); + } + + get apiEvents(): LegacyEventEmitter { + return this._eventsObj(); + } + + initialize(): void { + this.on('keyboardasyncload', (stub, completion) => { + this.engineConfig.alertHost?.wait('Installing keyboard
' + stub.name); + + completion.then(() => { + this.engineConfig.alertHost?.wait(); // cancels the wait. + }); + }); + + this.engineConfig.deferForInitialization.then(() => { + const device = this.engineConfig.hostDevice; + + const noPropagation = (event: Event) => event.stopPropagation() + + // For any elements being attached, or being enabled after having been disabled... + this.page.on('enabled', (elem) => { + if(!(elem._kmwAttachment.interface instanceof DesignIFrame)) { + // For anything attached but (design-mode) iframes... + + // This block: has to do with maintaining focus. + if(device.touchable) { + // Remove any handlers for "NonKMWTouch" elements, since we're enabling it here. + this.domEventTracker.detachDOMEvent(elem, 'touchstart', this.nonKMWTouchHandler); + + // Prevent base-page touch handlers from causing a defocus when interacting + // with attached input elements. + this.domEventTracker.attachDOMEvent(elem, 'touchmove', noPropagation, false); + this.domEventTracker.attachDOMEvent(elem, 'touchend', noPropagation, false); + } + + // This block: has to do with maintaining focus. + this.domEventTracker.attachDOMEvent(elem,'focus', this._ControlFocus); + this.domEventTracker.attachDOMEvent(elem,'blur', this._ControlBlur); + this.domEventTracker.attachDOMEvent(elem,'click', this._Click); + } else { + // For design-mode iframes: + + // This block: has to do with maintaining focus. + var Lelem=(elem as HTMLIFrameElement).contentWindow.document; + // I2404 - Attach to IFRAMEs child objects, only editable IFRAMEs here + if(device.browser == 'firefox') { + this.domEventTracker.attachDOMEvent(Lelem,'focus', this._ControlFocus); + this.domEventTracker.attachDOMEvent(Lelem,'blur', this._ControlBlur); + } else { // Chrome, Safari + this.domEventTracker.attachDOMEvent(Lelem.body,'focus', this._ControlFocus); + this.domEventTracker.attachDOMEvent(Lelem.body,'blur', this._ControlBlur); + } + } + }); + + // For any elements being detached, disabled, or deliberately not being attached (b/c nonKMWTouchHandler)... + this.page.on('disabled', (elem) => { + // Note: we may not actually be attached at this point. + if(!(nestedInstanceOf(elem, "HTMLIFrameElement"))) { + // For anything attached but (design-mode) iframes... + + // This block: has to do with maintaining focus. + if(device.touchable) { + this.domEventTracker.attachDOMEvent(elem, 'touchstart', this.nonKMWTouchHandler, false); + + // does not detach the touch-handlers added in 'enabled'? + } + + // This block: has to do with maintaining focus. + this.domEventTracker.detachDOMEvent(elem,'focus', this._ControlFocus); + this.domEventTracker.detachDOMEvent(elem,'blur', this._ControlBlur); + this.domEventTracker.detachDOMEvent(elem,'click', this._Click); + } else { + // For design-mode iframes: + + // This block: has to do with maintaining focus. + let Lelem = (elem as HTMLIFrameElement).contentWindow.document; + // Mozilla // I2404 - Attach to IFRAMEs child objects, only editable IFRAMEs here + if(device.browser == 'firefox') { + // Firefox won't handle these events on Lelem.body - only directly on Lelem (the doc) instead. + this.domEventTracker.detachDOMEvent(Lelem,'focus', this._ControlFocus); + this.domEventTracker.detachDOMEvent(Lelem,'blur', this._ControlBlur); + } else { // Chrome, Safari + this.domEventTracker.detachDOMEvent(Lelem.body,'focus', this._ControlFocus); + this.domEventTracker.detachDOMEvent(Lelem.body,'blur', this._ControlBlur); + } + } + + // This block: has to do with maintaining focus (and consequences) + var lastElem = this.mostRecentTarget?.getElement(); + if(lastElem && lastElem == elem) { + this.forgetActiveTarget(); // should already auto-hide the OSK while at it via event. + } + }); + + // This fires the events we just registered for. + this.page.install(this.engineConfig.attachType == 'manual'); + }); + } + + get activeTarget(): OutputTarget { + /* + * Assumption: the maintainingFocus flag may only be set when there is a current target. + * This is not enforced proactively at present, but the assumption should hold. (2023-05-03) + */ + const maintainingFocus = this.focusAssistant.maintainingFocus; + return this.currentTarget || (maintainingFocus ? this.mostRecentTarget : null); + } + + get lastActiveTarget(): OutputTarget { + return this.mostRecentTarget; + } + + public deactivateCurrentTarget() { + const priorTarget = this.activeTarget || this.lastActiveTarget; + if(priorTarget) { + this._BlurKeyboardSettings(priorTarget.getElement()); + } + + // Because of focus-maintenance effects + if(!this.activeTarget) { + this.setActiveTarget(null, true); + } + } + + public forgetActiveTarget() { + this.focusAssistant.maintainingFocus = false; + this.focusAssistant.restoringFocus = false; + + const priorTarget = this.activeTarget || this.lastActiveTarget; + if(priorTarget) { + this._BlurKeyboardSettings(priorTarget.getElement()); + } + + this.setActiveTarget(null, true); + } + + public setActiveTarget(target: OutputTarget, sendEvents: boolean) { + const previousTarget = this.mostRecentTarget; + const originalTarget = this.activeTarget; // may differ, depending on focus state. + + if(target == originalTarget) { + /** + * If it's already active, we should cancel early. + * + * The #1 reason - we don't want .resetContext calls in this scenario. + * In particular, moving the caret or setting the selection range of an + * or + +

or in this input field:

+ + + +

Add a keyboard by keyboard name:

+ + + +

Add a keyboard by BCP-47 language code:

+ + + +

Add a keyboard by language name:

+ + + + +

Return to samples home page

+ + + + + + + diff --git a/web/src/samples/index.html b/web/src/samples/index.html index 8d0ac059c08f57a1c0f708a805d4c5f38593fc30..2a9259063f919bdeb29a8f2d489c905be9b83f3d 100644 --- a/web/src/samples/index.html +++ b/web/src/samples/index.html @@ -20,9 +20,9 @@

KeymanWeb Samples

-

Test unminified Keymanweb

-

Test fully compiled Keymanweb

-

Full multilingual (compiled) Keymanweb

-

Return to main index. +

Example 1 - Toggle UI, all resources in same folder as page

+

Example 2 - Toggle UI, all resources in a common subfolder

+

Example 3 - Toolbar UI, all resources in a common subfolder

+

Example 4 - a more complex file-organization setup

diff --git a/web/src/samples/samplehdr.js b/web/src/samples/samplehdr.js index 2ce215c666e9d49a04e468f79e6a19b7b3f7fc26..129d54fc011ebe663f889fb914524d9dbf04799b 100644 --- a/web/src/samples/samplehdr.js +++ b/web/src/samples/samplehdr.js @@ -1,111 +1,146 @@ // JavaScript Document samplehdr.js: Keyboard management for KeymanWeb demonstration pages -/* +/* The keyboard name and/or BCP-47 language code must be specified for each keyboard that is to be available. If the same keyboard is used for several languages, it must be listed for each - language, but the keyboard itself will only be loaded once. + language, but the keyboard itself will only be loaded once. If two (or more) keyboards are to be available for a given language, both must be listed. - Any number of keyboards may be specified in one or more calls. - Keyboard paths may be absolute (with respect to the server root) or relative to the keyboards option path. + Any number of keyboards may be specified in one or more calls. + Keyboard paths may be absolute (with respect to the server root) or relative to the keyboards option path. The actual keyboard object will be downloaded asynchronously when first selected for use. - + Each argument to addKeyboards() is a string, for example: european2 loads the current version of the Eurolatin 2 keyboard (for its default language) european2@fr loads the current version of the Eurolatin 2 keyboard for French european2@fr@1.2 loads version 1.2 of the Eurolatin 2 keyboard for French - + Argument syntax also supports the following extensions: @fr load the current version of the default keyboard for French @fr$ load all available keyboards (current version) for French - - Each call to addKeyboards() requires a single call to the remote server, + + Each call to addKeyboards() requires a single call to the remote server, (unless all keyboards listed are local and fully specified) so it is better - to use multiple arguments rather than separate function calls. - - Calling addKeyboards() with no arguments returns a list of *all* available keyboards. - The Toolbar (desktop browser) UI is best suited for allowing users to select + to use multiple arguments rather than separate function calls. + + Calling addKeyboards() with no arguments returns a list of *all* available keyboards. + The Toolbar (desktop browser) UI is best suited for allowing users to select the appropriate language and keyboard in this case. Keyboards may also be specified by language name using addKeyboardsForLanguage() for example: keymanweb.addKeyboardsForLanguage('Burmese'); - + Appending $ to the language name will again cause all available keyboards for that language to be loaded rather than the default keyboard. - - The first call to addKeyboardsForLanguage() makes an additional call to the + + The first call to addKeyboardsForLanguage() makes an additional call to the keyman API to load the current list of keyboard/language associations. In this example, the following function loads the indicated keyboards, - and is called when the page loads. + and is called when the page loads. */ - function loadKeyboards() - { + function errToString(err) { + // Painful? Kinda. But needed on un-updated Android API 21! + if(Array.isArray(err)) { + var result = ''; + for(var i = 0; i < err.length; i++) { + var e = err[i]; + if(e.error instanceof Error) { + result += e.error.message + '\n'; + } else { + result += JSON.stringify(e) + '\n'; + } + } + return result; + } + if(err instanceof Error) { + return err.message; + } + return JSON.stringify(err); + } + + function doAddKeyboards(data) { + return keyman.addKeyboards(data).catch(function(err) { + console.error('keyman.addKeyboards failed with '+errToString(err)+' for '+JSON.stringify(data)); + }); + } + + function doAddKeyboardsForLanguage(data) { + return keyman.addKeyboardsForLanguage(data).catch(function(err) { + console.error('keyman.addKeyboardsForLanguage failed with '+errToString(err)+' for '+JSON.stringify(data)); + }); + } + + function loadKeyboards(nestLevel) { var kmw=keyman; - + + var base_prefix = '../'; + var prefix = './'; // The default - when prefix == 0. + + if(nestLevel !== undefined && nestLevel > 0) { + prefix = ''; + for(var i=0; i < nestLevel; i++) { + prefix = prefix + base_prefix; + } + } + // The first keyboard added will be the default keyboard for touch devices. // For faster loading, it may be best for the default keyboard to be // locally sourced. - kmw.addKeyboards({id:'us',name:'English',languages:{id:'en',name:'English'}, - filename:'./us-1.0.js'}); - - // Add more keyboards to the language menu, by keyboard name, - // keyboard name and language code, or just the BCP-47 language code. - kmw.addKeyboards('french', 'sil_euro_latin@no,sv', '@he'); // Loads all from uniquely-identifying strings. - + doAddKeyboards({id:'us',name:'English',languages:{id:'en',name:'English'}, + filename:(prefix + 'us-1.0.js')}); + + // Add more keyboards to the language menu by: + // 1. keyboard name ('french'), + // 2. keyboard name and language code ('sil_euro_latin@no,sv'), + // 3. or just the BCP-47 language code ('@he'). + kmw.addKeyboards('french', 'sil_euro_latin@no,sv', '@he'); + // Add a keyboard by language name. Note that the name must be spelled // correctly, or the keyboard will not be found. (Using BCP-47 codes is // usually easier.) - kmw.addKeyboardsForLanguage('Dzongkha'); - - // Add a fully-specified, locally-sourced, keyboard with custom font - kmw.addKeyboards({id:'lao_2008_basic',name:'Lao Basic', - languages:{ - id:'lo',name:'Lao',region:'Asia', - // A font can be specified here if its files are available. - // Example: - //font:{family:'LaoWeb',source:['../font/saysettha_web.ttf','../font/saysettha_web.woff','../font/saysettha_web.eot']} - }, - filename:'./lao_2008_basic-1.2.js' - }); - - // The following two optional calls should be delayed until language menus are fully loaded: - // (a) a specific mapped input element input is focused, to ensure that the OSK appears - // (b) a specific keyboard is loaded, rather than the keyboard last used. - //window.setTimeout(function(){kmw.setActiveElement('ta1',true);},2500); - //window.setTimeout(function(){kmw.setActiveKeyboard('Keyboard_french','fr');},3000); - - // Note that locally specified keyboards will be listed before keyboards + doAddKeyboardsForLanguage('Dzongkha'); + + // Add a fully-specified, locally-sourced, keyboard with custom font + doAddKeyboards({ + id:'lao_2008_basic', + name:'Lao Basic', + languages: { + id:'lo',name:'Lao',region:'Asia', + }, + filename:(prefix + 'lao_2008_basic-1.2.js') + }); + + // Note that locally specified keyboards will be listed before keyboards // requested from the remote server by user interfaces that do not order // keyboards alphabetically by language. } - - // Script to allow a user to add any keyboard to the keyboard menu - function addKeyboard(n) - { - var sKbd,kmw=keyman; - switch(n) - { + + // Script to allow a user to add any keyboard to the keyboard menu + function addKeyboard(n) { + var sKbd; + switch(n) { case 1: sKbd=document.getElementById('kbd_id1').value; - kmw.addKeyboards(sKbd); + doAddKeyboards(sKbd); break; case 2: sKbd=document.getElementById('kbd_id2').value.toLowerCase(); - kmw.addKeyboards('@'+sKbd); + doAddKeyboards('@'+sKbd); break; case 3: + // Add keyboard for comma-separated language name(s) sKbd=document.getElementById('kbd_id3').value; - kmw.addKeyboardsForLanguage(sKbd); + doAddKeyboardsForLanguage(sKbd); break; } } - + // Add keyboard on Enter (as well as pressing button) function clickOnEnter(e,id) { e = e || window.event; - if(e.keyCode == 13) addKeyboard(id); + if(e.keyCode == 13) addKeyboard(id); } diff --git a/web/src/samples/simplest/.gitignore b/web/src/samples/simplest/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..61b48a93bc558b4a380ea353106564be89a46d08 --- /dev/null +++ b/web/src/samples/simplest/.gitignore @@ -0,0 +1,3 @@ +**/* +!.gitignore +!index.html \ No newline at end of file diff --git a/web/src/samples/minified.html b/web/src/samples/simplest/index.html similarity index 89% rename from web/src/samples/minified.html rename to web/src/samples/simplest/index.html index 9b5d1674e89a537e6a33035bf31bcc2db8510b7a..c2ed9fd70bc53cdbfb61e754322cd9c72720522c 100644 --- a/web/src/samples/minified.html +++ b/web/src/samples/simplest/index.html @@ -23,7 +23,7 @@ - + - + - + -

KeymanWeb Sample Page - Fully Compiled/Minified Source

+

KeymanWeb Sample Page - Simple Subfolder + Toggle UI

- KeymanWeb Sample Page - Unminified Source + KeymanWeb Sample Page - Fully Compiled Source - - + + - + - + - + -

KeymanWeb Sample Page - Unminified Source

+

KeymanWeb Sample Page - Simple Subfolder + Toggle UI

+ diff --git a/web/src/samples/subfolder_toolbar/.gitignore b/web/src/samples/subfolder_toolbar/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..659fe973e2a11029bed8f9381d1dd9fa9b00e468 --- /dev/null +++ b/web/src/samples/subfolder_toolbar/.gitignore @@ -0,0 +1 @@ +keyman/**/* \ No newline at end of file diff --git a/web/src/samples/multilingual.html b/web/src/samples/subfolder_toolbar/index.html similarity index 82% rename from web/src/samples/multilingual.html rename to web/src/samples/subfolder_toolbar/index.html index de2174a49a8e4dd706ccc89480a4cfcc3347cbf4..7e9aed3f3f7074eca65c9603f410df2d24452607 100644 --- a/web/src/samples/multilingual.html +++ b/web/src/samples/subfolder_toolbar/index.html @@ -21,17 +21,16 @@ - + - + @@ -76,7 +72,7 @@ -

Return to samples home page

+

Return to samples home page

diff --git a/web/src/test/auto/CI.conf.js b/web/src/test/auto/CI.conf.js deleted file mode 100644 index d3d2b02e400591819170741716f02af73dce9277..0000000000000000000000000000000000000000 --- a/web/src/test/auto/CI.conf.js +++ /dev/null @@ -1,4 +0,0 @@ -var BASE_CONFIG = require("./base.conf.js"); -var ci_config_adapter = require("../../../../common/test/resources/karma-browserstack-config.js"); - -module.exports = ci_config_adapter(BASE_CONFIG, "full"); \ No newline at end of file diff --git a/web/src/test/auto/cases/attachmentAPI.js b/web/src/test/auto/cases/attachmentAPI.js deleted file mode 100644 index 0fe4d933b24130ca31f62924100b116569cd076b..0000000000000000000000000000000000000000 --- a/web/src/test/auto/cases/attachmentAPI.js +++ /dev/null @@ -1,257 +0,0 @@ -var assert = chai.assert; - -describe('Attachment API', function() { - this.timeout(testconfig.timeouts.standard); - - before(function() { - assert.isFalse(com.keyman.karma.DEVICE_DETECT_FAILURE, "Cannot run due to device detection failure."); - fixture.setBase('fixtures'); - - this.timeout(testconfig.timeouts.scriptLoad * 3); - return setupKMW({ attachType:'manual' }, testconfig.timeouts.scriptLoad).then(() => { - const kbd1 = loadKeyboardFromJSON("/keyboards/lao_2008_basic.json", testconfig.timeouts.scriptLoad, { passive: true }); - const kbd2 = loadKeyboardFromJSON("/keyboards/khmer_angkor.json", testconfig.timeouts.scriptLoad, { passive: true }); - return Promise.all([kbd1, kbd2]).then(() => { - return keyman.setActiveKeyboard("lao_2008_basic", "lo"); - }); - }); - }); - - after(function() { - keyman.removeKeyboards('lao_2008_basic'); - keyman.removeKeyboards('khmer_angkor'); - teardownKMW(); - }); - - beforeEach(function() { - fixture.load("robustAttachment.html"); - }); - - afterEach(function(done) { - fixture.cleanup(); - window.setTimeout(function(){ - done(); - }, testconfig.timeouts.eventDelay); - }); - - it("Attachment/Detachment", function(done) { - // Since we're in 'manual', we start detached. - var ele = document.getElementById(DynamicElements.addInput()); - - window.setTimeout(function() { - // Ensure we didn't auto-attach. - DynamicElements.assertDetached(ele); - let eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(DynamicElements.keyCommand); - - var val = ele.value; - ele.value = ""; - assert.equal(val, DynamicElements.disabledOutput, "'Detached' element performed keystroke processing!"); - - keyman.attachToControl(ele); - DynamicElements.assertAttached(ele); // Happens in-line, since we directly request the attachment. - - eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(DynamicElements.keyCommand); - - val = retrieveAndReset(ele); - - assert.equal(val, DynamicElements.enabledLaoOutput, "'Attached' element did not perform keystroke processing!"); - - done(); - }, testconfig.timeouts.eventDelay); - }); - - it("Enablement/Disablement", function(done) { - // Since we're in 'manual', we start detached. - var ele = document.getElementById(DynamicElements.addInput()); - window.setTimeout(function() { - keyman.attachToControl(ele); - keyman.disableControl(ele); - - // It appears that mobile devices do not instantly trigger the MutationObserver, so we need a small timeout - // for the change to take effect. - window.setTimeout(function() { - DynamicElements.assertAttached(ele); - let eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(ele); - assert.equal(val, DynamicElements.disabledOutput, "'Disabled' element performed keystroke processing!"); - - keyman.enableControl(ele); - window.setTimeout(function() { - DynamicElements.assertAttached(ele); // Happens in-line, since we directly request the attachment. - let eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(ele); - assert.equal(val, DynamicElements.enabledLaoOutput, "'Enabled' element did not perform keystroke processing!"); - done(); - }, testconfig.timeouts.eventDelay); - }, testconfig.timeouts.eventDelay); - }, testconfig.timeouts.eventDelay); - }); - - it("Keyboard Management (active control)", function() { - // It appears that event generation + inline event dispatching is a bit time-intensive on some browsers. - this.timeout(testconfig.timeouts.standard * 2); - - var input = document.getElementById(DynamicElements.addInput()); - var textarea = document.getElementById(DynamicElements.addText()); - - keyman.attachToControl(input); - keyman.attachToControl(textarea); - - keyman.setActiveElement(input); - // We assume from the other tests that running on the Lao keyboard will give proper output. - // It'd be a redundant check. - - // Set control with independent keyboard. - keyman.setKeyboardForControl(input, "khmer_angkor", "km"); - var eventDriver = new KMWRecorder.BrowserDriver(input); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(input); - assert.equal(val, DynamicElements.enabledKhmerOutput, "KMW did not use control's keyboard settings!"); - - // Swap to a global-linked control... - keyman.setActiveElement(textarea); - eventDriver = new KMWRecorder.BrowserDriver(textarea); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(textarea); - assert.equal(val, DynamicElements.enabledLaoOutput, "KMW did not use manage keyboard settings correctly for global-linked control!"); - - // Swap back and check that the settings persist. - keyman.setActiveElement(input); - eventDriver = new KMWRecorder.BrowserDriver(input); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(input); - assert.equal(val, DynamicElements.enabledKhmerOutput, "KMW forgot control's independent keyboard settings!"); - - // Finally, clear the independent setting. - keyman.setKeyboardForControl(input, null, null); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(input); - assert.equal(val, DynamicElements.enabledLaoOutput, "KMW did not properly clear control's independent keyboard settings!"); - }); - - it("Keyboard Management (inactive control)", function() { - // It appears that event generation + inline event dispatching is a bit time-intensive on some browsers. - this.timeout(testconfig.timeouts.standard * 2); - - var input = document.getElementById(DynamicElements.addInput()); - var textarea = document.getElementById(DynamicElements.addText()); - - keyman.attachToControl(input); - keyman.attachToControl(textarea); - - // We assume from the other tests that running on the Lao keyboard will give proper output. - // It'd be a redundant check. - - // We are testing that setting a specific keyboard for an inactive control does not affect - // the currently active control. The textarea control will be manually set to Khmer, - // and the input control will get the document default of Lao. - - keyman.setActiveElement(input); - // Set textarea control with independent keyboard khmer_angkor. - keyman.setKeyboardForControl(textarea, "khmer_angkor", "km"); - var eventDriver = new KMWRecorder.BrowserDriver(input); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(input); - assert.equal(val, DynamicElements.enabledLaoOutput, "KMW set independent keyboard for the incorrect control!"); - - // Swap to the textarea control with its overridden khmer_angkor keyboard... - keyman.setActiveElement(textarea); - eventDriver = new KMWRecorder.BrowserDriver(textarea); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(textarea); - assert.equal(val, DynamicElements.enabledKhmerOutput, "KMW did not properly store keyboard for the previously-inactive control!"); - - // Swap back to the input control and check that the settings persist. - keyman.setActiveElement(input); - keyman.setKeyboardForControl(textarea, null, null); - - eventDriver = new KMWRecorder.BrowserDriver(input); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(input); - assert.equal(val, DynamicElements.enabledLaoOutput, "KMW made a strange error when clearing an inactive control's keyboard setting!"); - - keyman.setActiveElement(textarea); - // Finally, after clearing the independent setting, check that we are back to Lao output as expected for the textarea - eventDriver = new KMWRecorder.BrowserDriver(textarea); - eventDriver.simulateEvent(DynamicElements.keyCommand); - val = retrieveAndReset(textarea); - assert.equal(val, DynamicElements.enabledLaoOutput, "KMW did not properly clear control's independent keyboard settings!"); - }); -}); - - -describe('Attachment Checks (Desktop, \'auto\')', function() { - - this.timeout(testconfig.timeouts.standard); - - before(function() { - this.timeout(testconfig.timeouts.scriptLoad); - - fixture.setBase('fixtures'); - return setupKMW({ attachType:'auto' }, testconfig.timeouts.scriptLoad); - }); - - beforeEach(function() { - fixture.load("robustAttachment.html"); - }); - - after(function() { - teardownKMW(); - }); - - afterEach(function(done) { - fixture.cleanup(); - window.setTimeout(function(){ - done(); - }, testconfig.timeouts.eventDelay); - }) - - describe('Element Type', function() { - it('', function(done) { - var ID = DynamicElements.addInput(); - var ele = document.getElementById(ID); - - DynamicElements.assertAttached(ele, done); - }); - - it('
\ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/dynamic-element-container.html b/web/src/test/auto/resources/fixtures/dynamic-element-container.html new file mode 100644 index 0000000000000000000000000000000000000000..b6cbe497058caaa006f2a52de0ed923eb075fe05 --- /dev/null +++ b/web/src/test/auto/resources/fixtures/dynamic-element-container.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/input-and-disabled-text.html b/web/src/test/auto/resources/fixtures/input-and-disabled-text.html new file mode 100644 index 0000000000000000000000000000000000000000..9f6f7e204c4860a8fc5fb493e222397dee00325b --- /dev/null +++ b/web/src/test/auto/resources/fixtures/input-and-disabled-text.html @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/input-and-text.html b/web/src/test/auto/resources/fixtures/input-and-text.html new file mode 100644 index 0000000000000000000000000000000000000000..b3a77bac09ac22593dc80d7fd47a75e366e1d244 --- /dev/null +++ b/web/src/test/auto/resources/fixtures/input-and-text.html @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/inputs-numeric-none-disabled_email.html b/web/src/test/auto/resources/fixtures/inputs-numeric-none-disabled_email.html new file mode 100644 index 0000000000000000000000000000000000000000..c2e5756aeecbe0b8f7e3dea83af48b0420367c46 --- /dev/null +++ b/web/src/test/auto/resources/fixtures/inputs-numeric-none-disabled_email.html @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/nested-iframe.html b/web/src/test/auto/resources/fixtures/nested-iframe.html new file mode 100644 index 0000000000000000000000000000000000000000..3fcbf6f27425c4367614dab44ebbfa917e6cb7fa --- /dev/null +++ b/web/src/test/auto/resources/fixtures/nested-iframe.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/simple-design-iframe.html b/web/src/test/auto/resources/fixtures/simple-design-iframe.html new file mode 100644 index 0000000000000000000000000000000000000000..be97af45a73bc659434257a144642a614acf0dec --- /dev/null +++ b/web/src/test/auto/resources/fixtures/simple-design-iframe.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/simple-editable-div.html b/web/src/test/auto/resources/fixtures/simple-editable-div.html new file mode 100644 index 0000000000000000000000000000000000000000..7cf99857eb36f9913d6fb5c40c46ca7b4b9054de --- /dev/null +++ b/web/src/test/auto/resources/fixtures/simple-editable-div.html @@ -0,0 +1 @@ +
Edit me!
\ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/simple-iframe-with-input.html b/web/src/test/auto/resources/fixtures/simple-iframe-with-input.html new file mode 100644 index 0000000000000000000000000000000000000000..870a19625ad1502a15053caa41dd1309281ad56b --- /dev/null +++ b/web/src/test/auto/resources/fixtures/simple-iframe-with-input.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/wild-absolute-positioning-extras.html b/web/src/test/auto/resources/fixtures/wild-absolute-positioning-extras.html new file mode 100644 index 0000000000000000000000000000000000000000..3ef8762c64ec683e3facd44dbf89fbaa9f20a5c4 --- /dev/null +++ b/web/src/test/auto/resources/fixtures/wild-absolute-positioning-extras.html @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/fixtures/wild-absolute-positioning.html b/web/src/test/auto/resources/fixtures/wild-absolute-positioning.html new file mode 100644 index 0000000000000000000000000000000000000000..95e0ce6458067b2cc31d455cd1b161ad90e6ac36 --- /dev/null +++ b/web/src/test/auto/resources/fixtures/wild-absolute-positioning.html @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/web/src/test/auto/resources/html/designIframe.html b/web/src/test/auto/resources/html/designIframe.html new file mode 100644 index 0000000000000000000000000000000000000000..604434258e2037f075bb32aa1e500a8ab459d495 --- /dev/null +++ b/web/src/test/auto/resources/html/designIframe.html @@ -0,0 +1,13 @@ + + + + + + + Edit me! + + \ No newline at end of file diff --git a/web/src/test/auto/resources/html/iframe.html b/web/src/test/auto/resources/html/iframe.html new file mode 100644 index 0000000000000000000000000000000000000000..366469f7bbc06d0fd75675546381611b1149c7eb --- /dev/null +++ b/web/src/test/auto/resources/html/iframe.html @@ -0,0 +1,7 @@ + + + +

Hello World!

+ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/html/nesting-inner-iframe.html b/web/src/test/auto/resources/html/nesting-inner-iframe.html new file mode 100644 index 0000000000000000000000000000000000000000..fed5398c43da181dff06deff08060ac8fe4af7a2 --- /dev/null +++ b/web/src/test/auto/resources/html/nesting-inner-iframe.html @@ -0,0 +1,7 @@ + + + +

Hello World!

+ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/html/nesting-outer-iframe.html b/web/src/test/auto/resources/html/nesting-outer-iframe.html new file mode 100644 index 0000000000000000000000000000000000000000..8428b4ab17f7e779dd9e0eca9e24a9f3d61ee74c --- /dev/null +++ b/web/src/test/auto/resources/html/nesting-outer-iframe.html @@ -0,0 +1,13 @@ + + + +

Hello World!

+

This is the 'outer' iframe, which itself hosts an INNER iframe used during the same test.

+ + + + +

Comments?

+ + + \ No newline at end of file diff --git a/web/src/test/auto/resources/query-mock-results/@dz.js.fixture b/web/src/test/auto/resources/query-mock-results/@dz.js.fixture new file mode 100644 index 0000000000000000000000000000000000000000..e63ab927651f573a24a702f1e4bc02b10aef959f --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/@dz.js.fixture @@ -0,0 +1,1609 @@ +keyman.register({ + "options": { + "context": "keyboard", + "dateFormat": "standard", + "device": "any", + "keyboardBaseUri": "https://s.keyman.com/keyboard/", + "fontBaseUri": "https://s.keyman.com/font/deploy/", + "keyboardid": "@dz", + "keyboardVersion": "current" + }, + "keyboard": [[{ + "id": "basic_kbddzo", + "name": "Dzongkha Basic", + "filename": "basic_kbddzo/1.0/basic_kbddzo-1.0.js", + "version": "1.0", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/basic/basic_kbddzo", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "dz", + "name": "Dzongkha", + "region": 3 + }] + }, { + "id": "tibetan_direct_input", + "name": "Tibetan Direct Input", + "filename": "tibetan_direct_input/1.4/tibetan_direct_input-1.4.js", + "version": "1.4", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/t/tibetan_direct_input", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "adi-tibt", + "name": "Adi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bft-tibt", + "name": "Balti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kjz-tibt", + "name": "Bumthangkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "cna-tibt", + "name": "Changthang", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dre-tibt", + "name": "Dolpo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dz", + "name": "Dzongkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bfu-tibt", + "name": "Gahri", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "clk-tibt", + "name": "Idu-Mishmi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "jya-tibt", + "name": "Jiarong", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kbg-tibt", + "name": "Khamba", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xkf-tibt", + "name": "Khengkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ght-tibt", + "name": "Kuke", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbj-tibt", + "name": "Ladakhi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lep-tibt", + "name": "Lepcha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "loy-tibt", + "name": "Lhowa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khk-tibt", + "name": "Mongolian, Halh", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "muk-tibt", + "name": "Mugom", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lae-tibt", + "name": "Pattani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "prx-tibt", + "name": "Purik", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xsr-tibt", + "name": "Sherpa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sip-tibt", + "name": "Sikkimese", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sbu-tibt", + "name": "Stod Bhoti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "taj-tibt", + "name": "Tamang, Eastern", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tdg-tibt", + "name": "Tamang, Western", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "adx-tibt", + "name": "Tibetan, Amdo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bo-tibt", + "name": "Tibetan, Central", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khg-tibt", + "name": "Tibetan, Khams", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbf-tibt", + "name": "Tinani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tsj", + "name": "Tshangla", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ola", + "name": "Walungge", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "zau-tibt", + "name": "Zangskari", + "region": 3 + }] + }, { + "id": "tibetan_ewts", + "name": "Tibetan EWTS", + "filename": "tibetan_ewts/2.4/tibetan_ewts-2.4.js", + "version": "2.4", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/t/tibetan_ewts", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "adi-tibt", + "name": "Adi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bft-tibt", + "name": "Balti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kjz-tibt", + "name": "Bumthangkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "cna-tibt", + "name": "Changthang", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xct-tibt", + "name": "Classical Tibetan", + "region": 1, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dre-tibt", + "name": "Dolpo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dz", + "name": "Dzongkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bfu-tibt", + "name": "Gahri", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "clk-tibt", + "name": "Idu-Mishmi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "jya-tibt", + "name": "Jiarong", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kbg-tibt", + "name": "Khamba", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xkf-tibt", + "name": "Khengkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ght-tibt", + "name": "Kuke", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbj-tibt", + "name": "Ladakhi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lep-tibt", + "name": "Lepcha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "loy-tibt", + "name": "Lhowa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khk-tibt", + "name": "Mongolian, Halh", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "muk-tibt", + "name": "Mugom", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "otb-tibt", + "name": "Old Tibetan", + "region": 1, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lae-tibt", + "name": "Pattani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "prx-tibt", + "name": "Purik", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xsr-tibt", + "name": "Sherpa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sip-tibt", + "name": "Sikkimese", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sbu-tibt", + "name": "Stod Bhoti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "taj-tibt", + "name": "Tamang, Eastern", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tdg-tibt", + "name": "Tamang, Western", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "adx-tibt", + "name": "Tibetan, Amdo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bo-tibt", + "name": "Tibetan, Central", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khg-tibt", + "name": "Tibetan, Khams", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbf-tibt", + "name": "Tinani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tsj-tibt", + "name": "Tshangla", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ola-tibt", + "name": "Walungge", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "zau-tibt", + "name": "Zangskari", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }] + }, { + "id": "dzongkha", + "name": "Dzongkha", + "filename": "dzongkha/1.0/dzongkha-1.0.js", + "version": "1.0", + "desktopKeyboardID": 652, + "source": "https://github.com/keymanapp/keyboards/tree/master/legacy/d/dzongkha", + "devices": { + "phone": 0, + "tablet": 0, + "desktop": 2 + }, + "languages": [{ + "id": "dz", + "name": "Dzongkha", + "region": 3 + }] + }, { + "id": "tibetan_direct_unicode", + "name": "Tibetan Direct-to-Unicode v2", + "filename": "tibetan_direct_unicode/1.1/tibetan_direct_unicode-1.1.js", + "version": "1.1", + "source": "https://github.com/keymanapp/keyboards/tree/master/legacy/t/tibetan_direct_unicode", + "devices": { + "phone": 1, + "tablet": 1, + "desktop": 2 + }, + "languages": [{ + "id": "dz", + "name": "Dzongkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bo", + "name": "Tibetan, Central", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }] + }, { + "id": "tibetan_unicode_direct_input", + "name": "Tibetan Unicode Direct Input", + "filename": "tibetan_unicode_direct_input/1.1/tibetan_unicode_direct_input-1.1.js", + "version": "1.1", + "source": "https://github.com/keymanapp/keyboards/tree/master/legacy/t/tibetan_unicode_direct_input", + "devices": { + "phone": 1, + "tablet": 1, + "desktop": 2 + }, + "languages": [{ + "id": "adi", + "name": "Adi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bft", + "name": "Balti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kjz", + "name": "Bumthangkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "cna", + "name": "Changthang", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dre", + "name": "Dolpo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dz", + "name": "Dzongkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bfu", + "name": "Gahri", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "clk", + "name": "Idu-Mishmi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "jya", + "name": "Jiarong", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kbg", + "name": "Khamba", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xkf", + "name": "Khengkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ght", + "name": "Kuke", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbj", + "name": "Ladakhi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lep", + "name": "Lepcha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "loy", + "name": "Lhowa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khk", + "name": "Mongolian, Halh", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "muk", + "name": "Mugom", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lae", + "name": "Pattani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "prx", + "name": "Purik", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xsr", + "name": "Sherpa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sip", + "name": "Sikkimese", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sbu", + "name": "Stod Bhoti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "taj", + "name": "Tamang, Eastern", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tdg", + "name": "Tamang, Western", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "adx", + "name": "Tibetan, Amdo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bo", + "name": "Tibetan, Central", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khg", + "name": "Tibetan, Khams", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbf", + "name": "Tinani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tsj", + "name": "Tshangla", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ola", + "name": "Walungge", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }] + }, { + "id": "tibetan_unicode_ewts", + "name": "Tibetan Unicode EWTS", + "filename": "tibetan_unicode_ewts/1.1/tibetan_unicode_ewts-1.1.js", + "version": "1.1", + "source": "https://github.com/keymanapp/keyboards/tree/master/legacy/t/tibetan_unicode_ewts", + "devices": { + "phone": 1, + "tablet": 1, + "desktop": 2 + }, + "languages": [{ + "id": "adi", + "name": "Adi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bft", + "name": "Balti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kjz", + "name": "Bumthangkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "cna", + "name": "Changthang", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dre", + "name": "Dolpo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "dz", + "name": "Dzongkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bfu", + "name": "Gahri", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "clk", + "name": "Idu-Mishmi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "jya", + "name": "Jiarong", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "kbg", + "name": "Khamba", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xkf", + "name": "Khengkha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ght", + "name": "Kuke", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbj", + "name": "Ladakhi", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lep", + "name": "Lepcha", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "loy", + "name": "Lhowa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khk", + "name": "Mongolian, Halh", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "muk", + "name": "Mugom", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lae", + "name": "Pattani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "prx", + "name": "Purik", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "xsr", + "name": "Sherpa", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sip", + "name": "Sikkimese", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "sbu", + "name": "Stod Bhoti", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "taj", + "name": "Tamang, Eastern", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tdg", + "name": "Tamang, Western", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "adx", + "name": "Tibetan, Amdo", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "bo", + "name": "Tibetan, Central", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "khg", + "name": "Tibetan, Khams", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "lbf", + "name": "Tinani", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "tsj", + "name": "Tshangla", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }, { + "id": "ola", + "name": "Walungge", + "region": 3, + "font": { + "family": "TibetanWeb", + "source": ["DDC_Uchen.ttf"] + }, + "oskFont": { + "family": "TibetanWebOsk", + "source": ["DDC_Uchen.ttf", "ddc_uchen-webfont.svg#ddc_uchenregular"] + } + }] + }]] +}); diff --git a/web/src/test/auto/resources/query-mock-results/khmer_angkor.hand-edited.js.fixture b/web/src/test/auto/resources/query-mock-results/khmer_angkor.hand-edited.js.fixture new file mode 100644 index 0000000000000000000000000000000000000000..63456e826c56008a74773870007fc7aa396cf680 --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/khmer_angkor.hand-edited.js.fixture @@ -0,0 +1,49 @@ +keyman.register( + { + "options": { + "context":"keyboard", + "dateFormat":"standard", + "device":"any", + // Change 1: redirect to /common/test/resources' subfolder. + // "keyboardBaseUri":"https://s.keyman.com/keyboard/", + "keyboardBaseUri":"/resources/keyboards/", + "fontBaseUri":"https://s.keyman.com/font/deploy/", + "keyboardid":"khmer_angkor", + "keyboardVersion":"current" + }, + "keyboard": [ + { + "id":"khmer_angkor", + "name":"Khmer Angkor", + // Change 2: Is unversioned within /common/test/resources. + // "filename":"khmer_angkor/1.3/khmer_angkor-1.3.js", + "filename": "khmer_angkor.js", + "version":"1.3", + "source":"https://github.com/keymanapp/keyboards/tree/master/release/k/khmer_angkor", + "devices": { + "phone":2, + "tablet":2, + "desktop":2 + }, + "languages": [ + { + "id":"km", + "name":"Khmer", + "region":3, + "font": { + "family":"Khmer Mondulkiri", + "source": [ + "Mondulkiri-R.ttf" + ] + }, "oskFont": { + "family":"Khmer Busra Kbd", + "source": [ + "khmer_busra_kbd.ttf" + ] + } + } + ] + } + ] + } +); \ No newline at end of file diff --git a/web/src/test/auto/resources/query-mock-results/languages.js.fixture b/web/src/test/auto/resources/query-mock-results/languages.js.fixture new file mode 100644 index 0000000000000000000000000000000000000000..dc4644444d14acad97f76881ddfefb300ec73221 --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/languages.js.fixture @@ -0,0 +1,8760 @@ +keyman.register({ + "options": { + "context": "language", + "dateFormat": "standard", + "device": "any", + "keyboardBaseUri": "https://s.keyman.com/keyboard/", + "fontBaseUri": "https://s.keyman.com/font/deploy/", + "keyboardVersion": "current" + }, + "languages": [{ + "name": "\u2019Are\u2019are (Latin)", + "id": "alu-latn", + "region": 7 + }, { + "name": "Abenaki, Western (Latin)", + "id": "abe-latn", + "region": 6 + }, { + "name": "Aceh (Latin)", + "id": "ace-latn", + "region": 3 + }, { + "name": "Achterhoeks", + "id": "act", + "region": 4 + }, { + "name": "Achumawi", + "id": "acv", + "region": 6 + }, { + "name": "Adara (Latin)", + "id": "kad-latn", + "region": 2 + }, { + "name": "Adasen (Latin)", + "id": "tiu-latn", + "region": 3 + }, { + "name": "Adi (Tibetan)", + "id": "adi-tibt", + "region": 3 + }, { + "name": "Adyghe (Latin)", + "id": "ady-latn", + "region": 4 + }, { + "name": "Aer (Arabic)", + "id": "aeq-arab", + "region": 3 + }, { + "name": "Afade", + "id": "aal", + "region": 2 + }, { + "name": "Afrikaans", + "id": "af", + "region": 2 + }, { + "name": "Afrikaans (Braille)", + "id": "af-brai", + "region": 2 + }, { + "name": "Agatu (Latin)", + "id": "agc-latn", + "region": 2 + }, { + "name": "Aghem", + "id": "agq", + "region": 2 + }, { + "name": "Agta, Casiguran Dumagat (Latin)", + "id": "dgc-latn", + "region": 3 + }, { + "name": "Agta, Central Cagayan (Latin)", + "id": "agt-latn", + "region": 3 + }, { + "name": "Agta, Dupaninan (Latin)", + "id": "duo-latn", + "region": 3 + }, { + "name": "Agta, Pahanan (Latin)", + "id": "apf-latn", + "region": 3 + }, { + "name": "Agta, Umiray Dumaget (Latin)", + "id": "due-latn", + "region": 3 + }, { + "name": "Agutaynen (Latin)", + "id": "agn-latn", + "region": 3 + }, { + "name": "Ahom (Ahom)", + "id": "aho-ahom", + "region": 3 + }, { + "name": "ajp-arab (Arabic)", + "id": "ajp-arab", + "region": 1 + }, { + "name": "Ajumbu", + "id": "muc", + "region": 2 + }, { + "name": "Akan (Latin)", + "id": "ak-latn", + "region": 2 + }, { + "name": "Akha", + "id": "ahk", + "region": 3 + }, { + "name": "Aklanon (Latin)", + "id": "akl-latn", + "region": 3 + }, { + "name": "Akoose", + "id": "bss", + "region": 2 + }, { + "name": "Akum", + "id": "aku", + "region": 2 + }, { + "name": "Alangan (Latin)", + "id": "alj-latn", + "region": 3 + }, { + "name": "Albanian", + "id": "sq", + "region": 1 + }, { + "name": "Albanian (Braille)", + "id": "sq-brai", + "region": 1 + }, { + "name": "Albanian, Arb\u00ebresh\u00eb", + "id": "aae", + "region": 4 + }, { + "name": "Albanian, Gheg", + "id": "aln", + "region": 4 + }, { + "name": "Albanian, Gheg (Braille)", + "id": "aln-brai", + "region": 4 + }, { + "name": "Albanian, Tosk (Latin)", + "id": "als-latn", + "region": 4 + }, { + "name": "Ale (Latin)", + "id": "gwd-latn", + "region": 2 + }, { + "name": "Aleut", + "id": "ale", + "region": 6 + }, { + "name": "Algonquin (Latin)", + "id": "alq-latn", + "region": 6 + }, { + "name": "Alsea", + "id": "aes", + "region": 1 + }, { + "name": "Alta, Southern (Latin)", + "id": "agy-latn", + "region": 3 + }, { + "name": "Aluo (Miao)", + "id": "yna-plrd", + "region": 3 + }, { + "name": "Alur (Latin)", + "id": "alz-latn", + "region": 2 + }, { + "name": "Ama (Latin)", + "id": "nyi-latn", + "region": 2 + }, { + "name": "Amba (Latin)", + "id": "rwm-latn", + "region": 2 + }, { + "name": "Ambele", + "id": "ael", + "region": 2 + }, { + "name": "Amharic", + "id": "am", + "region": 2 + }, { + "name": "Amharic (Braille)", + "id": "am-brai", + "region": 2 + }, { + "name": "Andaandi (Coptic)", + "id": "dgl-copt", + "region": 2 + }, { + "name": "Anii (Latin)", + "id": "blo-latn", + "region": 2 + }, { + "name": "Antigua and Barbuda Creole English", + "id": "aig", + "region": 6 + }, { + "name": "Anuak (Latin)", + "id": "anu-latn", + "region": 2 + }, { + "name": "Arabic", + "id": "ar", + "region": 1 + }, { + "name": "Arabic (Braille)", + "id": "ar-brai", + "region": 1 + }, { + "name": "Arabic (Algeria)", + "id": "ar-dz", + "region": 1 + }, { + "name": "Arabic, Algerian Spoken (Arabic)", + "id": "arq-arab", + "region": 2 + }, { + "name": "Arabic, Baharna Spoken (Arabic)", + "id": "abv-arab", + "region": 3 + }, { + "name": "Arabic, Chadian Spoken (Arabic)", + "id": "shu-arab", + "region": 2 + }, { + "name": "Arabic, Chadian Spoken (Latin)", + "id": "shu-latn", + "region": 2 + }, { + "name": "Arabic, Cypriot Spoken (Arabic)", + "id": "acy-arab", + "region": 3 + }, { + "name": "Arabic, Eastern Egyptian Bedawi Spoken (Arabic)", + "id": "avl-arab", + "region": 2 + }, { + "name": "Arabic, Egyptian Spoken (Arabic)", + "id": "arz-arab", + "region": 2 + }, { + "name": "Arabic, Egyptian Spoken (Braille)", + "id": "arz-brai", + "region": 2 + }, { + "name": "Arabic, Gulf Spoken (Arabic)", + "id": "afb-arab", + "region": 3 + }, { + "name": "Arabic, Hijazi Spoken (Arabic)", + "id": "acw-arab", + "region": 3 + }, { + "name": "Arabic, Libyan Spoken (Arabic)", + "id": "ayl-arab", + "region": 2 + }, { + "name": "Arabic, Mesopotamian Spoken (Arabic)", + "id": "acm-arab", + "region": 3 + }, { + "name": "Arabic, Moroccan Spoken (Arabic)", + "id": "ary-arab", + "region": 2 + }, { + "name": "Arabic, Najdi Spoken (Arabic)", + "id": "ars-arab", + "region": 3 + }, { + "name": "Arabic, North Levantine Spoken (Arabic)", + "id": "apc-arab", + "region": 3 + }, { + "name": "Arabic, North Mesopotamian Spoken (Arabic)", + "id": "ayp-arab", + "region": 3 + }, { + "name": "Arabic, Omani Spoken (Arabic)", + "id": "acx-arab", + "region": 3 + }, { + "name": "Arabic, Sa\u2019idi Spoken (Arabic)", + "id": "aec-arab", + "region": 2 + }, { + "name": "Arabic, Sanaani Spoken (Arabic)", + "id": "ayn-arab", + "region": 3 + }, { + "name": "Arabic, Standard (Arabic)", + "id": "arb-arab", + "region": 3 + }, { + "name": "Arabic, Standard (Braille)", + "id": "arb-brai", + "region": 3 + }, { + "name": "Arabic, Sudanese Spoken (Arabic)", + "id": "apd-arab", + "region": 2 + }, { + "name": "Arabic, Ta\u2019izzi-Adeni Spoken (Arabic)", + "id": "acq-arab", + "region": 3 + }, { + "name": "Arabic, Tajiki Spoken (Arabic)", + "id": "abh-arab", + "region": 3 + }, { + "name": "Arabic, Tunisian Spoken (Arabic)", + "id": "aeb-arab", + "region": 2 + }, { + "name": "Aragonese", + "id": "an", + "region": 4 + }, { + "name": "Araona", + "id": "aro", + "region": 6 + }, { + "name": "Argobba", + "id": "agj", + "region": 2 + }, { + "name": "Armenian", + "id": "hy", + "region": 3 + }, { + "name": "Armenian (Braille)", + "id": "hy-brai", + "region": 3 + }, { + "name": "Aromanian", + "id": "rup", + "region": 4 + }, { + "name": "Arpitan", + "id": "frp", + "region": 4 + }, { + "name": "Ashe", + "id": "ahs", + "region": 2 + }, { + "name": "Asoa (Latin)", + "id": "asv-latn", + "region": 2 + }, { + "name": "Assamese", + "id": "as", + "region": 3 + }, { + "name": "Assamese (India)", + "id": "as-in", + "region": 3 + }, { + "name": "Assamese (Latin)", + "id": "as-latn", + "region": 3 + }, { + "name": "Assangori", + "id": "sjg", + "region": 2 + }, { + "name": "Assiniboine (Latin)", + "id": "asb-latn", + "region": 6 + }, { + "name": "Assyrian Neo-Aramaic (Syriac)", + "id": "aii-syrc", + "region": 3 + }, { + "name": "Asturian", + "id": "ast", + "region": 4 + }, { + "name": "Asu (Latin)", + "id": "asa-latn", + "region": 2 + }, { + "name": "Athpariya", + "id": "aph", + "region": 3 + }, { + "name": "Atikamekw (Latin)", + "id": "atj-latn", + "region": 6 + }, { + "name": "Atong", + "id": "ato", + "region": 2 + }, { + "name": "Atta, Pamplona (Latin)", + "id": "att-latn", + "region": 3 + }, { + "name": "Avestan (Gujarati)", + "id": "ae-gujr", + "region": 3 + }, { + "name": "Avokaya (Latin)", + "id": "avu-latn", + "region": 2 + }, { + "name": "Awadhi (Devanagari)", + "id": "awa-deva", + "region": 3 + }, { + "name": "Awakateko", + "id": "agu", + "region": 6 + }, { + "name": "Awing", + "id": "azo", + "region": 2 + }, { + "name": "Awngi", + "id": "awn", + "region": 2 + }, { + "name": "Aymara", + "id": "ay", + "region": 1 + }, { + "name": "Ayta, Abellen (Latin)", + "id": "abp-latn", + "region": 3 + }, { + "name": "Ayta, Ambala (Latin)", + "id": "abc-latn", + "region": 3 + }, { + "name": "Ayta, Mag-antsi (Latin)", + "id": "sgb-latn", + "region": 3 + }, { + "name": "Ayta, Mag-Indi (Latin)", + "id": "blx-latn", + "region": 3 + }, { + "name": "Azerbaijani (Braille)", + "id": "az-brai", + "region": 1 + }, { + "name": "Azerbaijani (Cyrillic)", + "id": "az-cyrl", + "region": 1 + }, { + "name": "Azerbaijani, North (Braille)", + "id": "azj-brai", + "region": 3 + }, { + "name": "Azerbaijani, North (Cyrillic)", + "id": "azj-cyrl", + "region": 3 + }, { + "name": "Azerbaijani, North (Latin)", + "id": "azj-latn", + "region": 3 + }, { + "name": "Azerbaijani, North (Latin, Azerbaijan)", + "id": "azj-latn-az", + "region": 3 + }, { + "name": "Azerbaijani, South (Cyrillic)", + "id": "azb-cyrl", + "region": 3 + }, { + "name": "Baatonum (Latin)", + "id": "bba-latn", + "region": 2 + }, { + "name": "Babango (Latin)", + "id": "bbm-latn", + "region": 2 + }, { + "name": "Babanki", + "id": "bbk", + "region": 2 + }, { + "name": "Babine (Latin)", + "id": "bcr-latn", + "region": 6 + }, { + "name": "Badyara", + "id": "pbp", + "region": 2 + }, { + "name": "Bafanji", + "id": "bfj", + "region": 2 + }, { + "name": "Bafaw-Balong", + "id": "bwt", + "region": 2 + }, { + "name": "Bafia", + "id": "ksf", + "region": 2 + }, { + "name": "Bafut", + "id": "bfd", + "region": 2 + }, { + "name": "Bagirmi", + "id": "bmi", + "region": 2 + }, { + "name": "Bahamas Creole English", + "id": "bah", + "region": 6 + }, { + "name": "Bahing", + "id": "bhj", + "region": 3 + }, { + "name": "Bahnar (Latin)", + "id": "bdq-latn", + "region": 3 + }, { + "name": "Baka (Latin)", + "id": "bdh-latn", + "region": 2 + }, { + "name": "Baka", + "id": "bkc", + "region": 2 + }, { + "name": "Bakaka", + "id": "bqz", + "region": 2 + }, { + "name": "Bakoko", + "id": "bkh", + "region": 2 + }, { + "name": "Bakole", + "id": "kme", + "region": 2 + }, { + "name": "Balangao (Latin)", + "id": "blw-latn", + "region": 3 + }, { + "name": "Balanta-Ganja", + "id": "bjt", + "region": 2 + }, { + "name": "Balanta-Ganja (Latin)", + "id": "bjt-latn", + "region": 2 + }, { + "name": "Balanta-Kentohe", + "id": "ble", + "region": 2 + }, { + "name": "Baldemu", + "id": "bdn", + "region": 2 + }, { + "name": "Bali (Balinese)", + "id": "ban-bali", + "region": 3 + }, { + "name": "Bali (Latin)", + "id": "ban-latn", + "region": 3 + }, { + "name": "Bali (Latin)", + "id": "bcp-latn", + "region": 2 + }, { + "name": "Balkan Gagauz Turkish (Latin)", + "id": "bgx-latn", + "region": 3 + }, { + "name": "Balo", + "id": "bqo", + "region": 2 + }, { + "name": "Balochi, Eastern (Arabic)", + "id": "bgp-arab", + "region": 3 + }, { + "name": "Balti", + "id": "bft", + "region": 3 + }, { + "name": "Balti (Arabic, Pakistan)", + "id": "bft-arab-pk", + "region": 3 + }, { + "name": "Balti (Tibetan)", + "id": "bft-tibt", + "region": 3 + }, { + "name": "Baluchi", + "id": "bal", + "region": 1 + }, { + "name": "Baluchi (Latin)", + "id": "bal-latn", + "region": 1 + }, { + "name": "Bamali", + "id": "bbq", + "region": 2 + }, { + "name": "Bamanankan (Latin)", + "id": "bm-latn", + "region": 2 + }, { + "name": "Bamanankan (N\u2019Ko)", + "id": "bm-nkoo", + "region": 2 + }, { + "name": "Bambalang", + "id": "bmo", + "region": 2 + }, { + "name": "Bambili-Bambui", + "id": "baw", + "region": 2 + }, { + "name": "Bamenyam", + "id": "bce", + "region": 2 + }, { + "name": "Bamukumbit", + "id": "bqt", + "region": 2 + }, { + "name": "Bamun", + "id": "bax", + "region": 2 + }, { + "name": "Bamunka", + "id": "bvm", + "region": 2 + }, { + "name": "Bana", + "id": "bcw", + "region": 2 + }, { + "name": "Bandi (Latin)", + "id": "bza-latn", + "region": 2 + }, { + "name": "Bangala (Latin)", + "id": "bxg-latn", + "region": 2 + }, { + "name": "Bangandu", + "id": "bgf", + "region": 2 + }, { + "name": "Bangba (Latin)", + "id": "bbe-latn", + "region": 2 + }, { + "name": "Bangime (Latin)", + "id": "dba-latn", + "region": 2 + }, { + "name": "Bangolan", + "id": "bgj", + "region": 2 + }, { + "name": "Bangon (Latin)", + "id": "bnj-latn", + "region": 3 + }, { + "name": "Bangubangu (Latin)", + "id": "bnx-latn", + "region": 2 + }, { + "name": "Banjar (Latin)", + "id": "bjn-latn", + "region": 3 + }, { + "name": "Bankon", + "id": "abb", + "region": 2 + }, { + "name": "Bantawa", + "id": "bap", + "region": 3 + }, { + "name": "Bantoanon (Latin)", + "id": "bno-latn", + "region": 3 + }, { + "name": "Barambu (Latin)", + "id": "brm-latn", + "region": 2 + }, { + "name": "Barein", + "id": "bva", + "region": 2 + }, { + "name": "Bari (Latin)", + "id": "bfa-latn", + "region": 2 + }, { + "name": "Bariai", + "id": "bch", + "region": 7 + }, { + "name": "Barombi", + "id": "bbi", + "region": 2 + }, { + "name": "Barzani-Sandu Jewish Neo-Aramaic (Syriac)", + "id": "bjf-syrc", + "region": 3 + }, { + "name": "Basa (Latin)", + "id": "bzw-latn", + "region": 2 + }, { + "name": "Basaa", + "id": "bas", + "region": 2 + }, { + "name": "Bashkort (Cyrillic)", + "id": "ba-cyrl", + "region": 4 + }, { + "name": "Basketo (Ethiopic)", + "id": "bst-ethi", + "region": 2 + }, { + "name": "Basketo (Latin)", + "id": "bst-latn", + "region": 2 + }, { + "name": "Basque", + "id": "eu", + "region": 4 + }, { + "name": "Bassa (Latin)", + "id": "bsq-latn", + "region": 2 + }, { + "name": "Bassossi", + "id": "bsi", + "region": 2 + }, { + "name": "Bata", + "id": "bta", + "region": 2 + }, { + "name": "Batak (Latin)", + "id": "bya-latn", + "region": 3 + }, { + "name": "Batak Dairi", + "id": "btd", + "region": 3 + }, { + "name": "Batak Dairi (Latin)", + "id": "btd-latn", + "region": 3 + }, { + "name": "Batak Karo (Batak)", + "id": "btx-batk", + "region": 3 + }, { + "name": "Batak Karo (Latin)", + "id": "btx-latn", + "region": 3 + }, { + "name": "Batak Mandailing", + "id": "btm", + "region": 3 + }, { + "name": "Batak Simalungun (Batak)", + "id": "bts-batk", + "region": 3 + }, { + "name": "Batak Toba (Batak)", + "id": "bbc-batk", + "region": 3 + }, { + "name": "Batak Toba (Latin)", + "id": "bbc-latn", + "region": 3 + }, { + "name": "Batanga", + "id": "bnm", + "region": 2 + }, { + "name": "Bati", + "id": "btc", + "region": 2 + }, { + "name": "Bavarian", + "id": "bar", + "region": 4 + }, { + "name": "Bayot", + "id": "bda", + "region": 2 + }, { + "name": "Beaver", + "id": "bea", + "region": 6 + }, { + "name": "Beba", + "id": "bfp", + "region": 2 + }, { + "name": "Bebele", + "id": "beb", + "region": 2 + }, { + "name": "Bebil", + "id": "bxp", + "region": 2 + }, { + "name": "Bedjond", + "id": "bjv", + "region": 2 + }, { + "name": "Beeke (Latin)", + "id": "bkf-latn", + "region": 2 + }, { + "name": "Beezen", + "id": "bnz", + "region": 2 + }, { + "name": "Befang", + "id": "bby", + "region": 2 + }, { + "name": "Bekwel", + "id": "bkw", + "region": 2 + }, { + "name": "Belarusian", + "id": "be", + "region": 4 + }, { + "name": "Belarusian (Braille)", + "id": "be-brai", + "region": 4 + }, { + "name": "Belize Kriol English", + "id": "bzj", + "region": 6 + }, { + "name": "Bella Coola (Latin)", + "id": "blc-latn", + "region": 6 + }, { + "name": "Bemba (Braille)", + "id": "bem-brai", + "region": 2 + }, { + "name": "Bembe (Latin)", + "id": "bmb-latn", + "region": 2 + }, { + "name": "Bena (Latin)", + "id": "bez-latn", + "region": 2 + }, { + "name": "Bench (Ethiopic)", + "id": "bcq-ethi", + "region": 2 + }, { + "name": "Bench (Latin)", + "id": "bcq-latn", + "region": 2 + }, { + "name": "Bende (Latin)", + "id": "bdp-latn", + "region": 2 + }, { + "name": "Bendi (Latin)", + "id": "bct-latn", + "region": 2 + }, { + "name": "Bengali", + "id": "bn", + "region": 3 + }, { + "name": "Bengali (Braille)", + "id": "bn-brai", + "region": 3 + }, { + "name": "Bengali (Latin)", + "id": "bn-latn", + "region": 3 + }, { + "name": "Berakou", + "id": "bxv", + "region": 2 + }, { + "name": "Berom (Latin)", + "id": "bom-latn", + "region": 2 + }, { + "name": "Berta (Latin)", + "id": "wti-latn", + "region": 2 + }, { + "name": "Besme", + "id": "bes", + "region": 2 + }, { + "name": "bh-deva (Devanagari)", + "id": "bh-deva", + "region": 1 + }, { + "name": "Bhele (Latin)", + "id": "bhy-latn", + "region": 2 + }, { + "name": "bhk-latn (Latin)", + "id": "bhk-latn", + "region": 1 + }, { + "name": "Bhojpuri (Devanagari)", + "id": "bho-deva", + "region": 3 + }, { + "name": "Biali (Latin)", + "id": "beh-latn", + "region": 2 + }, { + "name": "Bidiyo", + "id": "bid", + "region": 2 + }, { + "name": "Bikol (Braille)", + "id": "bik-brai", + "region": 1 + }, { + "name": "Bikol (Latin)", + "id": "bik-latn", + "region": 1 + }, { + "name": "Bikol, Buhi\u2019non (Latin)", + "id": "ubl-latn", + "region": 3 + }, { + "name": "Bikol, Central (Braille)", + "id": "bcl-brai", + "region": 3 + }, { + "name": "Bikol, Central (Latin)", + "id": "bcl-latn", + "region": 3 + }, { + "name": "Bikol, Rinconada (Latin)", + "id": "bto-latn", + "region": 3 + }, { + "name": "Bikol, West Albay (Latin)", + "id": "fbl-latn", + "region": 3 + }, { + "name": "Bikya", + "id": "byb", + "region": 2 + }, { + "name": "Bila (Latin)", + "id": "bip-latn", + "region": 2 + }, { + "name": "Bilen (Ethiopic)", + "id": "byn-ethi", + "region": 2 + }, { + "name": "Binukid (Latin)", + "id": "bkd-latn", + "region": 3 + }, { + "name": "Bira (Latin)", + "id": "brf-latn", + "region": 2 + }, { + "name": "Birgit", + "id": "btf", + "region": 2 + }, { + "name": "Birifor, Malba (Latin)", + "id": "bfo-latn", + "region": 2 + }, { + "name": "Bisa (Latin)", + "id": "bib-latn", + "region": 2 + }, { + "name": "Bis\u00e3 (Latin)", + "id": "bqp-latn", + "region": 2 + }, { + "name": "Bishuo", + "id": "bwh", + "region": 2 + }, { + "name": "Bislama", + "id": "bi", + "region": 7 + }, { + "name": "Bislama (Latin)", + "id": "bi-latn", + "region": 7 + }, { + "name": "Bitare", + "id": "brt", + "region": 2 + }, { + "name": "Blaan, Koronadal (Latin)", + "id": "bpr-latn", + "region": 3 + }, { + "name": "Blaan, Sarangani (Latin)", + "id": "bps-latn", + "region": 3 + }, { + "name": "Blackfoot (Latin)", + "id": "bla-latn", + "region": 6 + }, { + "name": "Blang (Tai Le)", + "id": "blr-tale", + "region": 3 + }, { + "name": "Bobo Madar\u00e9, Southern (Latin)", + "id": "bwq-latn", + "region": 2 + }, { + "name": "Bodo Parja (Oriya)", + "id": "bdv-orya", + "region": 3 + }, { + "name": "Bogkalot (Latin)", + "id": "ilk-latn", + "region": 3 + }, { + "name": "Boguru (Latin)", + "id": "bqu-latn", + "region": 2 + }, { + "name": "Bohtan Neo-Aramaic (Syriac)", + "id": "bhn-syrc", + "region": 3 + }, { + "name": "Boko (Latin)", + "id": "bqc-latn", + "region": 2 + }, { + "name": "Bokobaru (Latin)", + "id": "bus-latn", + "region": 2 + }, { + "name": "Bokyi", + "id": "bky", + "region": 2 + }, { + "name": "Bolgo", + "id": "bvo", + "region": 2 + }, { + "name": "Bolinao (Latin)", + "id": "smk-latn", + "region": 3 + }, { + "name": "Bolon (Latin)", + "id": "bof-latn", + "region": 2 + }, { + "name": "Bom-Kim (Latin)", + "id": "bmf-latn", + "region": 2 + }, { + "name": "Bomu (Latin)", + "id": "bmq-latn", + "region": 2 + }, { + "name": "Bomwali", + "id": "bmw", + "region": 2 + }, { + "name": "Bon Gula", + "id": "glc", + "region": 2 + }, { + "name": "Bondei (Latin)", + "id": "bou-latn", + "region": 2 + }, { + "name": "Bondo (Oriya)", + "id": "bfw-orya", + "region": 3 + }, { + "name": "Bonkeng", + "id": "bvg", + "region": 2 + }, { + "name": "Bontok (Latin)", + "id": "bnc-latn", + "region": 1 + }, { + "name": "Bontok, Central (Latin)", + "id": "lbk-latn", + "region": 3 + }, { + "name": "Bontok, Eastern (Latin)", + "id": "ebk-latn", + "region": 3 + }, { + "name": "Boor", + "id": "bvf", + "region": 2 + }, { + "name": "Borna (Latin)", + "id": "bwo-latn", + "region": 2 + }, { + "name": "Bosnian", + "id": "bs", + "region": 4 + }, { + "name": "Bosnian (Cyrillic)", + "id": "bs-cyrl", + "region": 4 + }, { + "name": "Bote", + "id": "bmj", + "region": 3 + }, { + "name": "Bozo, Jenaama (Latin)", + "id": "bze-latn", + "region": 2 + }, { + "name": "Bozo, Kelengaxo (Latin)", + "id": "bzx-latn", + "region": 2 + }, { + "name": "Bozo, Tiemac\u00e8w\u00e8 (Latin)", + "id": "boo-latn", + "region": 2 + }, { + "name": "Bozo, Tieyaxo (Latin)", + "id": "boz-latn", + "region": 2 + }, { + "name": "Brahui", + "id": "brh", + "region": 3 + }, { + "name": "Brahui (Arabic)", + "id": "brh-arab", + "region": 3 + }, { + "name": "Brahui (Latin)", + "id": "brh-latn", + "region": 3 + }, { + "name": "Braj Bhasha (Devanagari)", + "id": "bra-deva", + "region": 3 + }, { + "name": "Breton", + "id": "br", + "region": 4 + }, { + "name": "Bru, Eastern (Latin)", + "id": "bru-latn", + "region": 3 + }, { + "name": "Brunei (Latin)", + "id": "kxd-latn", + "region": 3 + }, { + "name": "btk-batk (Batak)", + "id": "btk-batk", + "region": 1 + }, { + "name": "Bua", + "id": "bub", + "region": 2 + }, { + "name": "Buamu (Latin)", + "id": "box-latn", + "region": 2 + }, { + "name": "Bubia", + "id": "bbx", + "region": 2 + }, { + "name": "Budu (Latin)", + "id": "buu-latn", + "region": 2 + }, { + "name": "Buduma", + "id": "bdm", + "region": 2 + }, { + "name": "Bugis (Buginese)", + "id": "bug-bugi", + "region": 3 + }, { + "name": "Bugis (Latin)", + "id": "bug-latn", + "region": 3 + }, { + "name": "Buhid (Hanunoo)", + "id": "bku-hano", + "region": 3 + }, { + "name": "Buhid (Latin)", + "id": "bku-latn", + "region": 3 + }, { + "name": "Bulgarian", + "id": "bg", + "region": 4 + }, { + "name": "Bulgarian (Braille)", + "id": "bg-brai", + "region": 4 + }, { + "name": "Bulgarian (Latin)", + "id": "bg-latn", + "region": 4 + }, { + "name": "Bullom So (Latin)", + "id": "buy-latn", + "region": 2 + }, { + "name": "Bulu", + "id": "bum", + "region": 2 + }, { + "name": "Bum", + "id": "bmv", + "region": 2 + }, { + "name": "Bumthangkha (Tibetan)", + "id": "kjz-tibt", + "region": 3 + }, { + "name": "Bung", + "id": "bqd", + "region": 2 + }, { + "name": "Bungu (Latin)", + "id": "wun-latn", + "region": 2 + }, { + "name": "Burak (Latin)", + "id": "bys-latn", + "region": 2 + }, { + "name": "Bura-Pabir (Latin)", + "id": "bwr-latn", + "region": 2 + }, { + "name": "Burmese", + "id": "my", + "region": 3 + }, { + "name": "Burmese (Braille)", + "id": "my-brai", + "region": 3 + }, { + "name": "Burushaski (Arabic)", + "id": "bsk-arab", + "region": 3 + }, { + "name": "Burushaski (Latin)", + "id": "bsk-latn", + "region": 3 + }, { + "name": "Busam", + "id": "bxs", + "region": 2 + }, { + "name": "Buso", + "id": "bso", + "region": 2 + }, { + "name": "Busuu", + "id": "bju", + "region": 2 + }, { + "name": "Buwal", + "id": "bhs", + "region": 2 + }, { + "name": "Buyu (Latin)", + "id": "byi-latn", + "region": 2 + }, { + "name": "Bwa (Latin)", + "id": "bww-latn", + "region": 2 + }, { + "name": "Bwamu, Cwi (Latin)", + "id": "bwy-latn", + "region": 2 + }, { + "name": "Bwamu, L\u00e1\u00e1 L\u00e1\u00e1 (Latin)", + "id": "bwj-latn", + "region": 2 + }, { + "name": "Bwela (Latin)", + "id": "bwl-latn", + "region": 2 + }, { + "name": "Byep", + "id": "mkk", + "region": 2 + }, { + "name": "Cab\u00e9car (Latin)", + "id": "cjp-latn", + "region": 6 + }, { + "name": "Caka", + "id": "ckx", + "region": 2 + }, { + "name": "Cakfem-Mushere (Latin)", + "id": "cky-latn", + "region": 2 + }, { + "name": "Caluyanun (Latin)", + "id": "clu-latn", + "region": 3 + }, { + "name": "Capiznon (Latin)", + "id": "cps-latn", + "region": 3 + }, { + "name": "Cappadocian Greek (Greek)", + "id": "cpg-grek", + "region": 4 + }, { + "name": "Carrier, Southern (Unified Canadian Aboriginal Syllabics)", + "id": "caf-cans", + "region": 6 + }, { + "name": "Carrier, Southern (Latin)", + "id": "caf-latn", + "region": 6 + }, { + "name": "Catalan", + "id": "ca", + "region": 4 + }, { + "name": "Cavine\u00f1a", + "id": "cav", + "region": 6 + }, { + "name": "Cayuga (Latin)", + "id": "cay-latn", + "region": 6 + }, { + "name": "Cebuano (Braille)", + "id": "ceb-brai", + "region": 3 + }, { + "name": "Cebuano (Latin)", + "id": "ceb-latn", + "region": 3 + }, { + "name": "Cerma (Latin)", + "id": "cme-latn", + "region": 2 + }, { + "name": "Ch\u00e1cobo", + "id": "cao", + "region": 6 + }, { + "name": "Chadian Sign Language", + "id": "cds", + "region": 2 + }, { + "name": "Chakma (Chakma, Bangladesh)", + "id": "ccp-cakm-bd", + "region": 3 + }, { + "name": "Chaldean Neo-Aramaic (Syriac)", + "id": "cld-syrc", + "region": 3 + }, { + "name": "Chamling", + "id": "rab", + "region": 3 + }, { + "name": "Changthang (Tibetan)", + "id": "cna-tibt", + "region": 3 + }, { + "name": "Chavacano (Braille)", + "id": "cbk-brai", + "region": 3 + }, { + "name": "Chavacano (Latin)", + "id": "cbk-latn", + "region": 3 + }, { + "name": "Chechen (Latin)", + "id": "ce-latn", + "region": 4 + }, { + "name": "Chepang", + "id": "cdm", + "region": 3 + }, { + "name": "Chepang (Devanagari)", + "id": "cdm-deva", + "region": 3 + }, { + "name": "Cherokee (Cherokee)", + "id": "chr-cher", + "region": 6 + }, { + "name": "Cheyenne", + "id": "chy", + "region": 6 + }, { + "name": "Cheyenne (Latin)", + "id": "chy-latn", + "region": 6 + }, { + "name": "Chhattisgarhi (Devanagari)", + "id": "hne-deva", + "region": 3 + }, { + "name": "Chichewa (Braille)", + "id": "ny-brai", + "region": 2 + }, { + "name": "Chiga (Latin)", + "id": "cgg-latn", + "region": 2 + }, { + "name": "Chilcotin (Latin)", + "id": "clc-latn", + "region": 6 + }, { + "name": "Chinese (Braille)", + "id": "zh-brai", + "region": 1 + }, { + "name": "Chinese (Phags-pa, China)", + "id": "zh-phag-cn", + "region": 1 + }, { + "name": "Chinese, Mandarin (Braille)", + "id": "cmn-brai", + "region": 3 + }, { + "name": "Chinese, Mandarin (Han (Simplified variant))", + "id": "cmn-hans", + "region": 3 + }, { + "name": "Chinese, Min Nan (Latin, Taiwan, Province of China)", + "id": "nan-latn-tw", + "region": 3 + }, { + "name": "Chinese, Yue (Braille)", + "id": "yue-brai", + "region": 3 + }, { + "name": "Chinook Wawa", + "id": "chn", + "region": 6 + }, { + "name": "Chipaya", + "id": "cap", + "region": 6 + }, { + "name": "Chiquitano", + "id": "cax", + "region": 6 + }, { + "name": "Choctaw", + "id": "cho", + "region": 6 + }, { + "name": "Cimbrian", + "id": "cim", + "region": 4 + }, { + "name": "Clallam (Latin)", + "id": "clm-latn", + "region": 6 + }, { + "name": "Classical Nahuatl", + "id": "nci", + "region": 1 + }, { + "name": "Classical Quechua", + "id": "qwc", + "region": 1 + }, { + "name": "Classical Tibetan (Tibetan)", + "id": "xct-tibt", + "region": 1 + }, { + "name": "Comox (Latin)", + "id": "coo-latn", + "region": 6 + }, { + "name": "Cook Islands Maori (Latin)", + "id": "rar-latn", + "region": 7 + }, { + "name": "Coos", + "id": "csz", + "region": 6 + }, { + "name": "Coptic (Coptic)", + "id": "cop-copt", + "region": 2 + }, { + "name": "Coptic (Greek)", + "id": "cop-grek", + "region": 2 + }, { + "name": "Cornish", + "id": "kw", + "region": 4 + }, { + "name": "Corsican", + "id": "co", + "region": 4 + }, { + "name": "Cree", + "id": "cr", + "region": 1 + }, { + "name": "Cree (Latin)", + "id": "cr-latn", + "region": 1 + }, { + "name": "Cree, Moose (Unified Canadian Aboriginal Syllabics)", + "id": "crm-cans", + "region": 6 + }, { + "name": "Cree, Northern East", + "id": "crl", + "region": 6 + }, { + "name": "Cree, Northern East (Unified Canadian Aboriginal Syllabics)", + "id": "crl-cans", + "region": 6 + }, { + "name": "Cree, Northern East (Latin)", + "id": "crl-latn", + "region": 6 + }, { + "name": "Cree, Plains (Unified Canadian Aboriginal Syllabics)", + "id": "crk-cans", + "region": 6 + }, { + "name": "Cree, Plains (Latin)", + "id": "crk-latn", + "region": 6 + }, { + "name": "Cree, Southern East", + "id": "crj", + "region": 6 + }, { + "name": "Cree, Swampy", + "id": "csw", + "region": 6 + }, { + "name": "Cree, Swampy (Unified Canadian Aboriginal Syllabics)", + "id": "csw-cans", + "region": 6 + }, { + "name": "Crimean Tatar (Latin)", + "id": "crh-latn", + "region": 4 + }, { + "name": "Croatian", + "id": "hr", + "region": 4 + }, { + "name": "Croatian (Braille)", + "id": "hr-brai", + "region": 4 + }, { + "name": "cug", + "id": "cug", + "region": 1 + }, { + "name": "Cuvok", + "id": "cuv", + "region": 2 + }, { + "name": "Cuyonon (Latin)", + "id": "cyo-latn", + "region": 3 + }, { + "name": "Czech", + "id": "cs", + "region": 4 + }, { + "name": "Czech (Braille)", + "id": "cs-brai", + "region": 4 + }, { + "name": "Daba", + "id": "dbq", + "region": 2 + }, { + "name": "Dagaare, Southern (Latin)", + "id": "dga-latn", + "region": 2 + }, { + "name": "Dagaari Dioula (Latin)", + "id": "dgd-latn", + "region": 2 + }, { + "name": "Dagara, Northern (Latin)", + "id": "dgi-latn", + "region": 2 + }, { + "name": "Dagba", + "id": "dgk", + "region": 2 + }, { + "name": "Dagbani", + "id": "dag", + "region": 2 + }, { + "name": "Daju, Dar Daju", + "id": "djc", + "region": 2 + }, { + "name": "Daju, Dar Sila", + "id": "dau", + "region": 2 + }, { + "name": "Dakota (Latin)", + "id": "dak-latn", + "region": 6 + }, { + "name": "Dama", + "id": "dmm", + "region": 2 + }, { + "name": "Dameli (Arabic)", + "id": "dml-arab", + "region": 3 + }, { + "name": "Dan (Latin, Liberia)", + "id": "dnj-latn-lr", + "region": 2 + }, { + "name": "Dangal\u00e9at", + "id": "daa", + "region": 2 + }, { + "name": "Dangme (Latin)", + "id": "ada-latn", + "region": 2 + }, { + "name": "Danish", + "id": "da", + "region": 4 + }, { + "name": "Danish (Braille)", + "id": "da-brai", + "region": 4 + }, { + "name": "Danish (Runic)", + "id": "da-runr", + "region": 4 + }, { + "name": "Darai", + "id": "dry", + "region": 3 + }, { + "name": "Dari", + "id": "prs", + "region": 3 + }, { + "name": "Day", + "id": "dai", + "region": 2 + }, { + "name": "Dazaga", + "id": "dzg", + "region": 2 + }, { + "name": "Dek", + "id": "dek", + "region": 2 + }, { + "name": "Dene", + "id": "chp", + "region": 6 + }, { + "name": "Dene (Unified Canadian Aboriginal Syllabics)", + "id": "chp-cans", + "region": 6 + }, { + "name": "Dene (Latin)", + "id": "chp-latn", + "region": 6 + }, { + "name": "Denya", + "id": "anv", + "region": 2 + }, { + "name": "Desiya (Oriya)", + "id": "dso-orya", + "region": 3 + }, { + "name": "Dewas Rai", + "id": "dwz", + "region": 3 + }, { + "name": "Dewoin (Latin)", + "id": "dee-latn", + "region": 2 + }, { + "name": "Dhaiso (Latin)", + "id": "dhs-latn", + "region": 2 + }, { + "name": "Dhangu-Djangu (Latin)", + "id": "dhg-latn", + "region": 7 + }, { + "name": "Dhatki (Arabic)", + "id": "mki-arab", + "region": 3 + }, { + "name": "Dhimal", + "id": "dhi", + "region": 3 + }, { + "name": "Dhuwal (Latin)", + "id": "dwu-latn", + "region": 7 + }, { + "name": "Dii", + "id": "dur", + "region": 2 + }, { + "name": "Dimbong", + "id": "dii", + "region": 2 + }, { + "name": "Dinka (Latin)", + "id": "din-latn", + "region": 1 + }, { + "name": "Dinka, Northeastern (Latin)", + "id": "dip-latn", + "region": 2 + }, { + "name": "Dinka, Northwestern (Latin)", + "id": "diw-latn", + "region": 2 + }, { + "name": "Dinka, South Central (Latin)", + "id": "dib-latn", + "region": 2 + }, { + "name": "Dinka, Southeastern (Latin)", + "id": "dks-latn", + "region": 2 + }, { + "name": "Dinka, Southwestern (Latin)", + "id": "dik-latn", + "region": 2 + }, { + "name": "Dirasha (Ethiopic)", + "id": "gdl-ethi", + "region": 2 + }, { + "name": "Disa", + "id": "dsi", + "region": 2 + }, { + "name": "Ditidaht (Latin)", + "id": "dtd-latn", + "region": 6 + }, { + "name": "Dizin", + "id": "mdx", + "region": 2 + }, { + "name": "Dizin (Ethiopic)", + "id": "mdx-ethi", + "region": 2 + }, { + "name": "Dizin (Latin)", + "id": "mdx-latn", + "region": 2 + }, { + "name": "Doe (Latin)", + "id": "doe-latn", + "region": 2 + }, { + "name": "Dogon, Bondum Dom (Latin)", + "id": "dbu-latn", + "region": 2 + }, { + "name": "Dogon, Donno So (Latin)", + "id": "dds-latn", + "region": 2 + }, { + "name": "Dogon, Jamsay (Latin)", + "id": "djm-latn", + "region": 2 + }, { + "name": "Dogon, Tene Kan (Latin)", + "id": "dtk-latn", + "region": 2 + }, { + "name": "Dogon, Tomo Kan (Latin)", + "id": "dtm-latn", + "region": 2 + }, { + "name": "Dogon, Toro So (Latin)", + "id": "dts-latn", + "region": 2 + }, { + "name": "Dogon, Toro Tegu (Latin)", + "id": "dtt-latn", + "region": 2 + }, { + "name": "Dogoso (Latin)", + "id": "dgs-latn", + "region": 2 + }, { + "name": "Dogri (Arabic, Pakistan)", + "id": "dgo-arab-pk", + "region": 3 + }, { + "name": "Dogri (macrolanguage) (Dogra)", + "id": "doi-dogr", + "region": 1 + }, { + "name": "Dolpo (Tibetan)", + "id": "dre-tibt", + "region": 3 + }, { + "name": "Dongo (Latin)", + "id": "doo-latn", + "region": 2 + }, { + "name": "Dotyali", + "id": "dty", + "region": 3 + }, { + "name": "Doyayo", + "id": "dow", + "region": 2 + }, { + "name": "Drents", + "id": "drt", + "region": 4 + }, { + "name": "Duala", + "id": "dua", + "region": 2 + }, { + "name": "Dugun", + "id": "ndu", + "region": 2 + }, { + "name": "Dugwor", + "id": "dme", + "region": 2 + }, { + "name": "Duli-Gey", + "id": "duz", + "region": 2 + }, { + "name": "Duruwa (Oriya)", + "id": "pci-orya", + "region": 3 + }, { + "name": "Dutch", + "id": "nl", + "region": 4 + }, { + "name": "Dutch (Braille)", + "id": "nl-brai", + "region": 4 + }, { + "name": "Duungooma (Latin)", + "id": "dux-latn", + "region": 2 + }, { + "name": "Duupa", + "id": "dae", + "region": 2 + }, { + "name": "Duya", + "id": "ldb", + "region": 2 + }, { + "name": "Duya (Latin)", + "id": "ldb-latn", + "region": 2 + }, { + "name": "Dyan (Latin)", + "id": "dya-latn", + "region": 2 + }, { + "name": "Dza (Latin)", + "id": "jen-latn", + "region": 2 + }, { + "name": "Dzongkha", + "id": "dz", + "region": 3 + }, { + "name": "Dzongkha (Bhutan)", + "id": "dz-bt", + "region": 3 + }, { + "name": "Dz\u00f9\u00f9ngoo (Latin)", + "id": "dnn-latn", + "region": 2 + }, { + "name": "Eastern Franconian", + "id": "vmf", + "region": 4 + }, { + "name": "Ebira (Latin)", + "id": "igb-latn", + "region": 2 + }, { + "name": "Edo", + "id": "bin", + "region": 2 + }, { + "name": "Edo (Latin)", + "id": "bin-latn", + "region": 2 + }, { + "name": "Efe (Latin)", + "id": "efe-latn", + "region": 2 + }, { + "name": "Efik", + "id": "efi", + "region": 2 + }, { + "name": "Efik (Latin)", + "id": "efi-latn", + "region": 2 + }, { + "name": "Egyptian (Ancient) (Egyptian hieroglyphs)", + "id": "egy-egyp", + "region": 1 + }, { + "name": "Ejagham", + "id": "etu", + "region": 2 + }, { + "name": "Elip", + "id": "ekm", + "region": 2 + }, { + "name": "Eman", + "id": "emn", + "region": 2 + }, { + "name": "Ember\u00e1, Northern (Latin)", + "id": "emp-latn", + "region": 6 + }, { + "name": "Emilian", + "id": "egl", + "region": 4 + }, { + "name": "Enga (Latin)", + "id": "enq-latn", + "region": 7 + }, { + "name": "Engenni (Latin)", + "id": "enn-latn", + "region": 2 + }, { + "name": "Enggano", + "id": "eno", + "region": 3 + }, { + "name": "English", + "id": "en", + "region": 4 + }, { + "name": "English (Australia)", + "id": "en-au", + "region": 4 + }, { + "name": "English (Braille)", + "id": "en-brai", + "region": 4 + }, { + "name": "English", + "id": "en-qp", + "region": 4 + }, { + "name": "English (Runic)", + "id": "en-runr", + "region": 4 + }, { + "name": "English (Shavian)", + "id": "en-shaw", + "region": 4 + }, { + "name": "English (Shavian)", + "id": "en-shaw-uk", + "region": 4 + }, { + "name": "English (Shavian, United States)", + "id": "en-shaw-us", + "region": 4 + }, { + "name": "Enya (Latin)", + "id": "gey-latn", + "region": 2 + }, { + "name": "Ese Ejja", + "id": "ese", + "region": 6 + }, { + "name": "Esimbi", + "id": "ags", + "region": 2 + }, { + "name": "Esperanto", + "id": "eo", + "region": 4 + }, { + "name": "Estonian", + "id": "et", + "region": 1 + }, { + "name": "Estonian (Braille)", + "id": "et-brai", + "region": 1 + }, { + "name": "Eton", + "id": "eto", + "region": 2 + }, { + "name": "Evant", + "id": "bzz", + "region": 2 + }, { + "name": "\u00c9w\u00e9 (Braille)", + "id": "ee-brai", + "region": 2 + }, { + "name": "\u00c9w\u00e9 (Latin)", + "id": "ee-latn", + "region": 2 + }, { + "name": "Ewondo", + "id": "ewo", + "region": 2 + }, { + "name": "Extremaduran", + "id": "ext", + "region": 4 + }, { + "name": "Fala", + "id": "fax", + "region": 4 + }, { + "name": "Fali, North", + "id": "fll", + "region": 2 + }, { + "name": "Fali, South", + "id": "fal", + "region": 2 + }, { + "name": "Fang", + "id": "fak", + "region": 2 + }, { + "name": "Fang", + "id": "fan", + "region": 2 + }, { + "name": "Fania", + "id": "fni", + "region": 2 + }, { + "name": "Faroese", + "id": "fo", + "region": 4 + }, { + "name": "Faroese (Runic)", + "id": "fo-runr", + "region": 4 + }, { + "name": "Fe\u2019fe\u2019", + "id": "fmp", + "region": 2 + }, { + "name": "Fijian", + "id": "fj", + "region": 7 + }, { + "name": "Filipino", + "id": "fil", + "region": 3 + }, { + "name": "Filipino (Latin)", + "id": "fil-latn", + "region": 3 + }, { + "name": "Finnish", + "id": "fi", + "region": 4 + }, { + "name": "Finnish (Braille)", + "id": "fi-brai", + "region": 4 + }, { + "name": "Finnish, Kven", + "id": "fkv", + "region": 4 + }, { + "name": "Finnish, Tornedalen", + "id": "fit", + "region": 4 + }, { + "name": "Finongan (Latin)", + "id": "fag-latn", + "region": 7 + }, { + "name": "Fipa (Latin)", + "id": "fip-latn", + "region": 2 + }, { + "name": "Flaaitaal", + "id": "fly", + "region": 2 + }, { + "name": "Foma (Latin)", + "id": "fom-latn", + "region": 2 + }, { + "name": "Fon (Latin)", + "id": "fon-latn", + "region": 2 + }, { + "name": "Fongoro", + "id": "fgr", + "region": 2 + }, { + "name": "Frankish", + "id": "frk", + "region": 1 + }, { + "name": "French", + "id": "fr", + "region": 4 + }, { + "name": "French (Braille)", + "id": "fr-brai", + "region": 4 + }, { + "name": "French (Canada)", + "id": "fr-ca", + "region": 4 + }, { + "name": "French (Switzerland)", + "id": "fr-ch", + "region": 4 + }, { + "name": "French, Cajun", + "id": "frc", + "region": 6 + }, { + "name": "Frisian", + "id": "fy", + "region": 4 + }, { + "name": "Frisian, Northern", + "id": "frr", + "region": 4 + }, { + "name": "Friulian", + "id": "fur", + "region": 4 + }, { + "name": "Fulah", + "id": "ff", + "region": 1 + }, { + "name": "Fulah (Adlam)", + "id": "ff-adlm", + "region": 1 + }, { + "name": "Fulah (Arabic)", + "id": "ff-arab", + "region": 1 + }, { + "name": "Fulah (Latin)", + "id": "ff-latn", + "region": 1 + }, { + "name": "Fulfulde, Adamawa", + "id": "fub", + "region": 2 + }, { + "name": "Fulfulde, Adamawa (Arabic)", + "id": "fub-arab", + "region": 2 + }, { + "name": "Fulfulde, Adamawa (Latin)", + "id": "fub-latn", + "region": 2 + }, { + "name": "Fulfulde, Bagirmi", + "id": "fui", + "region": 2 + }, { + "name": "Fulfulde, Borgu", + "id": "fue", + "region": 2 + }, { + "name": "Fulfulde, Borgu (Latin)", + "id": "fue-latn", + "region": 2 + }, { + "name": "Fulfulde, Central-Eastern Niger", + "id": "fuq", + "region": 2 + }, { + "name": "Fulfulde, Maasina", + "id": "ffm", + "region": 2 + }, { + "name": "Fulfulde, Maasina (Arabic)", + "id": "ffm-arab", + "region": 2 + }, { + "name": "Fulfulde, Maasina (Latin)", + "id": "ffm-latn", + "region": 2 + }, { + "name": "Fulfulde, Nigerian", + "id": "fuv", + "region": 2 + }, { + "name": "Fulfulde, Nigerian (Arabic)", + "id": "fuv-arab", + "region": 2 + }, { + "name": "Fulfulde, Nigerian (Latin)", + "id": "fuv-latn", + "region": 2 + }, { + "name": "Fulfulde, Western Niger (Arabic)", + "id": "fuh-arab", + "region": 2 + }, { + "name": "Fulfulde, Western Niger (Latin)", + "id": "fuh-latn", + "region": 2 + }, { + "name": "Fuliiru (Latin)", + "id": "flr-latn", + "region": 2 + }, { + "name": "Fur", + "id": "fvr", + "region": 2 + }, { + "name": "Ga\u2019dang (Latin)", + "id": "gdg-latn", + "region": 3 + }, { + "name": "Gabri", + "id": "gab", + "region": 2 + }, { + "name": "Gadaba, Bodo (Oriya)", + "id": "gbj-orya", + "region": 3 + }, { + "name": "Gadaba, Pottangi Ollar (Oriya)", + "id": "gdb-orya", + "region": 3 + }, { + "name": "Gadang", + "id": "gdk", + "region": 2 + }, { + "name": "Gagauz", + "id": "gag", + "region": 4 + }, { + "name": "Gahri (Tibetan)", + "id": "bfu-tibt", + "region": 3 + }, { + "name": "Galician", + "id": "gl", + "region": 4 + }, { + "name": "Gamo (Latin)", + "id": "gmv-latn", + "region": 2 + }, { + "name": "Ganda (Latin)", + "id": "lg-latn", + "region": 2 + }, { + "name": "G\u0101ndh\u0101r\u012b (Latin)", + "id": "pgd-latn", + "region": 1 + }, { + "name": "Garo (Braille)", + "id": "grt-brai", + "region": 3 + }, { + "name": "Gavar", + "id": "gou", + "region": 2 + }, { + "name": "Gawar-Bati (Arabic)", + "id": "gwt-arab", + "region": 3 + }, { + "name": "Gawri (Arabic)", + "id": "gwc-arab", + "region": 3 + }, { + "name": "Gbaya (Central African Republic)", + "id": "gba", + "region": 1 + }, { + "name": "Gbaya, Northwest", + "id": "gya", + "region": 2 + }, { + "name": "Gbaya-Mbodomo", + "id": "gmm", + "region": 2 + }, { + "name": "Gbii (Latin)", + "id": "ggb-latn", + "region": 2 + }, { + "name": "Gedeo", + "id": "drs", + "region": 2 + }, { + "name": "Geez", + "id": "gez", + "region": 2 + }, { + "name": "Geez (Ethiopic)", + "id": "gez-ethi", + "region": 2 + }, { + "name": "Georgian", + "id": "ka", + "region": 3 + }, { + "name": "Georgian (Braille)", + "id": "ka-brai", + "region": 3 + }, { + "name": "Gepo (Miao)", + "id": "ygp-plrd", + "region": 3 + }, { + "name": "German, Colonia Tovar", + "id": "gct", + "region": 6 + }, { + "name": "German, Pennsylvania", + "id": "pdc", + "region": 6 + }, { + "name": "German, Standard", + "id": "de", + "region": 4 + }, { + "name": "German, Standard (Braille)", + "id": "de-brai", + "region": 4 + }, { + "name": "German, Standard (Runic)", + "id": "de-runr", + "region": 4 + }, { + "name": "German, Swiss", + "id": "gsw", + "region": 4 + }, { + "name": "Ghale, Southern", + "id": "ghe", + "region": 3 + }, { + "name": "Ghom\u00e1l\u00e1\u2019", + "id": "bbj", + "region": 2 + }, { + "name": "Gidar", + "id": "gid", + "region": 2 + }, { + "name": "Gilaki", + "id": "glk", + "region": 3 + }, { + "name": "Gilaki (Latin)", + "id": "glk-latn", + "region": 3 + }, { + "name": "Gimme", + "id": "kmp", + "region": 2 + }, { + "name": "Gimnime", + "id": "gmn", + "region": 2 + }, { + "name": "Gitxsan (Latin)", + "id": "git-latn", + "region": 6 + }, { + "name": "Giziga", + "id": "giz", + "region": 2 + }, { + "name": "Giziga, North", + "id": "gis", + "region": 2 + }, { + "name": "Glaro-Twabo (Latin)", + "id": "glr-latn", + "region": 2 + }, { + "name": "Glavda", + "id": "glw", + "region": 2 + }, { + "name": "Glio-Oubi (Latin)", + "id": "oub-latn", + "region": 2 + }, { + "name": "Goemai (Latin)", + "id": "ank-latn", + "region": 2 + }, { + "name": "Gofa (Latin)", + "id": "gof-latn", + "region": 2 + }, { + "name": "Gogo (Latin)", + "id": "gog-latn", + "region": 2 + }, { + "name": "Gokana (Latin)", + "id": "gkn-latn", + "region": 2 + }, { + "name": "Gola (Latin)", + "id": "gol-latn", + "region": 2 + }, { + "name": "Gondi (Devanagari)", + "id": "gon-deva", + "region": 1 + }, { + "name": "Gondi (Gunjala Gondi)", + "id": "gon-gong", + "region": 1 + }, { + "name": "Gondi (Masaram Gondi)", + "id": "gon-gonm", + "region": 1 + }, { + "name": "Gondi, Adilabad (Devanagari)", + "id": "wsg-deva", + "region": 3 + }, { + "name": "Gondi, Adilabad (Gunjala Gondi)", + "id": "wsg-gong", + "region": 3 + }, { + "name": "Gondi, Adilabad (Telugu)", + "id": "wsg-telu", + "region": 3 + }, { + "name": "Gondi, Aheri (Devanagari)", + "id": "esg-deva", + "region": 3 + }, { + "name": "Gondi, Northern (Devanagari)", + "id": "gno-deva", + "region": 3 + }, { + "name": "Gor", + "id": "gqr", + "region": 2 + }, { + "name": "Gorontalo (Latin)", + "id": "gor-latn", + "region": 3 + }, { + "name": "Gothic (Gothic)", + "id": "got-goth", + "region": 1 + }, { + "name": "Goundo", + "id": "goy", + "region": 2 + }, { + "name": "Gourmanch\u00e9ma (Latin)", + "id": "gux-latn", + "region": 2 + }, { + "name": "Grebo (Latin)", + "id": "grb-latn", + "region": 1 + }, { + "name": "Grebo, Barclayville (Latin)", + "id": "gry-latn", + "region": 2 + }, { + "name": "Grebo, Central (Latin)", + "id": "grv-latn", + "region": 2 + }, { + "name": "Grebo, Gboloo (Latin)", + "id": "gec-latn", + "region": 2 + }, { + "name": "Grebo, Northern (Latin)", + "id": "gbo-latn", + "region": 2 + }, { + "name": "Grebo, Southern (Latin)", + "id": "grj-latn", + "region": 2 + }, { + "name": "Greek", + "id": "el", + "region": 4 + }, { + "name": "Greek (Braille)", + "id": "el-brai", + "region": 4 + }, { + "name": "Greek (Latin)", + "id": "el-latn", + "region": 4 + }, { + "name": "Greek, Ancient (Greek)", + "id": "grc-grek", + "region": 4 + }, { + "name": "Greek, Ancient (Latin)", + "id": "grc-latn", + "region": 4 + }, { + "name": "Greenlandic", + "id": "kl", + "region": 6 + }, { + "name": "grk", + "id": "grk", + "region": 1 + }, { + "name": "Gronings", + "id": "gos", + "region": 4 + }, { + "name": "Guadeloupean Creole French", + "id": "gcf", + "region": 6 + }, { + "name": "Guarani", + "id": "gn", + "region": 1 + }, { + "name": "Guaran\u00ed, Ava", + "id": "nhd", + "region": 6 + }, { + "name": "Guaran\u00ed, Eastern Bolivian", + "id": "gui", + "region": 6 + }, { + "name": "Guaran\u00ed, Mby\u00e1", + "id": "gun", + "region": 6 + }, { + "name": "Guaran\u00ed, Western Bolivian", + "id": "gnw", + "region": 6 + }, { + "name": "Guarayu", + "id": "gyr", + "region": 6 + }, { + "name": "Gude", + "id": "gde", + "region": 2 + }, { + "name": "Guianese Creole French", + "id": "gcr", + "region": 6 + }, { + "name": "Guinea-Bissau Creole", + "id": "pov", + "region": 2 + }, { + "name": "Gujarati", + "id": "gu", + "region": 3 + }, { + "name": "Gujarati (Braille)", + "id": "gu-brai", + "region": 3 + }, { + "name": "Gujari", + "id": "gju", + "region": 3 + }, { + "name": "Gula", + "id": "glu", + "region": 2 + }, { + "name": "Gula Iro", + "id": "glj", + "region": 2 + }, { + "name": "Gulay", + "id": "gvl", + "region": 2 + }, { + "name": "Gumuz (Ethiopic)", + "id": "guk-ethi", + "region": 2 + }, { + "name": "Gumuz (Latin)", + "id": "guk-latn", + "region": 2 + }, { + "name": "Gungu (Latin)", + "id": "rub-latn", + "region": 2 + }, { + "name": "Gurung", + "id": "gvr", + "region": 3 + }, { + "name": "Gusilay", + "id": "gsl", + "region": 2 + }, { + "name": "Guyanese Creole English", + "id": "gyn", + "region": 6 + }, { + "name": "Gvoko", + "id": "ngs", + "region": 2 + }, { + "name": "Gwama (Latin)", + "id": "kmq-latn", + "region": 2 + }, { + "name": "Gweno (Latin)", + "id": "gwe-latn", + "region": 2 + }, { + "name": "Gwere (Latin)", + "id": "gwr-latn", + "region": 2 + }, { + "name": "Gwich\u2019in", + "id": "gwi", + "region": 6 + }, { + "name": "Gwich\u2019in (Latin)", + "id": "gwi-latn", + "region": 6 + }, { + "name": "Gyele", + "id": "gyi", + "region": 2 + }, { + "name": "Ha (Latin)", + "id": "haq-latn", + "region": 2 + }, { + "name": "Hadiyya (Latin)", + "id": "hdy-latn", + "region": 2 + }, { + "name": "Haida, Southern (Latin)", + "id": "hax-latn", + "region": 6 + }, { + "name": "Haisla (Latin)", + "id": "has-latn", + "region": 6 + }, { + "name": "Haitian Creole", + "id": "ht", + "region": 6 + }, { + "name": "Halkomelem (Latin)", + "id": "hur-latn", + "region": 6 + }, { + "name": "Hamba (Latin)", + "id": "hba-latn", + "region": 2 + }, { + "name": "Hamer-Banna", + "id": "amf", + "region": 2 + }, { + "name": "Hamer-Banna (Latin)", + "id": "amf-latn", + "region": 2 + }, { + "name": "Han (Latin)", + "id": "haa-latn", + "region": 6 + }, { + "name": "Hanga (Latin)", + "id": "hag-latn", + "region": 2 + }, { + "name": "Hangaza (Latin)", + "id": "han-latn", + "region": 2 + }, { + "name": "Hanunoo (Hanunoo)", + "id": "hnn-hano", + "region": 3 + }, { + "name": "Hanunoo (Latin)", + "id": "hnn-latn", + "region": 3 + }, { + "name": "Harari", + "id": "har", + "region": 2 + }, { + "name": "Haroi (Latin)", + "id": "hro-latn", + "region": 3 + }, { + "name": "Haryanvi (Devanagari)", + "id": "bgc-deva", + "region": 3 + }, { + "name": "Hassaniyya", + "id": "mey", + "region": 2 + }, { + "name": "Hausa", + "id": "ha", + "region": 2 + }, { + "name": "Hausa (Arabic)", + "id": "ha-arab", + "region": 2 + }, { + "name": "Hausa (Braille)", + "id": "ha-brai", + "region": 2 + }, { + "name": "Hausa (Latin)", + "id": "ha-latn", + "region": 2 + }, { + "name": "Havu (Latin)", + "id": "hav-latn", + "region": 2 + }, { + "name": "Hawai\u2019i Pidgin", + "id": "hwc", + "region": 6 + }, { + "name": "Hawaiian (Latin)", + "id": "haw-latn", + "region": 6 + }, { + "name": "Hawaiian (Latin, United States)", + "id": "haw-latn-us", + "region": 6 + }, { + "name": "Haya (Latin)", + "id": "hay-latn", + "region": 2 + }, { + "name": "Hazaragi (Arabic)", + "id": "haz-arab", + "region": 3 + }, { + "name": "Hdi", + "id": "xed", + "region": 2 + }, { + "name": "Hebrew", + "id": "he", + "region": 3 + }, { + "name": "Hebrew (Braille)", + "id": "he-brai", + "region": 3 + }, { + "name": "Hebrew (Latin)", + "id": "he-latn", + "region": 3 + }, { + "name": "Hebrew, Ancient (Hebrew)", + "id": "hbo-hebr", + "region": 3 + }, { + "name": "Hebrew, Ancient (Latin)", + "id": "hbo-latn", + "region": 3 + }, { + "name": "Hehe (Latin)", + "id": "heh-latn", + "region": 2 + }, { + "name": "Heiltsuk", + "id": "hei", + "region": 6 + }, { + "name": "Hema (Latin)", + "id": "nix-latn", + "region": 2 + }, { + "name": "Hemba (Latin)", + "id": "hem-latn", + "region": 2 + }, { + "name": "Herd\u00e9", + "id": "hed", + "region": 2 + }, { + "name": "H\u00e9rtevin (Syriac)", + "id": "hrt-syrc", + "region": 3 + }, { + "name": "Hiberno-Scottish Gaelic", + "id": "ghc", + "region": 1 + }, { + "name": "Higaonon (Latin)", + "id": "mba-latn", + "region": 3 + }, { + "name": "Hijuk", + "id": "hij", + "region": 2 + }, { + "name": "Hiligaynon (Braille)", + "id": "hil-brai", + "region": 3 + }, { + "name": "Hiligaynon (Latin)", + "id": "hil-latn", + "region": 3 + }, { + "name": "Hindi", + "id": "hi", + "region": 3 + }, { + "name": "Hindi (Braille)", + "id": "hi-brai", + "region": 3 + }, { + "name": "Hindi (Latin)", + "id": "hi-latn", + "region": 3 + }, { + "name": "Hindko, Northern", + "id": "hno", + "region": 3 + }, { + "name": "Hindko, Northern (Arabic)", + "id": "hno-arab", + "region": 3 + }, { + "name": "Hindko, Southern (Arabic)", + "id": "hnd-arab", + "region": 3 + }, { + "name": "Ho (Oriya)", + "id": "hoc-orya", + "region": 3 + }, { + "name": "Ho (Warang Citi)", + "id": "hoc-wara", + "region": 3 + }, { + "name": "Holoholo (Latin)", + "id": "hoo-latn", + "region": 2 + }, { + "name": "Hopi", + "id": "hop", + "region": 6 + }, { + "name": "Horo", + "id": "hor", + "region": 2 + }, { + "name": "Humla", + "id": "hut", + "region": 3 + }, { + "name": "Hunde (Latin)", + "id": "hke-latn", + "region": 2 + }, { + "name": "Hungarian", + "id": "hu", + "region": 4 + }, { + "name": "Hungarian (Braille)", + "id": "hu-brai", + "region": 4 + }, { + "name": "Hutterisch", + "id": "geh", + "region": 6 + }, { + "name": "Hya", + "id": "hya", + "region": 2 + }, { + "name": "Hyolmo", + "id": "scp", + "region": 3 + }, { + "name": "Ibaloi (Latin)", + "id": "ibl-latn", + "region": 3 + }, { + "name": "Ibanag (Latin)", + "id": "ibg-latn", + "region": 3 + }, { + "name": "Ibatan (Latin)", + "id": "ivb-latn", + "region": 3 + }, { + "name": "Icelandic", + "id": "is", + "region": 4 + }, { + "name": "Icelandic (Runic)", + "id": "is-runr", + "region": 4 + }, { + "name": "Iceve-Maci", + "id": "bec", + "region": 2 + }, { + "name": "Ido (Latin)", + "id": "io-latn", + "region": 1 + }, { + "name": "Idoma", + "id": "idu", + "region": 2 + }, { + "name": "Idoma (Latin)", + "id": "idu-latn", + "region": 2 + }, { + "name": "Idu-Mishmi (Tibetan)", + "id": "clk-tibt", + "region": 3 + }, { + "name": "Ifugao, Amganad (Latin)", + "id": "ifa-latn", + "region": 3 + }, { + "name": "Ifugao, Batad (Latin)", + "id": "ifb-latn", + "region": 3 + }, { + "name": "Ifugao, Mayoyao (Latin)", + "id": "ifu-latn", + "region": 3 + }, { + "name": "Ifugao, Tuwali (Latin)", + "id": "ifk-latn", + "region": 3 + }, { + "name": "Igbo", + "id": "ig", + "region": 2 + }, { + "name": "Igbo (Latin)", + "id": "ig-latn", + "region": 2 + }, { + "name": "Igede (Latin)", + "id": "ige-latn", + "region": 2 + }, { + "name": "Ignaciano", + "id": "ign", + "region": 6 + }, { + "name": "Ijo, Southeast", + "id": "ijs", + "region": 2 + }, { + "name": "Ik (Latin)", + "id": "ikx-latn", + "region": 2 + }, { + "name": "Ikizu (Latin)", + "id": "ikz-latn", + "region": 2 + }, { + "name": "Ikoma-Nata-Isenye (Latin)", + "id": "ntk-latn", + "region": 2 + }, { + "name": "Ikwere (Latin)", + "id": "ikw-latn", + "region": 2 + }, { + "name": "Ilocano (Braille)", + "id": "ilo-brai", + "region": 3 + }, { + "name": "Ilocano (Latin)", + "id": "ilo-latn", + "region": 3 + }, { + "name": "Inabaknon (Latin)", + "id": "abx-latn", + "region": 3 + }, { + "name": "Indonesian", + "id": "id", + "region": 3 + }, { + "name": "Indonesian (Braille)", + "id": "id-brai", + "region": 3 + }, { + "name": "Indonesian (Javanese)", + "id": "id-java", + "region": 3 + }, { + "name": "Innu (Latin)", + "id": "moe-latn", + "region": 6 + }, { + "name": "Inonhan (Latin)", + "id": "loc-latn", + "region": 3 + }, { + "name": "Inor", + "id": "ior", + "region": 2 + }, { + "name": "Interlingua (International Auxiliary Language Association) (Latin)", + "id": "ia-latn", + "region": 1 + }, { + "name": "Inuinnaqtun", + "id": "ikt", + "region": 6 + }, { + "name": "Inuinnaqtun (Latin)", + "id": "ikt-latn", + "region": 6 + }, { + "name": "Inuktitut (Unified Canadian Aboriginal Syllabics)", + "id": "iu-cans", + "region": 1 + }, { + "name": "Inuktitut (Latin)", + "id": "iu-latn", + "region": 1 + }, { + "name": "Inuktitut, Eastern Canadian (Unified Canadian Aboriginal Syllabics)", + "id": "ike-cans", + "region": 6 + }, { + "name": "Inuktitut, Eastern Canadian (Latin)", + "id": "ike-latn", + "region": 6 + }, { + "name": "Inupiaq", + "id": "ik", + "region": 1 + }, { + "name": "Inupiatun, North Alaskan", + "id": "esi", + "region": 6 + }, { + "name": "Iowa-Oto", + "id": "iow", + "region": 6 + }, { + "name": "Ipulo", + "id": "ass", + "region": 2 + }, { + "name": "Iranun (Latin)", + "id": "ilp-latn", + "region": 3 + }, { + "name": "Iraya (Latin)", + "id": "iry-latn", + "region": 3 + }, { + "name": "Irish", + "id": "ga", + "region": 4 + }, { + "name": "Isanzu (Latin)", + "id": "isn-latn", + "region": 2 + }, { + "name": "Ishkashimi (Cyrillic, Tajikistan)", + "id": "isk-cyrl-tj", + "region": 3 + }, { + "name": "Islander Creole English", + "id": "icr", + "region": 6 + }, { + "name": "Isnag (Latin)", + "id": "isd-latn", + "region": 3 + }, { + "name": "Isu", + "id": "isu", + "region": 2 + }, { + "name": "Isu", + "id": "szv", + "region": 2 + }, { + "name": "Italian", + "id": "it", + "region": 4 + }, { + "name": "Italian (Braille)", + "id": "it-brai", + "region": 4 + }, { + "name": "Itawit (Latin)", + "id": "itv-latn", + "region": 3 + }, { + "name": "itc-ital (Old Italic (Etruscan, Oscan, etc.))", + "id": "itc-ital", + "region": 1 + }, { + "name": "Itneg, Binongan (Latin)", + "id": "itb-latn", + "region": 3 + }, { + "name": "Itneg, Masadiit (Latin)", + "id": "tis-latn", + "region": 3 + }, { + "name": "Ivatan (Latin)", + "id": "ivv-latn", + "region": 3 + }, { + "name": "Ivbie North-Okpela-Arhe (Latin)", + "id": "atg-latn", + "region": 2 + }, { + "name": "Iyive", + "id": "uiv", + "region": 2 + }, { + "name": "Jalkunan (Latin)", + "id": "bxl-latn", + "region": 2 + }, { + "name": "Jamaican Creole English", + "id": "jam", + "region": 6 + }, { + "name": "Japanese", + "id": "ja", + "region": 3 + }, { + "name": "Japanese (Braille)", + "id": "ja-brai", + "region": 3 + }, { + "name": "Jarai (Khmer)", + "id": "jra-khmr", + "region": 3 + }, { + "name": "Jarai (Latin)", + "id": "jra-latn", + "region": 3 + }, { + "name": "Javanese (Javanese)", + "id": "jv-java", + "region": 3 + }, { + "name": "Javanese (Latin)", + "id": "jv-latn", + "region": 3 + }, { + "name": "Jaya", + "id": "jyy", + "region": 2 + }, { + "name": "Jewish Babylonian Aramaic (Syriac)", + "id": "tmr-syrc", + "region": 3 + }, { + "name": "Jiarong (Tibetan)", + "id": "jya-tibt", + "region": 3 + }, { + "name": "Jimjimen", + "id": "jim", + "region": 2 + }, { + "name": "Jina", + "id": "jia", + "region": 2 + }, { + "name": "Jirel", + "id": "jul", + "region": 3 + }, { + "name": "Jita (Latin)", + "id": "jit-latn", + "region": 2 + }, { + "name": "Jju (Latin)", + "id": "kaj-latn", + "region": 2 + }, { + "name": "Joba (Latin)", + "id": "job-latn", + "region": 2 + }, { + "name": "Jola-Felupe", + "id": "eja", + "region": 2 + }, { + "name": "Jola-Fonyi", + "id": "dyo", + "region": 2 + }, { + "name": "Jola-Kasa", + "id": "csk", + "region": 2 + }, { + "name": "Jonkor Bourmataguil", + "id": "jeu", + "region": 2 + }, { + "name": "Jowulu (Latin)", + "id": "jow-latn", + "region": 2 + }, { + "name": "Juang (Oriya)", + "id": "jun-orya", + "region": 3 + }, { + "name": "Jukun Takum", + "id": "jbu", + "region": 2 + }, { + "name": "Jukun Takum (Latin)", + "id": "jbu-latn", + "region": 2 + }, { + "name": "Jula (Latin)", + "id": "dyu-latn", + "region": 2 + }, { + "name": "Jula (N\u2019Ko)", + "id": "dyu-nkoo", + "region": 2 + }, { + "name": "Jumli", + "id": "jml", + "region": 3 + }, { + "name": "Jutish", + "id": "jut", + "region": 1 + }, { + "name": "Kaansa (Latin)", + "id": "gna-latn", + "region": 2 + }, { + "name": "Kaba D\u00e9m\u00e9, Sara", + "id": "kwg", + "region": 2 + }, { + "name": "Kaba Naa, Sara", + "id": "kwv", + "region": 2 + }, { + "name": "Kabalai", + "id": "kvf", + "region": 2 + }, { + "name": "Kabba", + "id": "ksp", + "region": 2 + }, { + "name": "Kabiy\u00e8 (Braille)", + "id": "kbp-brai", + "region": 2 + }, { + "name": "Kabiy\u00e8 (Latin)", + "id": "kbp-latn", + "region": 2 + }, { + "name": "Kabuverdianu", + "id": "kea", + "region": 2 + }, { + "name": "Kabwa (Latin)", + "id": "cwa-latn", + "region": 2 + }, { + "name": "Kabwari (Latin)", + "id": "kcw-latn", + "region": 2 + }, { + "name": "Kabyle (Tifinagh)", + "id": "kab-tfng", + "region": 2 + }, { + "name": "Kafa (Latin)", + "id": "kbr-latn", + "region": 2 + }, { + "name": "Kagayanen (Latin)", + "id": "cgc-latn", + "region": 3 + }, { + "name": "Kagulu (Latin)", + "id": "kki-latn", + "region": 2 + }, { + "name": "Kahe (Latin)", + "id": "hka-latn", + "region": 2 + }, { + "name": "Kaiku (Latin)", + "id": "kkq-latn", + "region": 2 + }, { + "name": "Kajakse", + "id": "ckq", + "region": 2 + }, { + "name": "Kako", + "id": "kkj", + "region": 2 + }, { + "name": "Kakwa (Latin)", + "id": "keo-latn", + "region": 2 + }, { + "name": "Kalagan", + "id": "kqe", + "region": 3 + }, { + "name": "Kalagan (Latin)", + "id": "kqe-latn", + "region": 3 + }, { + "name": "Kalagan, Kagan (Latin)", + "id": "kll-latn", + "region": 3 + }, { + "name": "Kalanguya (Latin)", + "id": "kak-latn", + "region": 3 + }, { + "name": "Kalasha (Arabic)", + "id": "kls-arab", + "region": 3 + }, { + "name": "Kalinga, Butbut (Latin)", + "id": "kyb-latn", + "region": 3 + }, { + "name": "Kalinga, Limos (Latin)", + "id": "kmk-latn", + "region": 3 + }, { + "name": "Kalinga, Lubuagan (Latin)", + "id": "knb-latn", + "region": 3 + }, { + "name": "Kalinga, Majukayang (Latin)", + "id": "kmd-latn", + "region": 3 + }, { + "name": "Kalinga, Southern (Latin)", + "id": "ksc-latn", + "region": 3 + }, { + "name": "Kalinga, Tanudan (Latin)", + "id": "kml-latn", + "region": 3 + }, { + "name": "Kallahan, Keley-i (Latin)", + "id": "ify-latn", + "region": 3 + }, { + "name": "Kambaata", + "id": "ktb", + "region": 2 + }, { + "name": "Kambaata (Latin)", + "id": "ktb-latn", + "region": 2 + }, { + "name": "Kami (Latin)", + "id": "kcu-latn", + "region": 2 + }, { + "name": "Kamwe (Latin)", + "id": "hig-latn", + "region": 2 + }, { + "name": "Kanembu", + "id": "kbl", + "region": 2 + }, { + "name": "Kango (Latin)", + "id": "kty-latn", + "region": 2 + }, { + "name": "Kango (Latin)", + "id": "kzy-latn", + "region": 2 + }, { + "name": "Kankanaey (Latin)", + "id": "kne-latn", + "region": 3 + }, { + "name": "Kankanay, Northern (Latin)", + "id": "xnn-latn", + "region": 3 + }, { + "name": "Kannada", + "id": "kn", + "region": 3 + }, { + "name": "Kannada (Braille)", + "id": "kn-brai", + "region": 3 + }, { + "name": "Kanu (Latin)", + "id": "khx-latn", + "region": 2 + }, { + "name": "Kanuri", + "id": "kr", + "region": 1 + }, { + "name": "Kanuri (Latin)", + "id": "kr-latn", + "region": 1 + }, { + "name": "Kanuri, Bilma", + "id": "bms", + "region": 2 + }, { + "name": "Kanuri, Manga", + "id": "kby", + "region": 2 + }, { + "name": "Kanuri, Manga (Latin)", + "id": "kby-latn", + "region": 2 + }, { + "name": "Kanuri, Tumari", + "id": "krt", + "region": 2 + }, { + "name": "Kanuri, Yerwa (Latin)", + "id": "knc-latn", + "region": 2 + }, { + "name": "Kapampangan (Braille)", + "id": "pam-brai", + "region": 3 + }, { + "name": "Kapampangan (Latin)", + "id": "pam-latn", + "region": 3 + }, { + "name": "Kara (Latin)", + "id": "reg-latn", + "region": 2 + }, { + "name": "Karakalpak", + "id": "kaa", + "region": 3 + }, { + "name": "Karakalpak (Latin)", + "id": "kaa-latn", + "region": 3 + }, { + "name": "Karang", + "id": "kzr", + "region": 2 + }, { + "name": "Karanga", + "id": "kth", + "region": 2 + }, { + "name": "Karao (Latin)", + "id": "kyj-latn", + "region": 3 + }, { + "name": "Kare", + "id": "kbn", + "region": 2 + }, { + "name": "Karelian", + "id": "krl", + "region": 4 + }, { + "name": "Karen, Bwe (Latin)", + "id": "bwe-latn", + "region": 3 + }, { + "name": "Karen, S\u2019gaw (Myanmar)", + "id": "ksw-mymr", + "region": 3 + }, { + "name": "Kari (Latin)", + "id": "kbj-latn", + "region": 2 + }, { + "name": "Karipuna Creole French", + "id": "kmv", + "region": 6 + }, { + "name": "Karon", + "id": "krx", + "region": 2 + }, { + "name": "Kasabe", + "id": "luw", + "region": 2 + }, { + "name": "Kasem (Latin)", + "id": "xsm-latn", + "region": 2 + }, { + "name": "Kashmiri (Arabic)", + "id": "ks-arab", + "region": 3 + }, { + "name": "Kashubian", + "id": "csb", + "region": 4 + }, { + "name": "Kaska (Latin)", + "id": "kkz-latn", + "region": 6 + }, { + "name": "Kavet (Khmer)", + "id": "krv-khmr", + "region": 3 + }, { + "name": "Kawi (Balinese)", + "id": "kaw-bali", + "region": 1 + }, { + "name": "Kawi (Javanese)", + "id": "kaw-java", + "region": 1 + }, { + "name": "Kayah, Western (Kayah Li)", + "id": "kyu-kali", + "region": 3 + }, { + "name": "Kayah, Western (Latin)", + "id": "kyu-latn", + "region": 3 + }, { + "name": "Kayah, Western (Myanmar)", + "id": "kyu-mymr", + "region": 3 + }, { + "name": "Kayan (Latin)", + "id": "pdu-latn", + "region": 3 + }, { + "name": "Kayaw (Latin)", + "id": "kvl-latn", + "region": 3 + }, { + "name": "Kazakh", + "id": "kk", + "region": 3 + }, { + "name": "Kazakh (Braille)", + "id": "kk-brai", + "region": 3 + }, { + "name": "Kele (Latin)", + "id": "khy-latn", + "region": 2 + }, { + "name": "Keliko (Latin)", + "id": "kbo-latn", + "region": 2 + }, { + "name": "Kemedzung", + "id": "dmo", + "region": 2 + }, { + "name": "Kendeje", + "id": "klf", + "region": 2 + }, { + "name": "Kendem", + "id": "kvm", + "region": 2 + }, { + "name": "Kenga", + "id": "kyq", + "region": 2 + }, { + "name": "Kenswei Nsei", + "id": "ndb", + "region": 2 + }, { + "name": "Kenyang", + "id": "ken", + "region": 2 + }, { + "name": "Kenye (Latin)", + "id": "lke-latn", + "region": 2 + }, { + "name": "Kera", + "id": "ker", + "region": 2 + }, { + "name": "Kerewe (Latin)", + "id": "ked-latn", + "region": 2 + }, { + "name": "Kerinci (Latin)", + "id": "kvr-latn", + "region": 3 + }, { + "name": "Khaling", + "id": "klr", + "region": 3 + }, { + "name": "Kham, Gamale", + "id": "kgj", + "region": 3 + }, { + "name": "Kham, Western Parbate", + "id": "kjl", + "region": 3 + }, { + "name": "Khamba (Tibetan)", + "id": "kbg-tibt", + "region": 3 + }, { + "name": "Khengkha (Tibetan)", + "id": "xkf-tibt", + "region": 3 + }, { + "name": "Khinalugh", + "id": "kjj", + "region": 3 + }, { + "name": "Khmer", + "id": "km", + "region": 3 + }, { + "name": "Khmer, Northern (Khmer)", + "id": "kxm-khmr", + "region": 3 + }, { + "name": "Khmu (Lao)", + "id": "kjg-laoo", + "region": 3 + }, { + "name": "Khowar (Arabic)", + "id": "khw-arab", + "region": 3 + }, { + "name": "Kibet", + "id": "kie", + "region": 2 + }, { + "name": "Kigiryama (Latin)", + "id": "nyf-latn", + "region": 2 + }, { + "name": "Kim", + "id": "kia", + "region": 2 + }, { + "name": "Kimbu (Latin)", + "id": "kiv-latn", + "region": 2 + }, { + "name": "Kimr\u00e9", + "id": "kqp", + "region": 2 + }, { + "name": "Kinaray-a (Latin)", + "id": "krj-latn", + "region": 3 + }, { + "name": "Kinga (Latin)", + "id": "zga-latn", + "region": 2 + }, { + "name": "Kinnauri (Devanagari)", + "id": "kfk-deva", + "region": 3 + }, { + "name": "Kinyarwanda (Braille)", + "id": "rw-brai", + "region": 2 + }, { + "name": "Kipfokomo (Latin)", + "id": "pkb-latn", + "region": 2 + }, { + "name": "Kiribati (Latin)", + "id": "gil-latn", + "region": 7 + }, { + "name": "Kisan (Oriya)", + "id": "xis-orya", + "region": 3 + }, { + "name": "Kisi (Latin)", + "id": "kiz-latn", + "region": 2 + }, { + "name": "Kisi, Southern (Latin)", + "id": "kss-latn", + "region": 2 + }, { + "name": "Kistane", + "id": "gru", + "region": 2 + }, { + "name": "Kistane (Latin)", + "id": "gru-latn", + "region": 2 + }, { + "name": "Kitharaka (Latin)", + "id": "thk-latn", + "region": 2 + }, { + "name": "Kituba", + "id": "ktu", + "region": 2 + }, { + "name": "Kituba", + "id": "mkw", + "region": 2 + }, { + "name": "Klao (Latin)", + "id": "klu-latn", + "region": 2 + }, { + "name": "Koalib", + "id": "kib", + "region": 2 + }, { + "name": "Kohistani, Indus (Arabic)", + "id": "mvy-arab", + "region": 3 + }, { + "name": "Koho (Latin)", + "id": "kpm-latn", + "region": 3 + }, { + "name": "Koke", + "id": "kou", + "region": 2 + }, { + "name": "Kol", + "id": "biw", + "region": 2 + }, { + "name": "Kolbila", + "id": "klc", + "region": 2 + }, { + "name": "Koli, Parkari (Arabic)", + "id": "kvx-arab", + "region": 3 + }, { + "name": "Koli, Wadiyari (Arabic)", + "id": "kxp-arab", + "region": 3 + }, { + "name": "Kom", + "id": "bkm", + "region": 2 + }, { + "name": "Koma", + "id": "kmy", + "region": 2 + }, { + "name": "Komo (Latin)", + "id": "kmw-latn", + "region": 2 + }, { + "name": "Komo (Latin)", + "id": "xom-latn", + "region": 2 + }, { + "name": "Konab\u00e9r\u00e9 (Latin)", + "id": "bbo-latn", + "region": 2 + }, { + "name": "Konkani", + "id": "knn", + "region": 3 + }, { + "name": "Konkani (Devanagari)", + "id": "knn-deva", + "region": 3 + }, { + "name": "Konkani (macrolanguage) (Devanagari)", + "id": "kok-deva", + "region": 1 + }, { + "name": "Konkani, Goan (Devanagari)", + "id": "gom-deva", + "region": 3 + }, { + "name": "Konkomba (Braille)", + "id": "xon-brai", + "region": 2 + }, { + "name": "Kono (Latin)", + "id": "kno-latn", + "region": 2 + }, { + "name": "Konongo (Latin)", + "id": "kcz-latn", + "region": 2 + }, { + "name": "Konso", + "id": "kxc", + "region": 2 + }, { + "name": "Konso (Ethiopic)", + "id": "kxc-ethi", + "region": 2 + }, { + "name": "Konso (Latin)", + "id": "kxc-latn", + "region": 2 + }, { + "name": "Konzo (Latin)", + "id": "koo-latn", + "region": 2 + }, { + "name": "Koonzime", + "id": "ozm", + "region": 2 + }, { + "name": "Koorete (Latin)", + "id": "kqy-latn", + "region": 2 + }, { + "name": "Korean", + "id": "ko", + "region": 3 + }, { + "name": "Korean (Braille)", + "id": "ko-brai", + "region": 3 + }, { + "name": "Koreguaje", + "id": "coe", + "region": 6 + }, { + "name": "Korop", + "id": "krp", + "region": 2 + }, { + "name": "Koshin", + "id": "kid", + "region": 2 + }, { + "name": "Koy Sanjaq Surat (Syriac)", + "id": "kqd-syrc", + "region": 3 + }, { + "name": "Koya (Oriya)", + "id": "kff-orya", + "region": 3 + }, { + "name": "Kpelle (Latin)", + "id": "kpe-latn", + "region": 1 + }, { + "name": "Kpelle, Liberia (Latin)", + "id": "xpe-latn", + "region": 2 + }, { + "name": "Krahn, Eastern (Latin)", + "id": "kqo-latn", + "region": 2 + }, { + "name": "Krahn, Western (Latin)", + "id": "krw-latn", + "region": 2 + }, { + "name": "Krio (Latin)", + "id": "kri-latn", + "region": 2 + }, { + "name": "Kriol", + "id": "rop", + "region": 7 + }, { + "name": "Krung (Khmer)", + "id": "krr-khmr", + "region": 3 + }, { + "name": "Kuay (Khmer)", + "id": "kdt-khmr", + "region": 3 + }, { + "name": "Kudmali (Oriya)", + "id": "kyw-orya", + "region": 3 + }, { + "name": "Kujarge", + "id": "vkj", + "region": 2 + }, { + "name": "Kuk", + "id": "kfn", + "region": 2 + }, { + "name": "Kuke (Tibetan)", + "id": "ght-tibt", + "region": 3 + }, { + "name": "Kukele (Latin)", + "id": "kez-latn", + "region": 2 + }, { + "name": "Kulfa", + "id": "kxj", + "region": 2 + }, { + "name": "Kulung", + "id": "kle", + "region": 3 + }, { + "name": "Kung", + "id": "kfl", + "region": 2 + }, { + "name": "Kuo", + "id": "xuo", + "region": 2 + }, { + "name": "Kuranko (Latin)", + "id": "knk-latn", + "region": 2 + }, { + "name": "Kurdish, Central (Arabic)", + "id": "ckb-arab", + "region": 3 + }, { + "name": "Kuria (Latin)", + "id": "kuj-latn", + "region": 2 + }, { + "name": "Kurumba, Kannada (Kannada)", + "id": "kfi-knda", + "region": 3 + }, { + "name": "Kutenai (Latin)", + "id": "kut-latn", + "region": 6 + }, { + "name": "Kutep", + "id": "kub", + "region": 2 + }, { + "name": "Kutu (Latin)", + "id": "kdc-latn", + "region": 2 + }, { + "name": "Kuvi (Oriya)", + "id": "kxv-orya", + "region": 3 + }, { + "name": "Kuwaa (Latin)", + "id": "blh-latn", + "region": 2 + }, { + "name": "Kuwaataay", + "id": "cwt", + "region": 2 + }, { + "name": "Kwa\u2019", + "id": "bko", + "region": 2 + }, { + "name": "Kwaja", + "id": "kdz", + "region": 2 + }, { + "name": "Kwakum", + "id": "kwu", + "region": 2 + }, { + "name": "Kwakwala (Latin)", + "id": "kwk-latn", + "region": 6 + }, { + "name": "Kwami (Latin)", + "id": "ktf-latn", + "region": 2 + }, { + "name": "Kwang", + "id": "kvi", + "region": 2 + }, { + "name": "Kwanja", + "id": "knp", + "region": 2 + }, { + "name": "Kwasio", + "id": "nmg", + "region": 2 + }, { + "name": "Kwaya (Latin)", + "id": "kya-latn", + "region": 2 + }, { + "name": "Kwere (Latin)", + "id": "cwe-latn", + "region": 2 + }, { + "name": "kxu-orya (Oriya)", + "id": "kxu-orya", + "region": 1 + }, { + "name": "Kyerung", + "id": "kgy", + "region": 3 + }, { + "name": "Kyrgyz (Cyrillic)", + "id": "ky-cyrl", + "region": 3 + }, { + "name": "La\u2019bi", + "id": "lbi", + "region": 2 + }, { + "name": "Laal", + "id": "gdm", + "region": 2 + }, { + "name": "Laalaa", + "id": "cae", + "region": 2 + }, { + "name": "Ladakhi (Tibetan)", + "id": "lbj-tibt", + "region": 3 + }, { + "name": "Ladin", + "id": "lld", + "region": 4 + }, { + "name": "Ladin (Latin)", + "id": "lld-latn", + "region": 4 + }, { + "name": "Ladino", + "id": "lad", + "region": 3 + }, { + "name": "Lagwan", + "id": "kot", + "region": 2 + }, { + "name": "Lahnda", + "id": "lah", + "region": 1 + }, { + "name": "Lahu (Latin)", + "id": "lhu-latn", + "region": 3 + }, { + "name": "Laimbue", + "id": "lmx", + "region": 2 + }, { + "name": "Laka", + "id": "lap", + "region": 2 + }, { + "name": "Lakkia (Lisu)", + "id": "lbc-lisu", + "region": 3 + }, { + "name": "Lakota (Latin)", + "id": "lkt-latn", + "region": 6 + }, { + "name": "Lamang (Latin)", + "id": "hia-latn", + "region": 2 + }, { + "name": "Lamkang", + "id": "lmk", + "region": 3 + }, { + "name": "Lamnso\u2019", + "id": "lns", + "region": 2 + }, { + "name": "Landoma (Latin)", + "id": "ldm-latn", + "region": 2 + }, { + "name": "Langi (Latin)", + "id": "lag-latn", + "region": 2 + }, { + "name": "Lao", + "id": "lo", + "region": 3 + }, { + "name": "Latin", + "id": "la", + "region": 4 + }, { + "name": "Latvian", + "id": "lv", + "region": 1 + }, { + "name": "Latvian (Braille)", + "id": "lv-brai", + "region": 1 + }, { + "name": "Lave (Khmer)", + "id": "brb-khmr", + "region": 3 + }, { + "name": "Laz (Khutsuri (Asomtavruli and Nuskhuri), Georgia)", + "id": "lzz-geok-ge", + "region": 3 + }, { + "name": "Laz (Latin)", + "id": "lzz-latn", + "region": 3 + }, { + "name": "Laz (Latin, Georgia)", + "id": "lzz-latn-ge", + "region": 3 + }, { + "name": "Lefa", + "id": "lfa", + "region": 2 + }, { + "name": "Lega-Mwenga (Latin)", + "id": "lgm-latn", + "region": 2 + }, { + "name": "Lega-Shabunda (Latin)", + "id": "lea-latn", + "region": 2 + }, { + "name": "Lele", + "id": "lln", + "region": 2 + }, { + "name": "Lendu (Latin)", + "id": "led-latn", + "region": 2 + }, { + "name": "Lengola (Latin)", + "id": "lej-latn", + "region": 2 + }, { + "name": "Lepcha", + "id": "lep", + "region": 3 + }, { + "name": "Lepcha (Tibetan)", + "id": "lep-tibt", + "region": 3 + }, { + "name": "Lese (Latin)", + "id": "les-latn", + "region": 2 + }, { + "name": "Leti", + "id": "leo", + "region": 2 + }, { + "name": "Lhomi", + "id": "lhm", + "region": 3 + }, { + "name": "Lhowa", + "id": "loy", + "region": 3 + }, { + "name": "Lhowa (Tibetan)", + "id": "loy-tibt", + "region": 3 + }, { + "name": "Lidzonka", + "id": "add", + "region": 2 + }, { + "name": "Ligenza (Latin)", + "id": "lgz-latn", + "region": 2 + }, { + "name": "Ligurian", + "id": "lij", + "region": 4 + }, { + "name": "Lika (Latin)", + "id": "lik-latn", + "region": 2 + }, { + "name": "Lillooet (Latin)", + "id": "lil-latn", + "region": 6 + }, { + "name": "Limba, West-Central (Latin)", + "id": "lia-latn", + "region": 2 + }, { + "name": "Limbu", + "id": "lif", + "region": 3 + }, { + "name": "Limbu (Devanagari)", + "id": "lif-deva", + "region": 3 + }, { + "name": "Limbu (Limbu)", + "id": "lif-limb", + "region": 3 + }, { + "name": "Limbum", + "id": "lmp", + "region": 2 + }, { + "name": "Limburgish", + "id": "li", + "region": 4 + }, { + "name": "Lingala", + "id": "ln", + "region": 2 + }, { + "name": "Lipo (Lisu)", + "id": "lpo-lisu", + "region": 3 + }, { + "name": "Lipo (Miao)", + "id": "lpo-plrd", + "region": 3 + }, { + "name": "Lisu (Lisu)", + "id": "lis-lisu", + "region": 3 + }, { + "name": "Lithuanian", + "id": "lt", + "region": 4 + }, { + "name": "Liv", + "id": "liv", + "region": 4 + }, { + "name": "Livvi-Karelian", + "id": "olo", + "region": 4 + }, { + "name": "Logo (Latin)", + "id": "log-latn", + "region": 2 + }, { + "name": "Lohorung", + "id": "lbr", + "region": 3 + }, { + "name": "Lokaa (Latin)", + "id": "yaz-latn", + "region": 2 + }, { + "name": "Loko (Latin)", + "id": "lok-latn", + "region": 2 + }, { + "name": "Loma (Latin)", + "id": "lom-latn", + "region": 2 + }, { + "name": "Lombard", + "id": "lmo", + "region": 4 + }, { + "name": "Lombi (Latin)", + "id": "lmi-latn", + "region": 2 + }, { + "name": "Lombo (Latin)", + "id": "loo-latn", + "region": 2 + }, { + "name": "Longto", + "id": "wok", + "region": 2 + }, { + "name": "Longuda (Latin)", + "id": "lnu-latn", + "region": 2 + }, { + "name": "Louisiana Creole", + "id": "lou", + "region": 6 + }, { + "name": "L\u00fc (New Tai Lue)", + "id": "khb-talu", + "region": 3 + }, { + "name": "Ludian", + "id": "lud", + "region": 4 + }, { + "name": "Lugbara (Latin)", + "id": "lgg-latn", + "region": 2 + }, { + "name": "Luguru (Latin)", + "id": "ruf-latn", + "region": 2 + }, { + "name": "Lushootseed", + "id": "lut", + "region": 6 + }, { + "name": "Lushootseed (Latin)", + "id": "lut-latn", + "region": 6 + }, { + "name": "Lutos", + "id": "ndy", + "region": 2 + }, { + "name": "Luxembourgish", + "id": "lb", + "region": 4 + }, { + "name": "Ma (Latin)", + "id": "msj-latn", + "region": 2 + }, { + "name": "Ma\u2019di", + "id": "mhi", + "region": 2 + }, { + "name": "Maan (Latin)", + "id": "mev-latn", + "region": 2 + }, { + "name": "Maba (Latin)", + "id": "mde-latn", + "region": 2 + }, { + "name": "Mabire", + "id": "muj", + "region": 2 + }, { + "name": "Macedonian", + "id": "mk", + "region": 4 + }, { + "name": "Macedonian (Braille)", + "id": "mk-brai", + "region": 4 + }, { + "name": "Machame (Latin)", + "id": "jmc-latn", + "region": 2 + }, { + "name": "Machinga (Latin)", + "id": "mvw-latn", + "region": 2 + }, { + "name": "Mada", + "id": "mxu", + "region": 2 + }, { + "name": "Madura (Javanese)", + "id": "mad-java", + "region": 3 + }, { + "name": "Mafa", + "id": "maf", + "region": 2 + }, { + "name": "Magahi (Devanagari)", + "id": "mag-deva", + "region": 3 + }, { + "name": "Magar, Eastern", + "id": "mgp", + "region": 3 + }, { + "name": "Magar, Western", + "id": "mrd", + "region": 3 + }, { + "name": "Magoma (Latin)", + "id": "gmx-latn", + "region": 2 + }, { + "name": "Maguindanaon (Latin)", + "id": "mdh-latn", + "region": 3 + }, { + "name": "Maithili", + "id": "mai", + "region": 3 + }, { + "name": "Maithili (Devanagari)", + "id": "mai-deva", + "region": 3 + }, { + "name": "Maithili (Tirhuta)", + "id": "mai-tirh", + "region": 3 + }, { + "name": "Majang (Latin)", + "id": "mpe-latn", + "region": 2 + }, { + "name": "Majera", + "id": "xmj", + "region": 2 + }, { + "name": "Majhi", + "id": "mjz", + "region": 3 + }, { + "name": "Makaa", + "id": "mcp", + "region": 2 + }, { + "name": "Makah (Latin)", + "id": "myh-latn", + "region": 6 + }, { + "name": "Makasar (Buginese)", + "id": "mak-bugi", + "region": 3 + }, { + "name": "Makasar (Latin)", + "id": "mak-latn", + "region": 3 + }, { + "name": "Makasar (Makasar)", + "id": "mak-maka", + "region": 3 + }, { + "name": "Makonde (Latin)", + "id": "kde-latn", + "region": 2 + }, { + "name": "Malagasy (Braille)", + "id": "mg-brai", + "region": 1 + }, { + "name": "Malagasy, Plateau (Braille)", + "id": "plt-brai", + "region": 2 + }, { + "name": "Malay (Latin)", + "id": "zlm-latn", + "region": 3 + }, { + "name": "Malay (macrolanguage)", + "id": "ms", + "region": 1 + }, { + "name": "Malay (macrolanguage) (Braille)", + "id": "ms-brai", + "region": 1 + }, { + "name": "Malay, Jambi (Latin)", + "id": "jax-latn", + "region": 3 + }, { + "name": "Malay, Kedah (Latin)", + "id": "meo-latn", + "region": 3 + }, { + "name": "Malay, Manado (Latin)", + "id": "xmm-latn", + "region": 3 + }, { + "name": "Malay, North Moluccan (Latin)", + "id": "max-latn", + "region": 3 + }, { + "name": "Malay, Pattani (Latin)", + "id": "mfa-latn", + "region": 3 + }, { + "name": "Malay, Standard (Braille)", + "id": "zsm-brai", + "region": 3 + }, { + "name": "Malay, Standard (Latin)", + "id": "zsm-latn", + "region": 3 + }, { + "name": "Malayalam", + "id": "ml", + "region": 3 + }, { + "name": "Malayalam (Braille)", + "id": "ml-brai", + "region": 3 + }, { + "name": "Maldivian", + "id": "dv", + "region": 3 + }, { + "name": "Male (Latin)", + "id": "mdy-latn", + "region": 2 + }, { + "name": "Malecite-Passamaquoddy (Latin)", + "id": "pqm-latn", + "region": 6 + }, { + "name": "Malgbe", + "id": "mxf", + "region": 2 + }, { + "name": "Malila (Latin)", + "id": "mgq-latn", + "region": 2 + }, { + "name": "Malimba", + "id": "mzd", + "region": 2 + }, { + "name": "Maltese", + "id": "mt", + "region": 4 + }, { + "name": "Maltese (Braille)", + "id": "mt-brai", + "region": 4 + }, { + "name": "Mamanwa (Latin)", + "id": "mmn-latn", + "region": 3 + }, { + "name": "Mambai", + "id": "mcs", + "region": 2 + }, { + "name": "Mambila, Cameroon", + "id": "mcu", + "region": 2 + }, { + "name": "Mamvu (Latin)", + "id": "mdi-latn", + "region": 2 + }, { + "name": "Manda (Latin)", + "id": "mgs-latn", + "region": 2 + }, { + "name": "Mandaic (Mandaic)", + "id": "mid-mand", + "region": 3 + }, { + "name": "Mandar (Buginese)", + "id": "mdr-bugi", + "region": 3 + }, { + "name": "Mandingo (N\u2019Ko)", + "id": "man-nkoo", + "region": 1 + }, { + "name": "Mandinka (Latin)", + "id": "mnk-latn", + "region": 2 + }, { + "name": "Mandjak", + "id": "mfv", + "region": 2 + }, { + "name": "Mandjak (Latin)", + "id": "mfv-latn", + "region": 2 + }, { + "name": "Mangbetu (Latin)", + "id": "mdj-latn", + "region": 2 + }, { + "name": "Mangbutu (Latin)", + "id": "mdk-latn", + "region": 2 + }, { + "name": "Mango", + "id": "mge", + "region": 2 + }, { + "name": "Maninkakan, Eastern (Latin)", + "id": "emk-latn", + "region": 2 + }, { + "name": "Maninkakan, Eastern (N\u2019Ko)", + "id": "emk-nkoo", + "region": 2 + }, { + "name": "Maninkakan, Kita (Latin)", + "id": "mwk-latn", + "region": 2 + }, { + "name": "Maninkakan, Western (Latin)", + "id": "mlq-latn", + "region": 2 + }, { + "name": "Mankanya", + "id": "knf", + "region": 2 + }, { + "name": "Mankanya (Latin)", + "id": "knf-latn", + "region": 2 + }, { + "name": "Manobo, Agusan (Latin)", + "id": "msm-latn", + "region": 3 + }, { + "name": "Manobo, Ata (Latin)", + "id": "atd-latn", + "region": 3 + }, { + "name": "Manobo, Cotabato (Latin)", + "id": "mta-latn", + "region": 3 + }, { + "name": "Manobo, Dibabawon (Latin)", + "id": "mbd-latn", + "region": 3 + }, { + "name": "Manobo, Ilianen (Latin)", + "id": "mbi-latn", + "region": 3 + }, { + "name": "Manobo, Matigsalug (Latin)", + "id": "mbt-latn", + "region": 3 + }, { + "name": "Manobo, Obo (Latin)", + "id": "obo-latn", + "region": 3 + }, { + "name": "Manobo, Sarangani (Latin)", + "id": "mbs-latn", + "region": 3 + }, { + "name": "Manobo, Western Bukidnon (Latin)", + "id": "mbb-latn", + "region": 3 + }, { + "name": "Mansaka (Latin)", + "id": "msk-latn", + "region": 3 + }, { + "name": "Manta", + "id": "myg", + "region": 2 + }, { + "name": "Manx", + "id": "gv", + "region": 4 + }, { + "name": "Manya (Latin)", + "id": "mzj-latn", + "region": 2 + }, { + "name": "Maori (Latin)", + "id": "mi-latn", + "region": 7 + }, { + "name": "Mapun (Latin)", + "id": "sjm-latn", + "region": 3 + }, { + "name": "Maranao (Latin)", + "id": "mrw-latn", + "region": 3 + }, { + "name": "Mararit", + "id": "mgb", + "region": 2 + }, { + "name": "Marathi", + "id": "mr", + "region": 3 + }, { + "name": "Marathi (Braille)", + "id": "mr-brai", + "region": 3 + }, { + "name": "Marathi (Modi)", + "id": "mr-modi", + "region": 3 + }, { + "name": "Marba", + "id": "mpg", + "region": 2 + }, { + "name": "Marfa", + "id": "mvu", + "region": 2 + }, { + "name": "Maria (Devanagari)", + "id": "mrr-deva", + "region": 3 + }, { + "name": "Maria, Dandami (Devanagari)", + "id": "daq-deva", + "region": 3 + }, { + "name": "Markweeta (Latin)", + "id": "enb-latn", + "region": 2 + }, { + "name": "Marwari (Arabic)", + "id": "mve-arab", + "region": 3 + }, { + "name": "Marwari", + "id": "rwr", + "region": 3 + }, { + "name": "Masaaba (Latin)", + "id": "myx-latn", + "region": 2 + }, { + "name": "Masalit", + "id": "mls", + "region": 2 + }, { + "name": "Masana", + "id": "mcn", + "region": 2 + }, { + "name": "Masbatenyo (Latin)", + "id": "msb-latn", + "region": 3 + }, { + "name": "Maslam", + "id": "msv", + "region": 2 + }, { + "name": "Masmaje", + "id": "mes", + "region": 2 + }, { + "name": "Massalat", + "id": "mdg", + "region": 2 + }, { + "name": "Matal", + "id": "mfh", + "region": 2 + }, { + "name": "Matengo (Latin)", + "id": "mgv-latn", + "region": 2 + }, { + "name": "Matlatzinca, Atzingo", + "id": "ocu", + "region": 6 + }, { + "name": "Mattokki (Coptic)", + "id": "xnz-copt", + "region": 2 + }, { + "name": "Matumbi (Latin)", + "id": "mgw-latn", + "region": 2 + }, { + "name": "Mawa", + "id": "mcw", + "region": 2 + }, { + "name": "Ma\u0300we\u0301s Aas\u02bce\u0300 (Latin)", + "id": "myf-latn", + "region": 2 + }, { + "name": "Mayogo (Latin)", + "id": "mdm-latn", + "region": 2 + }, { + "name": "Mazagway-Hidi", + "id": "dkx", + "region": 2 + }, { + "name": "Mba (Latin)", + "id": "mfc-latn", + "region": 2 + }, { + "name": "Mbara", + "id": "mpk", + "region": 2 + }, { + "name": "Mbay", + "id": "myb", + "region": 2 + }, { + "name": "Mbembe, Tigon", + "id": "nza", + "region": 2 + }, { + "name": "Mbesa (Latin)", + "id": "zms-latn", + "region": 2 + }, { + "name": "Mbo", + "id": "mbo", + "region": 2 + }, { + "name": "Mbo (Latin)", + "id": "zmw-latn", + "region": 2 + }, { + "name": "Mbo\u2019", + "id": "mtk", + "region": 2 + }, { + "name": "Mbole (Latin)", + "id": "mdq-latn", + "region": 2 + }, { + "name": "Mbonga", + "id": "xmb", + "region": 2 + }, { + "name": "Mbongno", + "id": "bgu", + "region": 2 + }, { + "name": "Mbudum", + "id": "xmd", + "region": 2 + }, { + "name": "Mbugwe (Latin)", + "id": "mgz-latn", + "region": 2 + }, { + "name": "Mbuko", + "id": "mqb", + "region": 2 + }, { + "name": "Mbula-Bwazza (Latin)", + "id": "mbu-latn", + "region": 2 + }, { + "name": "Mbule", + "id": "mlb", + "region": 2 + }, { + "name": "Mbum", + "id": "mdd", + "region": 2 + }, { + "name": "Mbunga (Latin)", + "id": "mgy-latn", + "region": 2 + }, { + "name": "Me\u2019en (Latin)", + "id": "mym-latn", + "region": 2 + }, { + "name": "Medumba", + "id": "byv", + "region": 2 + }, { + "name": "Mefele", + "id": "mfj", + "region": 2 + }, { + "name": "Meitei (Braille)", + "id": "mni-brai", + "region": 3 + }, { + "name": "Meitei (Meitei Mayek)", + "id": "mni-mtei", + "region": 3 + }, { + "name": "Memoni (Arabic)", + "id": "mby-arab", + "region": 3 + }, { + "name": "Mendankwe-Nkwen", + "id": "mfd", + "region": 2 + }, { + "name": "Mende", + "id": "men", + "region": 2 + }, { + "name": "Mengaka", + "id": "xmg", + "region": 2 + }, { + "name": "Mengisa", + "id": "mct", + "region": 2 + }, { + "name": "Menka", + "id": "mea", + "region": 2 + }, { + "name": "Merey", + "id": "meq", + "region": 2 + }, { + "name": "Mesaka", + "id": "iyo", + "region": 2 + }, { + "name": "Mesme", + "id": "zim", + "region": 2 + }, { + "name": "Mesqan", + "id": "mvz", + "region": 2 + }, { + "name": "Meta\u2019", + "id": "mgo", + "region": 2 + }, { + "name": "Mewahang, Eastern", + "id": "emg", + "region": 3 + }, { + "name": "Mfumte", + "id": "nfu", + "region": 2 + }, { + "name": "Mi\u2019kmaq (Latin)", + "id": "mic-latn", + "region": 6 + }, { + "name": "Miao, Large Flowery (Miao)", + "id": "hmd-plrd", + "region": 3 + }, { + "name": "Middle English (1100-1500) (Runic)", + "id": "enm-runr", + "region": 1 + }, { + "name": "Migaama", + "id": "mmy", + "region": 2 + }, { + "name": "Miltu", + "id": "mlj", + "region": 2 + }, { + "name": "Miluk", + "id": "iml", + "region": 1 + }, { + "name": "Mina", + "id": "hna", + "region": 2 + }, { + "name": "Minangkabau (Latin)", + "id": "min-latn", + "region": 3 + }, { + "name": "Mingrelian (Khutsuri (Asomtavruli and Nuskhuri))", + "id": "xmf-geok", + "region": 3 + }, { + "name": "Mingrelian (Latin)", + "id": "xmf-latn", + "region": 3 + }, { + "name": "Mirandese", + "id": "mwl", + "region": 4 + }, { + "name": "Mirandese (Latin)", + "id": "mwl-latn", + "region": 4 + }, { + "name": "Mirgan (Oriya)", + "id": "zrg-orya", + "region": 3 + }, { + "name": "Mituku (Latin)", + "id": "zmq-latn", + "region": 2 + }, { + "name": "Mizo (Braille)", + "id": "lus-brai", + "region": 3 + }, { + "name": "Mlahs\u00f6 (Syriac)", + "id": "lhs-syrc", + "region": 3 + }, { + "name": "Mmaala", + "id": "mmu", + "region": 2 + }, { + "name": "Mmen", + "id": "bfm", + "region": 2 + }, { + "name": "Mnong, Central (Khmer)", + "id": "cmo-khmr", + "region": 3 + }, { + "name": "Mnong, Central (Latin)", + "id": "cmo-latn", + "region": 3 + }, { + "name": "Mochi (Latin)", + "id": "old-latn", + "region": 2 + }, { + "name": "Mofu, North", + "id": "mfk", + "region": 2 + }, { + "name": "Mofu-Gudur", + "id": "mif", + "region": 2 + }, { + "name": "Mogum", + "id": "mou", + "region": 2 + }, { + "name": "Mohawk (Latin)", + "id": "moh-latn", + "region": 6 + }, { + "name": "Moingi (Latin)", + "id": "mwz-latn", + "region": 2 + }, { + "name": "Mokpwe", + "id": "bri", + "region": 2 + }, { + "name": "Molbog (Latin)", + "id": "pwm-latn", + "region": 3 + }, { + "name": "Moloko", + "id": "mlw", + "region": 2 + }, { + "name": "Mon", + "id": "mnw", + "region": 3 + }, { + "name": "Mongolian (Braille)", + "id": "mn-brai", + "region": 1 + }, { + "name": "Mongolian (Mongolian)", + "id": "mn-mong", + "region": 1 + }, { + "name": "Mongolian, Halh (Braille)", + "id": "khk-brai", + "region": 3 + }, { + "name": "Mongolian, Halh (Cyrillic)", + "id": "khk-cyrl", + "region": 3 + }, { + "name": "Mongolian, Halh (Cyrillic, Mongolia)", + "id": "khk-cyrl-mn", + "region": 3 + }, { + "name": "Mongolian, Halh (Mongolian)", + "id": "khk-mong", + "region": 3 + }, { + "name": "Mongolian, Halh (Phags-pa, Mongolia)", + "id": "khk-phag-mn", + "region": 3 + }, { + "name": "Mongolian, Halh (Tibetan)", + "id": "khk-tibt", + "region": 3 + }, { + "name": "Mongolian, Peripheral (Phags-pa, China)", + "id": "mvf-phag-cn", + "region": 3 + }, { + "name": "Mono", + "id": "mru", + "region": 2 + }, { + "name": "M\u00f2or\u00e9 (Braille)", + "id": "mos-brai", + "region": 2 + }, { + "name": "M\u00f2or\u00e9 (Latin)", + "id": "mos-latn", + "region": 2 + }, { + "name": "Morisyen", + "id": "mfe", + "region": 2 + }, { + "name": "Morom", + "id": "bdo", + "region": 2 + }, { + "name": "Motu, Hiri (Braille)", + "id": "ho-brai", + "region": 7 + }, { + "name": "Mpade", + "id": "mpi", + "region": 2 + }, { + "name": "Mpiemo", + "id": "mcx", + "region": 2 + }, { + "name": "Mpoto (Latin)", + "id": "mpa-latn", + "region": 2 + }, { + "name": "Mpumpong", + "id": "mgg", + "region": 2 + }, { + "name": "Mru (Mro)", + "id": "mro-mroo", + "region": 3 + }, { + "name": "Mser", + "id": "kqx", + "region": 2 + }, { + "name": "Mubako", + "id": "muo", + "region": 2 + }, { + "name": "Mubi", + "id": "mub", + "region": 2 + }, { + "name": "Mugom (Tibetan)", + "id": "muk-tibt", + "region": 3 + }, { + "name": "Mukulu", + "id": "moz", + "region": 2 + }, { + "name": "Mulgi", + "id": "mvh", + "region": 2 + }, { + "name": "Mundabli", + "id": "boe", + "region": 2 + }, { + "name": "Mundang", + "id": "mua", + "region": 2 + }, { + "name": "Mundani", + "id": "mnf", + "region": 2 + }, { + "name": "Mundari (Oriya)", + "id": "unr-orya", + "region": 3 + }, { + "name": "M\u00fcnd\u00fc (Latin)", + "id": "muh-latn", + "region": 2 + }, { + "name": "Mungaka", + "id": "mhk", + "region": 2 + }, { + "name": "Mungbam", + "id": "mij", + "region": 2 + }, { + "name": "Munji (Arabic)", + "id": "mnj-arab", + "region": 3 + }, { + "name": "Munsee (Latin)", + "id": "umu-latn", + "region": 6 + }, { + "name": "Muria, Eastern (Devanagari)", + "id": "emu-deva", + "region": 3 + }, { + "name": "Muria, Far Western (Devanagari)", + "id": "fmu-deva", + "region": 3 + }, { + "name": "Muria, Western (Devanagari)", + "id": "mut-deva", + "region": 3 + }, { + "name": "Mursi (Latin)", + "id": "muz-latn", + "region": 2 + }, { + "name": "Musey", + "id": "mse", + "region": 2 + }, { + "name": "Musgu", + "id": "mug", + "region": 2 + }, { + "name": "Musi (Latin)", + "id": "mui-latn", + "region": 3 + }, { + "name": "Muskum", + "id": "mje", + "region": 2 + }, { + "name": "Muyang", + "id": "muy", + "region": 2 + }, { + "name": "Mvuba (Latin)", + "id": "mxh-latn", + "region": 2 + }, { + "name": "Mwaghavul (Latin)", + "id": "sur-latn", + "region": 2 + }, { + "name": "Mwera (Latin)", + "id": "mjh-latn", + "region": 2 + }, { + "name": "Mwera (Latin)", + "id": "mwe-latn", + "region": 2 + }, { + "name": "N\u2019ko", + "id": "nqo", + "region": 2 + }, { + "name": "Naaba", + "id": "nao", + "region": 3 + }, { + "name": "Naami", + "id": "bzv", + "region": 2 + }, { + "name": "Naba", + "id": "mne", + "region": 2 + }, { + "name": "Naga, Makuri (Latin)", + "id": "jmn-latn", + "region": 3 + }, { + "name": "Naga, Tangshang (Tangsa)", + "id": "nst-tnsa", + "region": 3 + }, { + "name": "Naga, Wancho (Wancho)", + "id": "nnp-wcho", + "region": 3 + }, { + "name": "Nagumi", + "id": "ngv", + "region": 2 + }, { + "name": "Nahuatl, Central", + "id": "nhn", + "region": 6 + }, { + "name": "Nahuatl, Central Huasteca", + "id": "nch", + "region": 6 + }, { + "name": "Nahuatl, Central Puebla", + "id": "ncx", + "region": 6 + }, { + "name": "Nahuatl, Coatepec", + "id": "naz", + "region": 6 + }, { + "name": "Nahuatl, Eastern Durango", + "id": "azd", + "region": 6 + }, { + "name": "Nahuatl, Eastern Huasteca", + "id": "nhe", + "region": 6 + }, { + "name": "Nahuatl, Guerrero", + "id": "ngu", + "region": 6 + }, { + "name": "Nahuatl, Highland Puebla", + "id": "azz", + "region": 6 + }, { + "name": "Nahuatl, Huaxcaleca", + "id": "nhq", + "region": 6 + }, { + "name": "Nahuatl, Isthmus-Cosoleacaque", + "id": "nhk", + "region": 6 + }, { + "name": "Nahuatl, Isthmus-Mecayapan", + "id": "nhx", + "region": 6 + }, { + "name": "Nahuatl, Isthmus-Pajapan", + "id": "nhp", + "region": 6 + }, { + "name": "Nahuatl, Michoac\u00e1n", + "id": "ncl", + "region": 6 + }, { + "name": "Nahuatl, Morelos", + "id": "nhm", + "region": 6 + }, { + "name": "Nahuatl, Northern Oaxaca", + "id": "nhy", + "region": 6 + }, { + "name": "Nahuatl, Northern Puebla", + "id": "ncj", + "region": 6 + }, { + "name": "Nahuatl, Ometepec", + "id": "nht", + "region": 6 + }, { + "name": "Nahuatl, Orizaba", + "id": "nlv", + "region": 6 + }, { + "name": "Nahuatl, Santa Mar\u00eda la Alta", + "id": "nhz", + "region": 6 + }, { + "name": "Nahuatl, Sierra Negra", + "id": "nsu", + "region": 6 + }, { + "name": "Nahuatl, Southeastern Puebla", + "id": "npl", + "region": 6 + }, { + "name": "Nahuatl, Temascaltepec", + "id": "nhv", + "region": 6 + }, { + "name": "Nahuatl, Tetelcingo", + "id": "nhg", + "region": 6 + }, { + "name": "Nahuatl, Tlamacazapa", + "id": "nuz", + "region": 6 + }, { + "name": "Nahuatl, Western Durango", + "id": "azn", + "region": 6 + }, { + "name": "Nahuatl, Western Huasteca", + "id": "nhw", + "region": 6 + }, { + "name": "Nahuatl, Zacatl\u00e1n-Ahuacatl\u00e1n-Tepetzintla", + "id": "nhi", + "region": 6 + }, { + "name": "Naki", + "id": "mff", + "region": 2 + }, { + "name": "Nancere", + "id": "nnc", + "region": 2 + }, { + "name": "Nande (Latin)", + "id": "nnb-latn", + "region": 2 + }, { + "name": "Napoletano-Calabrese", + "id": "nap", + "region": 4 + }, { + "name": "Naskapi (Unified Canadian Aboriginal Syllabics)", + "id": "nsk-cans", + "region": 6 + }, { + "name": "Navajo (Latin)", + "id": "nv-latn", + "region": 6 + }, { + "name": "Naxi (Lisu)", + "id": "nxq-lisu", + "region": 3 + }, { + "name": "Nchane", + "id": "ncr", + "region": 2 + }, { + "name": "Nda\u2019nda\u2019", + "id": "nnz", + "region": 2 + }, { + "name": "Ndai", + "id": "gke", + "region": 2 + }, { + "name": "Ndaka (Latin)", + "id": "ndk-latn", + "region": 2 + }, { + "name": "Ndali (Latin)", + "id": "ndh-latn", + "region": 2 + }, { + "name": "Ndam", + "id": "ndm", + "region": 2 + }, { + "name": "Ndamba (Latin)", + "id": "ndj-latn", + "region": 2 + }, { + "name": "Ndebele", + "id": "nd", + "region": 2 + }, { + "name": "Ndebele (Braille)", + "id": "nr-brai", + "region": 2 + }, { + "name": "Ndemli", + "id": "nml", + "region": 2 + }, { + "name": "Ndendeule (Latin)", + "id": "dne-latn", + "region": 2 + }, { + "name": "Ndengereko (Latin)", + "id": "ndg-latn", + "region": 2 + }, { + "name": "Ndo (Latin)", + "id": "ndp-latn", + "region": 2 + }, { + "name": "Ndonde Hamba (Latin)", + "id": "njd-latn", + "region": 2 + }, { + "name": "Ndoola", + "id": "ndr", + "region": 2 + }, { + "name": "Ndrulo (Latin)", + "id": "dno-latn", + "region": 2 + }, { + "name": "Ndut", + "id": "ndv", + "region": 2 + }, { + "name": "Negeri Sembilan Malay (Latin)", + "id": "zmi-latn", + "region": 3 + }, { + "name": "Nepali (Braille)", + "id": "npi-brai", + "region": 3 + }, { + "name": "Nepali (Devanagari)", + "id": "npi-deva", + "region": 3 + }, { + "name": "Nepali (macrolanguage)", + "id": "ne", + "region": 1 + }, { + "name": "Nepali (macrolanguage) (Braille)", + "id": "ne-brai", + "region": 1 + }, { + "name": "Newar", + "id": "new", + "region": 3 + }, { + "name": "Newar (Newa)", + "id": "new-newa", + "region": 3 + }, { + "name": "Ngam", + "id": "nmc", + "region": 2 + }, { + "name": "Ngamambo", + "id": "nbv", + "region": 2 + }, { + "name": "Ngambay", + "id": "sba", + "region": 2 + }, { + "name": "Ngando (Latin)", + "id": "nxd-latn", + "region": 2 + }, { + "name": "Ngas (Latin)", + "id": "anc-latn", + "region": 2 + }, { + "name": "Ngbandi, Northern (Latin)", + "id": "ngb-latn", + "region": 2 + }, { + "name": "Ngbee (Latin)", + "id": "jgb-latn", + "region": 2 + }, { + "name": "Ngelima (Latin)", + "id": "agh-latn", + "region": 2 + }, { + "name": "Ngemba", + "id": "nge", + "region": 2 + }, { + "name": "Ngete", + "id": "nnn", + "region": 2 + }, { + "name": "Ngie", + "id": "ngj", + "region": 2 + }, { + "name": "Ngiemboon", + "id": "nnh", + "region": 2 + }, { + "name": "Ngindo (Latin)", + "id": "nnq-latn", + "region": 2 + }, { + "name": "Ngiti (Latin)", + "id": "niy-latn", + "region": 2 + }, { + "name": "ngo-latn (Latin)", + "id": "ngo-latn", + "region": 1 + }, { + "name": "Ngomba", + "id": "jgo", + "region": 2 + }, { + "name": "Ngombale", + "id": "nla", + "region": 2 + }, { + "name": "Ngoreme (Latin)", + "id": "ngq-latn", + "region": 2 + }, { + "name": "Ngoshie", + "id": "nsh", + "region": 2 + }, { + "name": "Ngulu (Latin)", + "id": "ngp-latn", + "region": 2 + }, { + "name": "Ngwe", + "id": "nwe", + "region": 2 + }, { + "name": "Ngwo", + "id": "ngn", + "region": 2 + }, { + "name": "Nias (Latin)", + "id": "nia-latn", + "region": 3 + }, { + "name": "Nicaragua Creole English", + "id": "bzk", + "region": 6 + }, { + "name": "Niellim", + "id": "nie", + "region": 2 + }, { + "name": "Nilamba (Latin)", + "id": "nim-latn", + "region": 2 + }, { + "name": "Nimbari", + "id": "nmr", + "region": 2 + }, { + "name": "Nindi (Latin)", + "id": "nxi-latn", + "region": 2 + }, { + "name": "Nisenan", + "id": "nsz", + "region": 6 + }, { + "name": "Nisga\u2019a (Latin)", + "id": "ncg-latn", + "region": 6 + }, { + "name": "Niue", + "id": "niu", + "region": 7 + }, { + "name": "Nizaa", + "id": "sgi", + "region": 2 + }, { + "name": "Njen", + "id": "njj", + "region": 2 + }, { + "name": "Njyem", + "id": "njy", + "region": 2 + }, { + "name": "Nkongho", + "id": "nkc", + "region": 2 + }, { + "name": "Nkonya", + "id": "nko", + "region": 2 + }, { + "name": "Nobiin (Coptic)", + "id": "fia-copt", + "region": 2 + }, { + "name": "Nomaande", + "id": "lem", + "region": 2 + }, { + "name": "Noon", + "id": "snf", + "region": 2 + }, { + "name": "Noon (Latin)", + "id": "snf-latn", + "region": 2 + }, { + "name": "Noone", + "id": "nhu", + "region": 2 + }, { + "name": "Norwegian", + "id": "no", + "region": 4 + }, { + "name": "Norwegian (Runic)", + "id": "no-runr", + "region": 4 + }, { + "name": "Norwegian Bokm\u00e5l", + "id": "nb", + "region": 1 + }, { + "name": "Noy", + "id": "noy", + "region": 2 + }, { + "name": "Ntcham (Braille)", + "id": "bud-brai", + "region": 2 + }, { + "name": "Nubaca", + "id": "baf", + "region": 2 + }, { + "name": "Nuer (Latin)", + "id": "nus-latn", + "region": 2 + }, { + "name": "Nugunu", + "id": "yas", + "region": 2 + }, { + "name": "Numanggang (Latin)", + "id": "nop-latn", + "region": 7 + }, { + "name": "Nuosu", + "id": "ii", + "region": 3 + }, { + "name": "Nuu-chah-nulth (Latin)", + "id": "nuk-latn", + "region": 6 + }, { + "name": "Nya Huba (Latin)", + "id": "hbb-latn", + "region": 2 + }, { + "name": "Nyakyusa-Ngonde (Latin)", + "id": "nyy-latn", + "region": 2 + }, { + "name": "Nyali (Latin)", + "id": "nlj-latn", + "region": 2 + }, { + "name": "Nyambo (Latin)", + "id": "now-latn", + "region": 2 + }, { + "name": "Nyamwezi (Latin)", + "id": "nym-latn", + "region": 2 + }, { + "name": "Nyanga-li (Latin)", + "id": "nyc-latn", + "region": 2 + }, { + "name": "Nyangatom (Latin)", + "id": "nnj-latn", + "region": 2 + }, { + "name": "Nyankore (Latin)", + "id": "nyn-latn", + "region": 2 + }, { + "name": "Nyankpa", + "id": "yes", + "region": 2 + }, { + "name": "Nyaturu (Latin)", + "id": "rim-latn", + "region": 2 + }, { + "name": "Nyiha, Tanzania (Latin)", + "id": "nih-latn", + "region": 2 + }, { + "name": "Nyika, Tanzania (Latin)", + "id": "nkt-latn", + "region": 2 + }, { + "name": "Nyindu (Latin)", + "id": "nyg-latn", + "region": 2 + }, { + "name": "Nyokon", + "id": "nvo", + "region": 2 + }, { + "name": "Nyole (Latin)", + "id": "nuj-latn", + "region": 2 + }, { + "name": "Nyoro (Latin)", + "id": "nyo-latn", + "region": 2 + }, { + "name": "Nzakambay", + "id": "nzy", + "region": 2 + }, { + "name": "Nzakara (Latin)", + "id": "nzk-latn", + "region": 2 + }, { + "name": "Nzanyi", + "id": "nja", + "region": 2 + }, { + "name": "Nzema (Latin)", + "id": "nzi-latn", + "region": 2 + }, { + "name": "Oadki (Arabic)", + "id": "odk-arab", + "region": 3 + }, { + "name": "Oblo", + "id": "obl", + "region": 2 + }, { + "name": "Obolo (Latin)", + "id": "ann-latn", + "region": 2 + }, { + "name": "Occitan", + "id": "oc", + "region": 4 + }, { + "name": "Odia (Braille)", + "id": "ory-brai", + "region": 3 + }, { + "name": "Official Aramaic (700-300 BCE) (Hatran)", + "id": "arc-hatr", + "region": 1 + }, { + "name": "Official Aramaic (700-300 BCE) (Nabataean)", + "id": "arc-nbat", + "region": 1 + }, { + "name": "Official Aramaic (700-300 BCE) (Syriac)", + "id": "arc-syrc", + "region": 1 + }, { + "name": "Ojibwa (Unified Canadian Aboriginal Syllabics)", + "id": "oj-cans", + "region": 1 + }, { + "name": "Ojibwa, Northwestern (Unified Canadian Aboriginal Syllabics)", + "id": "ojb-cans", + "region": 6 + }, { + "name": "Oji-Cree", + "id": "ojs", + "region": 6 + }, { + "name": "Oji-Cree (Unified Canadian Aboriginal Syllabics)", + "id": "ojs-cans", + "region": 6 + }, { + "name": "Okanagan (Latin)", + "id": "oka-latn", + "region": 6 + }, { + "name": "Oku", + "id": "oku", + "region": 2 + }, { + "name": "Old Aramaic (up to 700 BCE) (Syriac)", + "id": "oar-syrc", + "region": 1 + }, { + "name": "Old English (ca. 450-1100)", + "id": "ang", + "region": 1 + }, { + "name": "Old English (ca. 450-1100) (Runic)", + "id": "ang-runr", + "region": 1 + }, { + "name": "Old Hungarian (Old Hungarian)", + "id": "ohu-hung", + "region": 1 + }, { + "name": "Old Norse", + "id": "non", + "region": 1 + }, { + "name": "Old Norse (Runic)", + "id": "non-runr", + "region": 1 + }, { + "name": "Old Tibetan (Tibetan)", + "id": "otb-tibt", + "region": 1 + }, { + "name": "Old Turkish (Old Turkic)", + "id": "otk-orkh", + "region": 1 + }, { + "name": "Ombo (Latin)", + "id": "oml-latn", + "region": 2 + }, { + "name": "Omi (Latin)", + "id": "omi-latn", + "region": 2 + }, { + "name": "Oneida (Latin)", + "id": "one-latn", + "region": 6 + }, { + "name": "Oniyan", + "id": "bsc", + "region": 2 + }, { + "name": "Onondaga (Latin)", + "id": "ono-latn", + "region": 6 + }, { + "name": "Oorlams", + "id": "oor", + "region": 2 + }, { + "name": "Oriya (macrolanguage)", + "id": "or", + "region": 1 + }, { + "name": "Oriya (macrolanguage) (Braille)", + "id": "or-brai", + "region": 1 + }, { + "name": "Orma", + "id": "orc", + "region": 2 + }, { + "name": "Orma (Latin)", + "id": "orc-latn", + "region": 2 + }, { + "name": "Ormuri (Arabic)", + "id": "oru-arab", + "region": 3 + }, { + "name": "Oroko", + "id": "bdu", + "region": 2 + }, { + "name": "Oromo", + "id": "om", + "region": 1 + }, { + "name": "Osage", + "id": "osa", + "region": 6 + }, { + "name": "Osage (Osage)", + "id": "osa-osge", + "region": 6 + }, { + "name": "Osatu", + "id": "ost", + "region": 2 + }, { + "name": "Oshiwambo (Braille)", + "id": "kj-brai", + "region": 2 + }, { + "name": "Osing (Javanese)", + "id": "osi-java", + "region": 3 + }, { + "name": "P\u00e1ez", + "id": "pbb", + "region": 6 + }, { + "name": "Pagibete (Latin)", + "id": "pae-latn", + "region": 2 + }, { + "name": "Pahari-Potwari", + "id": "phr", + "region": 3 + }, { + "name": "Pahari-Potwari (Arabic)", + "id": "phr-arab", + "region": 3 + }, { + "name": "Palatinate Franconian", + "id": "pfl", + "region": 4 + }, { + "name": "Palawano, Brooke\u2019s Point (Latin)", + "id": "plw-latn", + "region": 3 + }, { + "name": "Palawano, Central (Latin)", + "id": "plc-latn", + "region": 3 + }, { + "name": "Palawano, Southwest (Latin)", + "id": "plv-latn", + "region": 3 + }, { + "name": "Paloor", + "id": "fap", + "region": 2 + }, { + "name": "Palula", + "id": "phl", + "region": 3 + }, { + "name": "Palula (Arabic)", + "id": "phl-arab", + "region": 3 + }, { + "name": "Pam", + "id": "pmn", + "region": 2 + }, { + "name": "Pambia (Latin)", + "id": "pmb-latn", + "region": 2 + }, { + "name": "Pana", + "id": "pnz", + "region": 2 + }, { + "name": "Pangasinan (Braille)", + "id": "pag-brai", + "region": 3 + }, { + "name": "Pangasinan (Latin)", + "id": "pag-latn", + "region": 3 + }, { + "name": "Pangu", + "id": "png", + "region": 2 + }, { + "name": "Pangwa (Latin)", + "id": "pbr-latn", + "region": 2 + }, { + "name": "Papel", + "id": "pbo", + "region": 2 + }, { + "name": "Papiamentu", + "id": "pap", + "region": 6 + }, { + "name": "Paranan (Latin)", + "id": "prf-latn", + "region": 3 + }, { + "name": "Parkwa", + "id": "pbi", + "region": 2 + }, { + "name": "Pashai, Southeast (Arabic)", + "id": "psi-arab", + "region": 3 + }, { + "name": "Pashto, Central (Arabic)", + "id": "pst-arab", + "region": 3 + }, { + "name": "Pashto, Southern (Arabic)", + "id": "pbt-arab", + "region": 3 + }, { + "name": "Pattani (Tibetan)", + "id": "lae-tibt", + "region": 3 + }, { + "name": "Pengo (Oriya)", + "id": "peg-orya", + "region": 3 + }, { + "name": "Pere", + "id": "pfe", + "region": 2 + }, { + "name": "Persian", + "id": "fa", + "region": 1 + }, { + "name": "Persian, Iranian", + "id": "pes", + "region": 3 + }, { + "name": "Persian, Iranian (Arabic)", + "id": "pes-arab", + "region": 3 + }, { + "name": "P\u00e9v\u00e9", + "id": "lme", + "region": 2 + }, { + "name": "Phuie (Latin)", + "id": "pug-latn", + "region": 2 + }, { + "name": "Picard", + "id": "pcd", + "region": 4 + }, { + "name": "Pidgin, Cameroon", + "id": "wes", + "region": 2 + }, { + "name": "Pidgin, Cameroon (Latin)", + "id": "wes-latn", + "region": 2 + }, { + "name": "Piedmontese", + "id": "pms", + "region": 4 + }, { + "name": "Pijin", + "id": "pis", + "region": 7 + }, { + "name": "Pimbwe (Latin)", + "id": "piw-latn", + "region": 2 + }, { + "name": "Pingelapese (Latin)", + "id": "pif-latn", + "region": 7 + }, { + "name": "Pintupi-Luritja (Latin)", + "id": "piu-latn", + "region": 7 + }, { + "name": "Pinyin", + "id": "pny", + "region": 2 + }, { + "name": "Pitjantjatjara (Latin)", + "id": "pjt-latn", + "region": 7 + }, { + "name": "Plautdietsch", + "id": "pdt", + "region": 6 + }, { + "name": "Pogolo (Latin)", + "id": "poy-latn", + "region": 2 + }, { + "name": "Poke (Latin)", + "id": "pof-latn", + "region": 2 + }, { + "name": "P\u00f6koot (Latin)", + "id": "pko-latn", + "region": 2 + }, { + "name": "Pol", + "id": "pmm", + "region": 2 + }, { + "name": "Polish", + "id": "pl", + "region": 4 + }, { + "name": "Polish (Braille)", + "id": "pl-brai", + "region": 4 + }, { + "name": "Pontic (Greek)", + "id": "pnt-grek", + "region": 4 + }, { + "name": "Portuguese", + "id": "pt", + "region": 4 + }, { + "name": "Portuguese (Brazil)", + "id": "pt-br", + "region": 4 + }, { + "name": "Portuguese (Braille)", + "id": "pt-brai", + "region": 4 + }, { + "name": "Potawatomi (Latin)", + "id": "pot-latn", + "region": 6 + }, { + "name": "Primitive Irish (Ogham)", + "id": "pgl-ogam", + "region": 1 + }, { + "name": "Prussian", + "id": "prg", + "region": 4 + }, { + "name": "Psikye", + "id": "kvj", + "region": 2 + }, { + "name": "Pukapuka (Latin)", + "id": "pkp-latn", + "region": 7 + }, { + "name": "Pulaar (Latin)", + "id": "fuc-latn", + "region": 2 + }, { + "name": "Pular", + "id": "fuf", + "region": 2 + }, { + "name": "Pular (Adlam)", + "id": "fuf-adlm", + "region": 2 + }, { + "name": "Pular (Arabic)", + "id": "fuf-arab", + "region": 2 + }, { + "name": "Pular (Latin)", + "id": "fuf-latn", + "region": 2 + }, { + "name": "Punjabi, Eastern", + "id": "pa", + "region": 3 + }, { + "name": "Punjabi, Western", + "id": "pnb", + "region": 3 + }, { + "name": "Punjabi, Western (Arabic)", + "id": "pnb-arab", + "region": 3 + }, { + "name": "Purik (Tibetan)", + "id": "prx-tibt", + "region": 3 + }, { + "name": "Pushto", + "id": "ps", + "region": 1 + }, { + "name": "Qimant", + "id": "ahg", + "region": 2 + }, { + "name": "Quechua", + "id": "qu", + "region": 1 + }, { + "name": "Quechua (Bolivia)", + "id": "qu-bo", + "region": 1 + }, { + "name": "Quechua, Ambo-Pasco", + "id": "qva", + "region": 6 + }, { + "name": "Quechua, Arequipa-La Uni\u00f3n", + "id": "qxu", + "region": 6 + }, { + "name": "Quechua, Ayacucho", + "id": "quy", + "region": 6 + }, { + "name": "Quechua, Cajamarca", + "id": "qvc", + "region": 6 + }, { + "name": "Quechua, Cajatambo North Lima", + "id": "qvl", + "region": 6 + }, { + "name": "Quechua, Chachapoyas", + "id": "quk", + "region": 6 + }, { + "name": "Quechua, Chaupihuaranga", + "id": "qur", + "region": 6 + }, { + "name": "Quechua, Chincha", + "id": "qxc", + "region": 6 + }, { + "name": "Quechua, Chiqui\u00e1n", + "id": "qxa", + "region": 6 + }, { + "name": "Quechua, Corongo Ancash", + "id": "qwa", + "region": 6 + }, { + "name": "Quechua, Eastern Apur\u00edmac", + "id": "qve", + "region": 6 + }, { + "name": "Quechua, Huallaga", + "id": "qub", + "region": 6 + }, { + "name": "Quechua, Huamal\u00edes-Dos de Mayo Hu\u00e1nuco", + "id": "qvh", + "region": 6 + }, { + "name": "Quechua, Huaylas Ancash", + "id": "qwh", + "region": 6 + }, { + "name": "Quechua, Huaylla Wanca", + "id": "qvw", + "region": 6 + }, { + "name": "Quechua, Jauja Wanca", + "id": "qxw", + "region": 6 + }, { + "name": "Quechua, Lambayeque", + "id": "quf", + "region": 6 + }, { + "name": "Quechua, Margos-Yarowilca-Lauricocha", + "id": "qvm", + "region": 6 + }, { + "name": "Quechua, North Bolivian", + "id": "qul", + "region": 6 + }, { + "name": "Quechua, North Jun\u00edn", + "id": "qvn", + "region": 6 + }, { + "name": "Quechua, Northern Conchucos Ancash", + "id": "qxn", + "region": 6 + }, { + "name": "Quechua, Pacaraos", + "id": "qvp", + "region": 6 + }, { + "name": "Quechua, Panao", + "id": "qxh", + "region": 6 + }, { + "name": "Quechua, Puno", + "id": "qxp", + "region": 6 + }, { + "name": "Quechua, San Mart\u00edn", + "id": "qvs", + "region": 6 + }, { + "name": "Quechua, Santa Ana de Tusi Pasco", + "id": "qxt", + "region": 6 + }, { + "name": "Quechua, Sihuas Ancash", + "id": "qws", + "region": 6 + }, { + "name": "Quechua, Southern Conchucos", + "id": "qxo", + "region": 6 + }, { + "name": "Quechua, Southern Pastaza", + "id": "qup", + "region": 6 + }, { + "name": "Quechua, Yauyos", + "id": "qux", + "region": 6 + }, { + "name": "Quichua, Calder\u00f3n Highland", + "id": "qud", + "region": 6 + }, { + "name": "Quichua, Ca\u00f1ar Highland", + "id": "qxr", + "region": 6 + }, { + "name": "Quichua, Chimborazo Highland", + "id": "qug", + "region": 6 + }, { + "name": "Quichua, Imbabura Highland", + "id": "qvi", + "region": 6 + }, { + "name": "Quichua, Loja Highland", + "id": "qvj", + "region": 6 + }, { + "name": "Quichua, Napo", + "id": "qvo", + "region": 6 + }, { + "name": "Quichua, Northern Pastaza", + "id": "qvz", + "region": 6 + }, { + "name": "Quichua, Salasaca Highland", + "id": "qxl", + "region": 6 + }, { + "name": "Quichua, Santiago del Estero", + "id": "qus", + "region": 6 + }, { + "name": "Quichua, Tena Lowland", + "id": "quw", + "region": 6 + }, { + "name": "Quinault (Latin)", + "id": "qun-latn", + "region": 6 + }, { + "name": "Rade (Latin)", + "id": "rad-latn", + "region": 3 + }, { + "name": "Rajasthani (Devanagari)", + "id": "raj-deva", + "region": 1 + }, { + "name": "Rajbanshi", + "id": "rjs", + "region": 3 + }, { + "name": "Rawang (Latin)", + "id": "raw-latn", + "region": 3 + }, { + "name": "Rejang (Rejang)", + "id": "rej-rjng", + "region": 3 + }, { + "name": "Rendille (Latin)", + "id": "rel-latn", + "region": 2 + }, { + "name": "R\u00e9union Creole French", + "id": "rcf", + "region": 2 + }, { + "name": "Rigwe (Latin)", + "id": "iri-latn", + "region": 2 + }, { + "name": "Ripuarian", + "id": "ksh", + "region": 4 + }, { + "name": "Rohingya (Arabic)", + "id": "rhg-arab", + "region": 3 + }, { + "name": "Romagnol", + "id": "rgn", + "region": 4 + }, { + "name": "Romani, Balkan", + "id": "rmn", + "region": 4 + }, { + "name": "Romani, Carpathian", + "id": "rmc", + "region": 4 + }, { + "name": "Romani, Kalo Finnish", + "id": "rmf", + "region": 4 + }, { + "name": "Romani, Sinte", + "id": "rmo", + "region": 4 + }, { + "name": "Romani, Welsh", + "id": "rmw", + "region": 4 + }, { + "name": "Romanian", + "id": "ro", + "region": 4 + }, { + "name": "Romanian (Braille)", + "id": "ro-brai", + "region": 4 + }, { + "name": "Romanian, Megleno", + "id": "ruq", + "region": 4 + }, { + "name": "Romansh", + "id": "rm", + "region": 4 + }, { + "name": "Romany", + "id": "rom", + "region": 1 + }, { + "name": "Romblomanon (Latin)", + "id": "rol-latn", + "region": 3 + }, { + "name": "Rombo (Latin)", + "id": "rof-latn", + "region": 2 + }, { + "name": "Rotokas (Latin)", + "id": "roo-latn", + "region": 7 + }, { + "name": "Rufiji (Latin)", + "id": "rui-latn", + "region": 2 + }, { + "name": "Rundi (Braille)", + "id": "rn-brai", + "region": 2 + }, { + "name": "Runga", + "id": "rou", + "region": 2 + }, { + "name": "Rungwa (Latin)", + "id": "rnw-latn", + "region": 2 + }, { + "name": "Ruruuli-Runyala (Latin)", + "id": "ruc-latn", + "region": 2 + }, { + "name": "Russian", + "id": "ru", + "region": 4 + }, { + "name": "Russian (Braille)", + "id": "ru-brai", + "region": 4 + }, { + "name": "Rwa (Latin)", + "id": "rwk-latn", + "region": 2 + }, { + "name": "Saafi-Saafi", + "id": "sav", + "region": 2 + }, { + "name": "Saami, Inari", + "id": "smn", + "region": 4 + }, { + "name": "Saami, Lule", + "id": "smj", + "region": 4 + }, { + "name": "Saami, North", + "id": "se", + "region": 4 + }, { + "name": "Saami, North (Latin)", + "id": "se-latn", + "region": 4 + }, { + "name": "Saami, Skolt", + "id": "sms", + "region": 4 + }, { + "name": "Saami, South", + "id": "sma", + "region": 4 + }, { + "name": "Saami, South (Latin)", + "id": "sma-latn", + "region": 4 + }, { + "name": "Saami, Ume", + "id": "sju", + "region": 4 + }, { + "name": "Saamya-Gwe (Latin)", + "id": "lsm-latn", + "region": 2 + }, { + "name": "Saba", + "id": "saa", + "region": 2 + }, { + "name": "Sabaean (Old South Arabian)", + "id": "xsa-sarb", + "region": 1 + }, { + "name": "Sabaot (Latin)", + "id": "spy-latn", + "region": 2 + }, { + "name": "Sadri (Devanagari)", + "id": "sck-deva", + "region": 3 + }, { + "name": "Sadri, Oraon (Devanagari)", + "id": "sdr-deva", + "region": 3 + }, { + "name": "Safwa (Latin)", + "id": "sbk-latn", + "region": 2 + }, { + "name": "Sagala (Latin)", + "id": "sbm-latn", + "region": 2 + }, { + "name": "Saho (Latin)", + "id": "ssy-latn", + "region": 2 + }, { + "name": "Sahu (Latin)", + "id": "saj-latn", + "region": 3 + }, { + "name": "Saint Lucian Creole French", + "id": "acf", + "region": 6 + }, { + "name": "Salish, Straits", + "id": "str", + "region": 6 + }, { + "name": "Sama, Balangingih (Latin)", + "id": "sse-latn", + "region": 3 + }, { + "name": "Sama, Central (Latin)", + "id": "sml-latn", + "region": 3 + }, { + "name": "Sama, Pangutaran (Latin)", + "id": "slm-latn", + "region": 3 + }, { + "name": "Sama, Southern (Latin)", + "id": "ssb-latn", + "region": 3 + }, { + "name": "Samaritan Aramaic (Syriac)", + "id": "sam-syrc", + "region": 3 + }, { + "name": "Samba Leko", + "id": "ndi", + "region": 2 + }, { + "name": "Sambal (Latin)", + "id": "xsb-latn", + "region": 3 + }, { + "name": "Sambal, Botolan (Latin)", + "id": "sbl-latn", + "region": 3 + }, { + "name": "Sambalpuri (Oriya)", + "id": "spv-orya", + "region": 3 + }, { + "name": "Samoan", + "id": "sm", + "region": 7 + }, { + "name": "Sampang", + "id": "rav", + "region": 3 + }, { + "name": "Sanga (Braille)", + "id": "sng-brai", + "region": 2 + }, { + "name": "Sangil (Latin)", + "id": "snl-latn", + "region": 3 + }, { + "name": "Sango", + "id": "sg", + "region": 2 + }, { + "name": "Sangu (Latin)", + "id": "sbp-latn", + "region": 2 + }, { + "name": "Sanskrit", + "id": "sa", + "region": 3 + }, { + "name": "Sanskrit (Bhaiksuki)", + "id": "sa-bhks", + "region": 3 + }, { + "name": "Sanskrit (Brahmi)", + "id": "sa-brah", + "region": 3 + }, { + "name": "Sanskrit (Devanagari)", + "id": "sa-deva", + "region": 3 + }, { + "name": "Sanskrit (Kharoshthi)", + "id": "sa-khar", + "region": 3 + }, { + "name": "Sanskrit (Latin)", + "id": "sa-latn", + "region": 3 + }, { + "name": "Sanskrit (Myanmar)", + "id": "sa-mymr", + "region": 3 + }, { + "name": "Sanskrit (Siddham)", + "id": "sa-sidd", + "region": 3 + }, { + "name": "Sanskrit (Tirhuta)", + "id": "sa-tirh", + "region": 3 + }, { + "name": "Santhali (Latin)", + "id": "sat-latn", + "region": 3 + }, { + "name": "Santhali (Ol Chiki)", + "id": "sat-olck", + "region": 3 + }, { + "name": "Santhali (Oriya)", + "id": "sat-orya", + "region": 3 + }, { + "name": "Sapo (Latin)", + "id": "krn-latn", + "region": 2 + }, { + "name": "Sar", + "id": "mwm", + "region": 2 + }, { + "name": "Saraiki (Arabic)", + "id": "skr-arab", + "region": 3 + }, { + "name": "Saraiki (Gurmukhi)", + "id": "skr-guru", + "region": 3 + }, { + "name": "Saraiki (Multani)", + "id": "skr-mult", + "region": 3 + }, { + "name": "Saramaccan", + "id": "srm", + "region": 6 + }, { + "name": "Sardinian, Campidanese", + "id": "sro", + "region": 4 + }, { + "name": "Sardinian, Gallurese", + "id": "sdn", + "region": 4 + }, { + "name": "Sardinian, Sassarese", + "id": "sdc", + "region": 4 + }, { + "name": "Sari", + "id": "asj", + "region": 2 + }, { + "name": "Sarsi (Latin)", + "id": "srs-latn", + "region": 6 + }, { + "name": "Sarua", + "id": "swy", + "region": 2 + }, { + "name": "Sasak (Javanese)", + "id": "sas-java", + "region": 3 + }, { + "name": "Saterfriesisch", + "id": "stq", + "region": 4 + }, { + "name": "Sauria Paharia (Devanagari)", + "id": "mjt-deva", + "region": 3 + }, { + "name": "Saxon, East Frisian Low", + "id": "frs", + "region": 4 + }, { + "name": "Saxon, Low", + "id": "nds", + "region": 4 + }, { + "name": "Saxon, Low (Runic)", + "id": "nds-runr", + "region": 4 + }, { + "name": "Saxon, Upper", + "id": "sxu", + "region": 4 + }, { + "name": "Saxon, Upper (Runic)", + "id": "sxu-runr", + "region": 4 + }, { + "name": "Scots", + "id": "sco", + "region": 4 + }, { + "name": "Scots (Runic)", + "id": "sco-runr", + "region": 4 + }, { + "name": "Scottish Gaelic", + "id": "gd", + "region": 4 + }, { + "name": "Scottish Gaelic (Latin)", + "id": "gd-latn", + "region": 4 + }, { + "name": "Sea Island Creole English", + "id": "gul", + "region": 6 + }, { + "name": "Sebat Bet Gurage", + "id": "sgw", + "region": 2 + }, { + "name": "Sebat Bet Gurage (Ethiopic)", + "id": "sgw-ethi", + "region": 2 + }, { + "name": "Sechelt (Latin)", + "id": "sec-latn", + "region": 6 + }, { + "name": "Segeju (Latin)", + "id": "seg-latn", + "region": 2 + }, { + "name": "Sekani (Latin)", + "id": "sek-latn", + "region": 6 + }, { + "name": "Seke", + "id": "skj", + "region": 3 + }, { + "name": "Senaya (Syriac)", + "id": "syn-syrc", + "region": 3 + }, { + "name": "Seneca (Latin)", + "id": "see-latn", + "region": 6 + }, { + "name": "S\u00e9noufo, Mamara (Latin)", + "id": "myk-latn", + "region": 2 + }, { + "name": "S\u00e9noufo, Shempire (Latin)", + "id": "seb-latn", + "region": 2 + }, { + "name": "S\u00e9noufo, Supyire (Latin)", + "id": "spp-latn", + "region": 2 + }, { + "name": "S\u00e9noufo, Syenara (Latin)", + "id": "shz-latn", + "region": 2 + }, { + "name": "Serbian", + "id": "sr", + "region": 4 + }, { + "name": "Serbian (Braille)", + "id": "sr-brai", + "region": 4 + }, { + "name": "Serbian (Latin)", + "id": "sr-latn", + "region": 4 + }, { + "name": "Serbo-Croatian", + "id": "sh", + "region": 1 + }, { + "name": "Sere (Latin)", + "id": "swf-latn", + "region": 2 + }, { + "name": "Serer-Sine", + "id": "srr", + "region": 2 + }, { + "name": "Serer-Sine (Arabic)", + "id": "srr-arab", + "region": 2 + }, { + "name": "Serer-Sine (Latin)", + "id": "srr-latn", + "region": 2 + }, { + "name": "Setswana", + "id": "tn", + "region": 2 + }, { + "name": "Seychelles Creole French", + "id": "crs", + "region": 2 + }, { + "name": "Shambala (Latin)", + "id": "ksb-latn", + "region": 2 + }, { + "name": "Shan", + "id": "shn", + "region": 3 + }, { + "name": "Shan (Myanmar)", + "id": "shn-mymr", + "region": 3 + }, { + "name": "Shan (Tai Le)", + "id": "shn-tale", + "region": 3 + }, { + "name": "Sharwa", + "id": "swq", + "region": 2 + }, { + "name": "Shekkacho (Latin)", + "id": "moy-latn", + "region": 2 + }, { + "name": "Sheko (Latin)", + "id": "she-latn", + "region": 2 + }, { + "name": "Sherbro (Latin)", + "id": "bun-latn", + "region": 2 + }, { + "name": "Sherpa", + "id": "xsr", + "region": 3 + }, { + "name": "Sherpa (Tibetan)", + "id": "xsr-tibt", + "region": 3 + }, { + "name": "Shi (Latin)", + "id": "shr-latn", + "region": 2 + }, { + "name": "Shina", + "id": "scl", + "region": 3 + }, { + "name": "Shina (Arabic, India)", + "id": "scl-arab-in", + "region": 3 + }, { + "name": "Shina (Arabic, Pakistan)", + "id": "scl-arab-pk", + "region": 3 + }, { + "name": "Shona (Braille)", + "id": "sn-brai", + "region": 2 + }, { + "name": "Shubi (Latin)", + "id": "suj-latn", + "region": 2 + }, { + "name": "Shuswap (Latin)", + "id": "shs-latn", + "region": 6 + }, { + "name": "Sicilian", + "id": "scn", + "region": 4 + }, { + "name": "Sidamo", + "id": "sid", + "region": 2 + }, { + "name": "Sidamo (Latin)", + "id": "sid-latn", + "region": 2 + }, { + "name": "Sikkimese (Tibetan)", + "id": "sip-tibt", + "region": 3 + }, { + "name": "Silesian", + "id": "szl", + "region": 4 + }, { + "name": "Silesian, Lower", + "id": "sli", + "region": 4 + }, { + "name": "Silt\u2019e", + "id": "stv", + "region": 2 + }, { + "name": "Sindhi", + "id": "sd", + "region": 3 + }, { + "name": "Sindhi (Arabic)", + "id": "sd-arab", + "region": 3 + }, { + "name": "Sindhi (Gurmukhi)", + "id": "sd-guru", + "region": 3 + }, { + "name": "Sindhi (Khojki)", + "id": "sd-khoj", + "region": 3 + }, { + "name": "Sinhala", + "id": "si", + "region": 3 + }, { + "name": "Sinhala (Braille)", + "id": "si-brai", + "region": 3 + }, { + "name": "Sinyar", + "id": "sys", + "region": 2 + }, { + "name": "Sirion\u00f3", + "id": "srq", + "region": 6 + }, { + "name": "Siuslaw", + "id": "sis", + "region": 6 + }, { + "name": "Slave (Athapascan)", + "id": "den", + "region": 1 + }, { + "name": "Slavey, North", + "id": "scs", + "region": 6 + }, { + "name": "Slavey, North (Latin)", + "id": "scs-latn", + "region": 6 + }, { + "name": "Slavey, South", + "id": "xsl", + "region": 6 + }, { + "name": "Slavey, South (Latin)", + "id": "xsl-latn", + "region": 6 + }, { + "name": "Slovak", + "id": "sk", + "region": 4 + }, { + "name": "Slovak (Braille)", + "id": "sk-brai", + "region": 4 + }, { + "name": "Slovene", + "id": "sl", + "region": 4 + }, { + "name": "Slovene (Braille)", + "id": "sl-brai", + "region": 4 + }, { + "name": "So (Latin)", + "id": "soc-latn", + "region": 2 + }, { + "name": "Soga (Latin)", + "id": "xog-latn", + "region": 2 + }, { + "name": "Sokoro", + "id": "sok", + "region": 2 + }, { + "name": "Somali", + "id": "so", + "region": 2 + }, { + "name": "Somali (Osmanya)", + "id": "so-osma", + "region": 2 + }, { + "name": "Somrai", + "id": "sor", + "region": 2 + }, { + "name": "Songhay, Humburi Senni (Latin)", + "id": "hmb-latn", + "region": 2 + }, { + "name": "Songhay, Koyra Chiini (Latin)", + "id": "khq-latn", + "region": 2 + }, { + "name": "Songhay, Koyraboro Senni (Latin)", + "id": "ses-latn", + "region": 2 + }, { + "name": "Songoora (Latin)", + "id": "sod-latn", + "region": 2 + }, { + "name": "Soninke (Latin)", + "id": "snk-latn", + "region": 2 + }, { + "name": "Soqotri (Arabic)", + "id": "sqt-arab", + "region": 3 + }, { + "name": "Sora (Oriya)", + "id": "srb-orya", + "region": 3 + }, { + "name": "Sora (Sora Sompeng)", + "id": "srb-sora", + "region": 3 + }, { + "name": "Sorbian, Lower", + "id": "dsb", + "region": 4 + }, { + "name": "Sorbian, Upper", + "id": "hsb", + "region": 4 + }, { + "name": "Sotho, Northern (Braille)", + "id": "nso-brai", + "region": 2 + }, { + "name": "Sougb (Latin)", + "id": "mnx-latn", + "region": 3 + }, { + "name": "Spanish", + "id": "es", + "region": 4 + }, { + "name": "Spanish (Braille)", + "id": "es-brai", + "region": 4 + }, { + "name": "Spanish, Charapa", + "id": "spq", + "region": 6 + }, { + "name": "Squamish (Latin)", + "id": "squ-latn", + "region": 6 + }, { + "name": "Sranan Tongo", + "id": "srn", + "region": 6 + }, { + "name": "Stellingwerfs", + "id": "stl", + "region": 4 + }, { + "name": "Stod Bhoti (Tibetan)", + "id": "sbu-tibt", + "region": 3 + }, { + "name": "Stoney (Latin)", + "id": "sto-latn", + "region": 6 + }, { + "name": "Subanen, Central (Latin)", + "id": "syb-latn", + "region": 3 + }, { + "name": "Subanen, Northern (Latin)", + "id": "stb-latn", + "region": 3 + }, { + "name": "Subanen, Southern (Latin)", + "id": "laa-latn", + "region": 3 + }, { + "name": "Subanon, Kolibugan (Latin)", + "id": "skn-latn", + "region": 3 + }, { + "name": "Subanon, Western (Latin)", + "id": "suc-latn", + "region": 3 + }, { + "name": "Suba-Simbiti (Latin)", + "id": "ssc-latn", + "region": 2 + }, { + "name": "Sukuma (Latin)", + "id": "suk-latn", + "region": 2 + }, { + "name": "Sumbwa (Latin)", + "id": "suw-latn", + "region": 2 + }, { + "name": "Sunda (Javanese)", + "id": "su-java", + "region": 3 + }, { + "name": "Sunda (Latin)", + "id": "su-latn", + "region": 3 + }, { + "name": "Sunda (Sundanese)", + "id": "su-sund", + "region": 3 + }, { + "name": "Sunwar", + "id": "suz", + "region": 3 + }, { + "name": "Supapya", + "id": "bbw", + "region": 2 + }, { + "name": "Surbakhal", + "id": "sbj", + "region": 2 + }, { + "name": "Suri (Ethiopic)", + "id": "suq-ethi", + "region": 2 + }, { + "name": "Suri (Latin)", + "id": "suq-latn", + "region": 2 + }, { + "name": "Surigaonon (Latin)", + "id": "sgd-latn", + "region": 3 + }, { + "name": "Swabian", + "id": "swg", + "region": 4 + }, { + "name": "Swahili (Braille)", + "id": "swh-brai", + "region": 2 + }, { + "name": "Swahili (Latin)", + "id": "swh-latn", + "region": 2 + }, { + "name": "Swahili (macrolanguage)", + "id": "sw", + "region": 1 + }, { + "name": "Swahili, Congo (Latin)", + "id": "swc-latn", + "region": 2 + }, { + "name": "Swati", + "id": "ss", + "region": 2 + }, { + "name": "Swedish", + "id": "sv", + "region": 4 + }, { + "name": "Swedish (Braille)", + "id": "sv-brai", + "region": 4 + }, { + "name": "Swedish (Runic)", + "id": "sv-runr", + "region": 4 + }, { + "name": "Swo", + "id": "sox", + "region": 2 + }, { + "name": "Sylheti (Bengali)", + "id": "syl-beng", + "region": 3 + }, { + "name": "Sylheti (Syloti Nagri)", + "id": "syl-sylo", + "region": 3 + }, { + "name": "Syriac (Syriac)", + "id": "syc-syrc", + "region": 3 + }, { + "name": "Syriac (Syriac)", + "id": "syr-syrc", + "region": 1 + }, { + "name": "Syuba", + "id": "syw", + "region": 3 + }, { + "name": "T\u2019apo (Latin)", + "id": "lgn-latn", + "region": 2 + }, { + "name": "Taabwa (Latin)", + "id": "tap-latn", + "region": 2 + }, { + "name": "Tacana", + "id": "tna", + "region": 6 + }, { + "name": "Tachawit (Tifinagh)", + "id": "shy-tfng", + "region": 2 + }, { + "name": "Tachelhit (Latin)", + "id": "shi-latn", + "region": 2 + }, { + "name": "Tadaksahak (Latin)", + "id": "dsq-latn", + "region": 2 + }, { + "name": "Tagabawa (Latin)", + "id": "bgs-latn", + "region": 3 + }, { + "name": "Tagakaulo (Latin)", + "id": "klg-latn", + "region": 3 + }, { + "name": "Tagalog", + "id": "tl", + "region": 3 + }, { + "name": "Tagalog (Braille)", + "id": "tl-brai", + "region": 3 + }, { + "name": "Tagalog (Buhid)", + "id": "tl-buhd", + "region": 3 + }, { + "name": "Tagalog (Hanunoo)", + "id": "tl-hano", + "region": 3 + }, { + "name": "Tagalog (Tagalog)", + "id": "tl-tglg", + "region": 3 + }, { + "name": "Tagargrent (Latin)", + "id": "oua-latn", + "region": 2 + }, { + "name": "Tagbanwa (Hanunoo)", + "id": "tbw-hano", + "region": 3 + }, { + "name": "Tagbanwa (Latin)", + "id": "tbw-latn", + "region": 3 + }, { + "name": "Tagbanwa, Calamian (Hanunoo)", + "id": "tbk-hano", + "region": 3 + }, { + "name": "Tagbanwa, Calamian (Latin)", + "id": "tbk-latn", + "region": 3 + }, { + "name": "Tagbanwa, Central (Hanunoo)", + "id": "tgt-hano", + "region": 3 + }, { + "name": "Tagbanwa, Central (Latin)", + "id": "tgt-latn", + "region": 3 + }, { + "name": "Tagbanwa, Central (Tagbanwa)", + "id": "tgt-tagb", + "region": 3 + }, { + "name": "Tagdal (Tifinagh)", + "id": "tda-tfng", + "region": 2 + }, { + "name": "Tagish (Latin)", + "id": "tgx-latn", + "region": 6 + }, { + "name": "Tahitian (Latin)", + "id": "ty-latn", + "region": 7 + }, { + "name": "Tahltan (Latin)", + "id": "tht-latn", + "region": 6 + }, { + "name": "Tai Dam", + "id": "blt", + "region": 3 + }, { + "name": "Tai Dam (Lao)", + "id": "blt-laoo", + "region": 3 + }, { + "name": "Tai Dam (Latin)", + "id": "blt-latn", + "region": 3 + }, { + "name": "Tai Dam (Tai Viet)", + "id": "blt-tavt", + "region": 3 + }, { + "name": "Tai D\u00f3n (Latin)", + "id": "twh-latn", + "region": 3 + }, { + "name": "Tai D\u00f3n (Tai Viet)", + "id": "twh-tavt", + "region": 3 + }, { + "name": "Tai N\u00fca (Tai Le)", + "id": "tdd-tale", + "region": 3 + }, { + "name": "Tajik (Cyrillic)", + "id": "tg-cyrl", + "region": 3 + }, { + "name": "Tajuasohn (Latin)", + "id": "tja-latn", + "region": 2 + }, { + "name": "Talinga-Bwisi (Latin)", + "id": "tlj-latn", + "region": 2 + }, { + "name": "Tama", + "id": "tma", + "region": 2 + }, { + "name": "Tamahaq, Tahaggart (Tifinagh)", + "id": "thv-tfng", + "region": 2 + }, { + "name": "Tamajaq, Tawallammat (Latin)", + "id": "ttq-latn", + "region": 2 + }, { + "name": "Tamajaq, Tawallammat (Tifinagh)", + "id": "ttq-tfng", + "region": 2 + }, { + "name": "Tamajeq, Tayart (Tifinagh)", + "id": "thz-tfng", + "region": 2 + }, { + "name": "Tamang, Eastern", + "id": "taj", + "region": 3 + }, { + "name": "Tamang, Eastern (Tibetan)", + "id": "taj-tibt", + "region": 3 + }, { + "name": "Tamang, Western", + "id": "tdg", + "region": 3 + }, { + "name": "Tamang, Western (Tibetan)", + "id": "tdg-tibt", + "region": 3 + }, { + "name": "Tamashek", + "id": "tmh", + "region": 1 + }, { + "name": "Tamashek (Latin)", + "id": "tmh-latn", + "region": 1 + }, { + "name": "Tamashek (Tifinagh)", + "id": "tmh-tfng", + "region": 1 + }, { + "name": "Tamasheq (Latin)", + "id": "taq-latn", + "region": 2 + }, { + "name": "Tamasheq (Tifinagh)", + "id": "taq-tfng", + "region": 2 + }, { + "name": "Tamazight, Central Atlas", + "id": "tzm", + "region": 2 + }, { + "name": "Tamazight, Central Atlas (Tifinagh)", + "id": "tzm-tfng", + "region": 2 + }, { + "name": "Tamazight, Central Atlas (Tifinagh, Morocco)", + "id": "tzm-tfng-ma", + "region": 2 + }, { + "name": "Tamazight, Standard Moroccan (Tifinagh)", + "id": "zgh-tfng", + "region": 2 + }, { + "name": "Tamil", + "id": "ta", + "region": 3 + }, { + "name": "Tamil (Braille)", + "id": "ta-brai", + "region": 3 + }, { + "name": "Tamil (Latin)", + "id": "ta-latn", + "region": 3 + }, { + "name": "Tamki", + "id": "tax", + "region": 2 + }, { + "name": "Tampuan (Khmer)", + "id": "tpu-khmr", + "region": 3 + }, { + "name": "Tanacross (Latin)", + "id": "tcb-latn", + "region": 6 + }, { + "name": "Tanana, Upper (Latin)", + "id": "tau-latn", + "region": 6 + }, { + "name": "Tarifit (Tifinagh)", + "id": "rif-tfng", + "region": 2 + }, { + "name": "Tatar (Cyrillic)", + "id": "tt-cyrl", + "region": 4 + }, { + "name": "Tausug (Latin)", + "id": "tsg-latn", + "region": 3 + }, { + "name": "Tawbuid (Latin)", + "id": "twb-latn", + "region": 3 + }, { + "name": "Tayo", + "id": "cks", + "region": 7 + }, { + "name": "Tboli (Latin)", + "id": "tbl-latn", + "region": 3 + }, { + "name": "Tedaga", + "id": "tuq", + "region": 2 + }, { + "name": "Teduray (Latin)", + "id": "tiy-latn", + "region": 3 + }, { + "name": "Telugu", + "id": "te", + "region": 3 + }, { + "name": "Telugu (Braille)", + "id": "te-brai", + "region": 3 + }, { + "name": "Tem (Arabic)", + "id": "kdh-arab", + "region": 2 + }, { + "name": "Tembo (Latin)", + "id": "tbt-latn", + "region": 2 + }, { + "name": "Temi (Latin)", + "id": "soz-latn", + "region": 2 + }, { + "name": "Tengger (Javanese)", + "id": "tes-java", + "region": 3 + }, { + "name": "Tepehuan, Southeastern (Latin)", + "id": "stp-latn", + "region": 6 + }, { + "name": "Tepehuan, Southwestern (Latin)", + "id": "tla-latn", + "region": 6 + }, { + "name": "Termanu (Latin)", + "id": "twu-latn", + "region": 3 + }, { + "name": "Tetun (Latin)", + "id": "tet-latn", + "region": 3 + }, { + "name": "Thai", + "id": "th", + "region": 3 + }, { + "name": "Thai (Braille)", + "id": "th-brai", + "region": 3 + }, { + "name": "Thangmi", + "id": "thf", + "region": 3 + }, { + "name": "Tharu, Dangaura", + "id": "thl", + "region": 3 + }, { + "name": "Tharu, Madhya Ksetriya", + "id": "the", + "region": 3 + }, { + "name": "Tharu, Madhya-Purbiya", + "id": "thq", + "region": 3 + }, { + "name": "Themne", + "id": "tem", + "region": 2 + }, { + "name": "Thompson (Latin)", + "id": "thp-latn", + "region": 6 + }, { + "name": "Thulung", + "id": "tdh", + "region": 3 + }, { + "name": "Tibea", + "id": "ngy", + "region": 2 + }, { + "name": "Tibetan, Amdo (Tibetan)", + "id": "adx-tibt", + "region": 3 + }, { + "name": "Tibetan, Central (Phags-pa, China)", + "id": "bo-phag-cn", + "region": 3 + }, { + "name": "Tibetan, Central (Tibetan)", + "id": "bo-tibt", + "region": 3 + }, { + "name": "Tibetan, Central (Tibetan, China)", + "id": "bo-tibt-cn", + "region": 3 + }, { + "name": "Tibetan, Khams (Tibetan)", + "id": "khg-tibt", + "region": 3 + }, { + "name": "Tigr\u00e9", + "id": "tig", + "region": 2 + }, { + "name": "Tigr\u00e9 (Ethiopic)", + "id": "tig-ethi", + "region": 2 + }, { + "name": "Tigrigna", + "id": "ti", + "region": 2 + }, { + "name": "Tigrigna (Ethiopia)", + "id": "ti-et", + "region": 2 + }, { + "name": "Tikar", + "id": "tik", + "region": 2 + }, { + "name": "Tinani (Tibetan)", + "id": "lbf-tibt", + "region": 3 + }, { + "name": "Tiv", + "id": "tiv", + "region": 2 + }, { + "name": "Tiv (Latin)", + "id": "tiv-latn", + "region": 2 + }, { + "name": "Tlicho", + "id": "dgr", + "region": 6 + }, { + "name": "Tlicho (Latin)", + "id": "dgr-latn", + "region": 6 + }, { + "name": "Tlingit", + "id": "tli", + "region": 6 + }, { + "name": "To", + "id": "toz", + "region": 2 + }, { + "name": "Toba (Latin)", + "id": "tob-latn", + "region": 6 + }, { + "name": "Tobanga", + "id": "tng", + "region": 2 + }, { + "name": "Tohono O\u2019odham (Latin)", + "id": "ood-latn", + "region": 6 + }, { + "name": "Tok Pisin", + "id": "tpi", + "region": 7 + }, { + "name": "Tongan", + "id": "to", + "region": 7 + }, { + "name": "Tongwe (Latin)", + "id": "tny-latn", + "region": 2 + }, { + "name": "Tooro (Latin)", + "id": "ttj-latn", + "region": 2 + }, { + "name": "Toraja-Sa\u2019dan (Latin)", + "id": "sda-latn", + "region": 3 + }, { + "name": "Toram", + "id": "trj", + "region": 2 + }, { + "name": "Torres Strait Creole", + "id": "tcs", + "region": 7 + }, { + "name": "Torwali (Arabic)", + "id": "trw-arab", + "region": 3 + }, { + "name": "Toto", + "id": "txo", + "region": 3 + }, { + "name": "Tsakonian", + "id": "tsd", + "region": 4 + }, { + "name": "Tsakonian (Greek)", + "id": "tsd-grek", + "region": 4 + }, { + "name": "Tsamai (Latin)", + "id": "tsb-latn", + "region": 2 + }, { + "name": "Tshangla", + "id": "tsj", + "region": 3 + }, { + "name": "Tshangla (Tibetan)", + "id": "tsj-tibt", + "region": 3 + }, { + "name": "Tsikimba (Latin)", + "id": "kdl-latn", + "region": 2 + }, { + "name": "Tsimshian (Latin)", + "id": "tsi-latn", + "region": 6 + }, { + "name": "Tsonga (Braille)", + "id": "ts-brai", + "region": 2 + }, { + "name": "Tsuvan", + "id": "tsh", + "region": 2 + }, { + "name": "Tuki", + "id": "bag", + "region": 2 + }, { + "name": "Tumak", + "id": "tmc", + "region": 2 + }, { + "name": "Tumzabt (Tifinagh)", + "id": "mzb-tfng", + "region": 2 + }, { + "name": "Tunen", + "id": "tvu", + "region": 2 + }, { + "name": "Tunia", + "id": "tug", + "region": 2 + }, { + "name": "Tuotomb", + "id": "ttf", + "region": 2 + }, { + "name": "Tupuri", + "id": "tui", + "region": 2 + }, { + "name": "Turkish", + "id": "tr", + "region": 3 + }, { + "name": "Turkish (Braille)", + "id": "tr-brai", + "region": 3 + }, { + "name": "Turkmen", + "id": "tk", + "region": 3 + }, { + "name": "Turkmen (Cyrillic)", + "id": "tk-cyrl", + "region": 3 + }, { + "name": "Turkmen (Latin)", + "id": "tk-latn", + "region": 3 + }, { + "name": "Turoyo (Syriac)", + "id": "tru-syrc", + "region": 3 + }, { + "name": "Tuscarora (Latin)", + "id": "tus-latn", + "region": 6 + }, { + "name": "Tutchone, Northern (Latin)", + "id": "ttm-latn", + "region": 6 + }, { + "name": "Tutchone, Southern (Latin)", + "id": "tce-latn", + "region": 6 + }, { + "name": "Twendi", + "id": "twn", + "region": 2 + }, { + "name": "Twents", + "id": "twd", + "region": 4 + }, { + "name": "Tyap (Latin)", + "id": "kcg-latn", + "region": 2 + }, { + "name": "Ubi", + "id": "ubi", + "region": 2 + }, { + "name": "Udi (Latin)", + "id": "udi-latn", + "region": 3 + }, { + "name": "Ukrainian", + "id": "uk", + "region": 4 + }, { + "name": "Ukwuani-Aboh-Ndoni", + "id": "ukw", + "region": 2 + }, { + "name": "Ulithian (Latin)", + "id": "uli-latn", + "region": 7 + }, { + "name": "Umatilla", + "id": "uma", + "region": 6 + }, { + "name": "Undetermined", + "id": "und", + "region": 1 + }, { + "name": "Undetermined", + "id": "und-fonipa", + "region": 1 + }, { + "name": "Undetermined (Latin)", + "id": "und-latn", + "region": 1 + }, { + "name": "Urdu", + "id": "ur", + "region": 3 + }, { + "name": "Urdu (Braille)", + "id": "ur-brai", + "region": 3 + }, { + "name": "Urhobo", + "id": "urh", + "region": 2 + }, { + "name": "Usaghade", + "id": "usk", + "region": 2 + }, { + "name": "Ushojo (Arabic)", + "id": "ush-arab", + "region": 3 + }, { + "name": "Uyghur (Arabic)", + "id": "ug-arab", + "region": 3 + }, { + "name": "Uzbek (Braille)", + "id": "uz-brai", + "region": 1 + }, { + "name": "Uzbek (Latin)", + "id": "uz-latn", + "region": 1 + }, { + "name": "Uzbek, Northern (Braille)", + "id": "uzn-brai", + "region": 3 + }, { + "name": "Uzbek, Northern (Cyrillic)", + "id": "uzn-cyrl", + "region": 3 + }, { + "name": "Vai (Latin)", + "id": "vai-latn", + "region": 2 + }, { + "name": "Vai (Vai)", + "id": "vai-vaii", + "region": 2 + }, { + "name": "Vame", + "id": "mlr", + "region": 2 + }, { + "name": "Vanuma (Latin)", + "id": "vau-latn", + "region": 2 + }, { + "name": "Vemgo-Mabas", + "id": "vem", + "region": 2 + }, { + "name": "Venda (Braille)", + "id": "ve-brai", + "region": 2 + }, { + "name": "Venetian", + "id": "vec", + "region": 4 + }, { + "name": "Venetian (Latin)", + "id": "vec-latn", + "region": 4 + }, { + "name": "Vengo", + "id": "bav", + "region": 2 + }, { + "name": "Veps", + "id": "vep", + "region": 4 + }, { + "name": "Verre", + "id": "ver", + "region": 2 + }, { + "name": "Vidunda (Latin)", + "id": "vid-latn", + "region": 2 + }, { + "name": "Vietnamese", + "id": "vi", + "region": 3 + }, { + "name": "Vietnamese (Braille)", + "id": "vi-brai", + "region": 3 + }, { + "name": "Vincentian Creole English", + "id": "svc", + "region": 6 + }, { + "name": "Vinza (Latin)", + "id": "vin-latn", + "region": 2 + }, { + "name": "Virgin Islands Creole English", + "id": "vic", + "region": 6 + }, { + "name": "Vlaams", + "id": "vls", + "region": 4 + }, { + "name": "Vod", + "id": "vot", + "region": 4 + }, { + "name": "Vunjo (Latin)", + "id": "vun-latn", + "region": 2 + }, { + "name": "Vute", + "id": "vut", + "region": 2 + }, { + "name": "Vwanji (Latin)", + "id": "wbi-latn", + "region": 2 + }, { + "name": "Wakhi (Arabic)", + "id": "wbl-arab", + "region": 3 + }, { + "name": "Wakhi (Cyrillic, Tajikistan)", + "id": "wbl-cyrl-tj", + "region": 3 + }, { + "name": "Wakhi (Latin)", + "id": "wbl-latn", + "region": 3 + }, { + "name": "Wakhi (Latin, Pakistan)", + "id": "wbl-latn-pk", + "region": 3 + }, { + "name": "Walloon", + "id": "wa", + "region": 4 + }, { + "name": "Walser", + "id": "wae", + "region": 4 + }, { + "name": "Walungge", + "id": "ola", + "region": 3 + }, { + "name": "Walungge (Tibetan)", + "id": "ola-tibt", + "region": 3 + }, { + "name": "Wambule", + "id": "wme", + "region": 3 + }, { + "name": "Wamey", + "id": "cou", + "region": 2 + }, { + "name": "Wanda (Latin)", + "id": "wbh-latn", + "region": 2 + }, { + "name": "Wandala", + "id": "mfi", + "region": 2 + }, { + "name": "Waray-Waray (Braille)", + "id": "war-brai", + "region": 3 + }, { + "name": "Waray-Waray (Latin)", + "id": "war-latn", + "region": 3 + }, { + "name": "Wawa", + "id": "www", + "region": 2 + }, { + "name": "Wayuu", + "id": "guc", + "region": 6 + }, { + "name": "Weh", + "id": "weh", + "region": 2 + }, { + "name": "Welsh", + "id": "cy", + "region": 4 + }, { + "name": "Welsh (Braille)", + "id": "cy-brai", + "region": 4 + }, { + "name": "Western Neo-Aramaic (Syriac (Western variant))", + "id": "amw-syrj", + "region": 3 + }, { + "name": "Westphalien", + "id": "wep", + "region": 4 + }, { + "name": "Wolaytta (Latin)", + "id": "wal-latn", + "region": 2 + }, { + "name": "Wolof", + "id": "wo", + "region": 2 + }, { + "name": "Wolof (Arabic)", + "id": "wo-arab", + "region": 2 + }, { + "name": "Wolof (Latin)", + "id": "wo-latn", + "region": 2 + }, { + "name": "Wolof, Gambian", + "id": "wof", + "region": 2 + }, { + "name": "Wumboko", + "id": "bqm", + "region": 2 + }, { + "name": "Wushi", + "id": "bse", + "region": 2 + }, { + "name": "Wuzlam", + "id": "udl", + "region": 2 + }, { + "name": "wya-latn (Latin)", + "id": "wya-latn", + "region": 1 + }, { + "name": "Wymysorys", + "id": "wym", + "region": 4 + }, { + "name": "Xaasongaxango (Latin)", + "id": "kao-latn", + "region": 2 + }, { + "name": "Xamtanga", + "id": "xan", + "region": 2 + }, { + "name": "Xav\u00e1nte", + "id": "xav", + "region": 6 + }, { + "name": "Xhosa", + "id": "xh", + "region": 2 + }, { + "name": "Xhosa (Braille)", + "id": "xh-brai", + "region": 2 + }, { + "name": "Yadgha (Arabic)", + "id": "ydg-arab", + "region": 3 + }, { + "name": "Yakama (Latin)", + "id": "yak-latn", + "region": 6 + }, { + "name": "Yakan (Latin)", + "id": "yka-latn", + "region": 3 + }, { + "name": "Yakkha", + "id": "ybh", + "region": 3 + }, { + "name": "Yakut (Cyrillic, Russian Federation)", + "id": "sah-cyrl-ru", + "region": 4 + }, { + "name": "Yamba", + "id": "yam", + "region": 2 + }, { + "name": "Yambeta", + "id": "yat", + "region": 2 + }, { + "name": "Yaminahua", + "id": "yaa", + "region": 6 + }, { + "name": "Yamphu", + "id": "ybi", + "region": 3 + }, { + "name": "Yangben", + "id": "yav", + "region": 2 + }, { + "name": "Yapese (Latin)", + "id": "yap-latn", + "region": 7 + }, { + "name": "Yasa", + "id": "yko", + "region": 2 + }, { + "name": "Yele", + "id": "yle", + "region": 7 + }, { + "name": "Yemba", + "id": "ybb", + "region": 2 + }, { + "name": "Yemsa", + "id": "jnj", + "region": 2 + }, { + "name": "Yeni", + "id": "yei", + "region": 2 + }, { + "name": "Yevanic (Greek)", + "id": "yej-grek", + "region": 3 + }, { + "name": "Yi, Wuding-Luquan (Miao)", + "id": "ywq-plrd", + "region": 3 + }, { + "name": "Yiddish, Eastern (Hebrew)", + "id": "ydd-hebr", + "region": 3 + }, { + "name": "Yiddish, Western (Hebrew)", + "id": "yih-hebr", + "region": 4 + }, { + "name": "Yogad (Latin)", + "id": "yog-latn", + "region": 3 + }, { + "name": "Yoruba", + "id": "yo", + "region": 2 + }, { + "name": "Yoruba (Braille)", + "id": "yo-brai", + "region": 2 + }, { + "name": "Yoruba (Latin)", + "id": "yo-latn", + "region": 2 + }, { + "name": "Yukuben", + "id": "ybl", + "region": 2 + }, { + "name": "Yupik, Central", + "id": "esu", + "region": 6 + }, { + "name": "Yupik, Pacific Gulf", + "id": "ems", + "region": 6 + }, { + "name": "Yupik, Saint Lawrence Island", + "id": "ess", + "region": 6 + }, { + "name": "Yupik, Saint Lawrence Island (Cyrillic)", + "id": "ess-cyrl", + "region": 6 + }, { + "name": "Zaghawa", + "id": "zag", + "region": 2 + }, { + "name": "Zaiwa (Lisu)", + "id": "atb-lisu", + "region": 3 + }, { + "name": "Zan Gula", + "id": "zna", + "region": 2 + }, { + "name": "Zanaki (Latin)", + "id": "zak-latn", + "region": 2 + }, { + "name": "Zande (Latin)", + "id": "zne-latn", + "region": 2 + }, { + "name": "Zangskari (Tibetan)", + "id": "zau-tibt", + "region": 3 + }, { + "name": "Zapotec", + "id": "zap", + "region": 1 + }, { + "name": "Zapotec, Alo\u00e1pam", + "id": "zaq", + "region": 6 + }, { + "name": "Zapotec, Amatl\u00e1n", + "id": "zpo", + "region": 6 + }, { + "name": "Zapotec, Asunci\u00f3n Mixtepec", + "id": "zoo", + "region": 6 + }, { + "name": "Zapotec, Ayoquesco", + "id": "zaf", + "region": 6 + }, { + "name": "Zapotec, Cajonos", + "id": "zad", + "region": 6 + }, { + "name": "Zapotec, Chichicapan", + "id": "zpv", + "region": 6 + }, { + "name": "Zapotec, Choapan", + "id": "zpc", + "region": 6 + }, { + "name": "Zapotec, Coatecas Altas", + "id": "zca", + "region": 6 + }, { + "name": "Zapotec, Coatl\u00e1n", + "id": "zps", + "region": 6 + }, { + "name": "Zapotec, El Alto", + "id": "zpp", + "region": 6 + }, { + "name": "Zapotec, Elotepec", + "id": "zte", + "region": 6 + }, { + "name": "Zapotec, Guevea de Humboldt", + "id": "zpg", + "region": 6 + }, { + "name": "Zapotec, G\u00fcil\u00e1", + "id": "ztu", + "region": 6 + }, { + "name": "Zapotec, Lachiguiri", + "id": "zpa", + "region": 6 + }, { + "name": "Zapotec, Lachix\u00edo", + "id": "zpl", + "region": 6 + }, { + "name": "Zapotec, Lapagu\u00eda-Guivini", + "id": "ztl", + "region": 6 + }, { + "name": "Zapotec, Loxicha", + "id": "ztp", + "region": 6 + }, { + "name": "Zapotec, Mazaltepec", + "id": "zpy", + "region": 6 + }, { + "name": "Zapotec, Miahuatl\u00e1n", + "id": "zam", + "region": 6 + }, { + "name": "Zapotec, Mitla", + "id": "zaw", + "region": 6 + }, { + "name": "Zapotec, Mixtepec", + "id": "zpm", + "region": 6 + }, { + "name": "Zapotec, Ocotl\u00e1n", + "id": "zac", + "region": 6 + }, { + "name": "Zapotec, Ozolotepec", + "id": "zao", + "region": 6 + }, { + "name": "Zapotec, Petapa", + "id": "zpe", + "region": 6 + }, { + "name": "Zapotec, Quiavicuzas", + "id": "zpj", + "region": 6 + }, { + "name": "Zapotec, Quioquitani-Quier\u00ed", + "id": "ztq", + "region": 6 + }, { + "name": "Zapotec, Rinc\u00f3n", + "id": "zar", + "region": 6 + }, { + "name": "Zapotec, San Agust\u00edn Mixtepec", + "id": "ztm", + "region": 6 + }, { + "name": "Zapotec, San Baltazar Loxicha", + "id": "zpx", + "region": 6 + }, { + "name": "Zapotec, San Pedro Quiatoni", + "id": "zpf", + "region": 6 + }, { + "name": "Zapotec, San Vicente Coatl\u00e1n", + "id": "zpt", + "region": 6 + }, { + "name": "Zapotec, Santa Catarina Albarradas", + "id": "ztn", + "region": 6 + }, { + "name": "Zapotec, Santa In\u00e9s Yatzechi", + "id": "zpn", + "region": 6 + }, { + "name": "Zapotec, Santa Mar\u00eda Quiegolani", + "id": "zpi", + "region": 6 + }, { + "name": "Zapotec, Santiago Xanica", + "id": "zpr", + "region": 6 + }, { + "name": "Zapotec, Santo Domingo Albarradas", + "id": "zas", + "region": 6 + }, { + "name": "Zapotec, Sierra de Ju\u00e1rez", + "id": "zaa", + "region": 6 + }, { + "name": "Zapotec, Southeastern Ixtl\u00e1n", + "id": "zpd", + "region": 6 + }, { + "name": "Zapotec, Southern Rincon", + "id": "zsr", + "region": 6 + }, { + "name": "Zapotec, Tabaa", + "id": "zat", + "region": 6 + }, { + "name": "Zapotec, Tejalapan", + "id": "ztt", + "region": 6 + }, { + "name": "Zapotec, Texmelucan", + "id": "zpz", + "region": 6 + }, { + "name": "Zapotec, Tilquiapan", + "id": "zts", + "region": 6 + }, { + "name": "Zapotec, Tlacolulita", + "id": "zpk", + "region": 6 + }, { + "name": "Zapotec, Totomachapan", + "id": "zph", + "region": 6 + }, { + "name": "Zapotec, Western Tlacolula Valley", + "id": "zab", + "region": 6 + }, { + "name": "Zapotec, Xadani", + "id": "zax", + "region": 6 + }, { + "name": "Zapotec, Xanagu\u00eda", + "id": "ztg", + "region": 6 + }, { + "name": "Zapotec, Yal\u00e1lag", + "id": "zpu", + "region": 6 + }, { + "name": "Zapotec, Yareni", + "id": "zae", + "region": 6 + }, { + "name": "Zapotec, Yatee", + "id": "zty", + "region": 6 + }, { + "name": "Zapotec, Yatzachi", + "id": "zav", + "region": 6 + }, { + "name": "Zapotec, Yautepec", + "id": "zpb", + "region": 6 + }, { + "name": "Zapotec, Zaachila", + "id": "ztx", + "region": 6 + }, { + "name": "Zapotec, Zaniza", + "id": "zpw", + "region": 6 + }, { + "name": "Zapotec, Zoogocho", + "id": "zpq", + "region": 6 + }, { + "name": "Zaramo (Latin)", + "id": "zaj-latn", + "region": 2 + }, { + "name": "Zarma (Braille)", + "id": "dje-brai", + "region": 2 + }, { + "name": "Zarma (Latin)", + "id": "dje-latn", + "region": 2 + }, { + "name": "Zay", + "id": "zwa", + "region": 2 + }, { + "name": "Zaysete (Ethiopic)", + "id": "zay-ethi", + "region": 2 + }, { + "name": "Zaysete (Latin)", + "id": "zay-latn", + "region": 2 + }, { + "name": "Zeeuws", + "id": "zea", + "region": 4 + }, { + "name": "Zenaga (Tifinagh)", + "id": "zen-tfng", + "region": 2 + }, { + "name": "Zerenkel", + "id": "zrn", + "region": 2 + }, { + "name": "Zhoa", + "id": "zhw", + "region": 2 + }, { + "name": "Zhuang", + "id": "za", + "region": 1 + }, { + "name": "Zhuang, Dai", + "id": "zhd", + "region": 3 + }, { + "name": "Zhuang, Liujiang", + "id": "zlj", + "region": 3 + }, { + "name": "Zhuang, Nong", + "id": "zhn", + "region": 3 + }, { + "name": "Zhuang, Qiubei (Latin)", + "id": "zqe-latn", + "region": 3 + }, { + "name": "Zhuang, Youjiang", + "id": "zyj", + "region": 3 + }, { + "name": "Zigula (Latin)", + "id": "ziw-latn", + "region": 2 + }, { + "name": "Zimba (Latin)", + "id": "zmb-latn", + "region": 2 + }, { + "name": "Zinza (Latin)", + "id": "zin-latn", + "region": 2 + }, { + "name": "Zizilivakan", + "id": "ziz", + "region": 2 + }, { + "name": "Zulgo-Gemzek", + "id": "gnd", + "region": 2 + }, { + "name": "Zulu", + "id": "zu", + "region": 2 + }, { + "name": "Zulu (Braille)", + "id": "zu-brai", + "region": 2 + }, { + "name": "Zumaya", + "id": "zuy", + "region": 2 + }], + "timerid": "36" +}); diff --git a/web/src/test/auto/resources/query-mock-results/readme.md b/web/src/test/auto/resources/query-mock-results/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..77495eb2032710c3a0adf785919d1a8e27ff69a5 --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/readme.md @@ -0,0 +1,4 @@ +Note that the `timerid` field is omitted from these objects. + +It's difficult to mock within these files due to aspects of the current design - the requester-objects +autogenerate a value dynamically, and in production, said value is reflected in that field. \ No newline at end of file diff --git a/web/src/test/auto/resources/query-mock-results/sil_cameroon_azerty.js.fixture b/web/src/test/auto/resources/query-mock-results/sil_cameroon_azerty.js.fixture new file mode 100644 index 0000000000000000000000000000000000000000..39b61253275932bc333d83c6aaf4a7bd2850a78e --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/sil_cameroon_azerty.js.fixture @@ -0,0 +1,2248 @@ +keyman.register({ + "options": { + "context": "keyboard", + "dateFormat": "standard", + "device": "any", + "keyboardBaseUri": "https://s.keyman.com/keyboard/", + "fontBaseUri": "https://s.keyman.com/font/deploy/", + "keyboardid": "sil_cameroon_azerty", + "keyboardVersion": "current" + }, + "keyboard": [{ + "id": "sil_cameroon_azerty", + "name": "Cameroon AZERTY", + "filename": "sil_cameroon_azerty/6.0.8/sil_cameroon_azerty-6.0.8.js", + "version": "6.0.8", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/sil/sil_cameroon_azerty", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "aal", + "name": "Afade", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "agq", + "name": "Aghem", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "muc", + "name": "Ajumbu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bss", + "name": "Akoose", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "aku", + "name": "Akum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ael", + "name": "Ambele", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ato", + "name": "Atong", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "azo", + "name": "Awing", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbk", + "name": "Babanki", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bfj", + "name": "Bafanji", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bwt", + "name": "Bafaw-Balong", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ksf", + "name": "Bafia", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bfd", + "name": "Bafut", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bkc", + "name": "Baka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bqz", + "name": "Bakaka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bkh", + "name": "Bakoko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kme", + "name": "Bakole", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bdn", + "name": "Baldemu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bqo", + "name": "Balo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbq", + "name": "Bamali", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bmo", + "name": "Bambalang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "baw", + "name": "Bambili-Bambui", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bce", + "name": "Bamenyam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bqt", + "name": "Bamukumbit", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bax", + "name": "Bamun", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bvm", + "name": "Bamunka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bcw", + "name": "Bana", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bgf", + "name": "Bangandu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bgj", + "name": "Bangolan", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "abb", + "name": "Bankon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbi", + "name": "Barombi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bas", + "name": "Basaa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bsi", + "name": "Bassossi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bta", + "name": "Bata", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bnm", + "name": "Batanga", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "btc", + "name": "Bati", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bfp", + "name": "Beba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "beb", + "name": "Bebele", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bxp", + "name": "Bebil", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bnz", + "name": "Beezen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bby", + "name": "Befang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bkw", + "name": "Bekwel", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "byb", + "name": "Bikya", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bwh", + "name": "Bishuo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "brt", + "name": "Bitare", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bky", + "name": "Bokyi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bmw", + "name": "Bomwali", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bvg", + "name": "Bonkeng", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbx", + "name": "Bubia", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bdm", + "name": "Buduma", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bum", + "name": "Bulu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bmv", + "name": "Bum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bqd", + "name": "Bung", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bxs", + "name": "Busam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bju", + "name": "Busuu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bhs", + "name": "Buwal", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mkk", + "name": "Byep", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ckx", + "name": "Caka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "cug", + "name": "cug", + "region": 1, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "cuv", + "name": "Cuvok", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dbq", + "name": "Daba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dmm", + "name": "Dama", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dek", + "name": "Dek", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "anv", + "name": "Denya", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dur", + "name": "Dii", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dii", + "name": "Dimbong", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dow", + "name": "Doyayo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dua", + "name": "Duala", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ndu", + "name": "Dugun", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dme", + "name": "Dugwor", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "duz", + "name": "Duli-Gey", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dae", + "name": "Duupa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "etu", + "name": "Ejagham", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ekm", + "name": "Elip", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "emn", + "name": "Eman", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ags", + "name": "Esimbi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "eto", + "name": "Eton", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bzz", + "name": "Evant", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ewo", + "name": "Ewondo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fll", + "name": "Fali, North", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fal", + "name": "Fali, South", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fak", + "name": "Fang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fan", + "name": "Fang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fmp", + "name": "Fe\u2019fe\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fub-latn", + "name": "Fulfulde, Adamawa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "fuv", + "name": "Fulfulde, Nigerian", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gou", + "name": "Gavar", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gba", + "name": "Gbaya (Central African Republic)", + "region": 1, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gmm", + "name": "Gbaya-Mbodomo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbj", + "name": "Ghom\u00e1l\u00e1\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gid", + "name": "Gidar", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kmp", + "name": "Gimme", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gmn", + "name": "Gimnime", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "giz", + "name": "Giziga", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gis", + "name": "Giziga, North", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "glw", + "name": "Glavda", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gde", + "name": "Gude", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ngs", + "name": "Gvoko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gyi", + "name": "Gyele", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xed", + "name": "Hdi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "hij", + "name": "Hijuk", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "hya", + "name": "Hya", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bec", + "name": "Iceve-Maci", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ass", + "name": "Ipulo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "isu", + "name": "Isu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "szv", + "name": "Isu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "uiv", + "name": "Iyive", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "jim", + "name": "Jimjimen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "jia", + "name": "Jina", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "jbu", + "name": "Jukun Takum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kkj", + "name": "Kako", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kzr", + "name": "Karang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kbn", + "name": "Kare", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "luw", + "name": "Kasabe", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dmo", + "name": "Kemedzung", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kvm", + "name": "Kendem", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ndb", + "name": "Kenswei Nsei", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ken", + "name": "Kenyang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ker", + "name": "Kera", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "biw", + "name": "Kol", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "klc", + "name": "Kolbila", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bkm", + "name": "Kom", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kmy", + "name": "Koma", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ozm", + "name": "Koonzime", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "krp", + "name": "Korop", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kid", + "name": "Koshin", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kfn", + "name": "Kuk", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kfl", + "name": "Kung", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xuo", + "name": "Kuo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kub", + "name": "Kutep", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bko", + "name": "Kwa\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kdz", + "name": "Kwaja", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kwu", + "name": "Kwakum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "knp", + "name": "Kwanja", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nmg", + "name": "Kwasio", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lbi", + "name": "La\u2019bi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kot", + "name": "Lagwan", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lmx", + "name": "Laimbue", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lns", + "name": "Lamnso\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lfa", + "name": "Lefa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "leo", + "name": "Leti", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "add", + "name": "Lidzonka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lmp", + "name": "Limbum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "wok", + "name": "Longto", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mxu", + "name": "Mada", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "maf", + "name": "Mafa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xmj", + "name": "Majera", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mcp", + "name": "Makaa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mxf", + "name": "Malgbe", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mzd", + "name": "Malimba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mcs", + "name": "Mambai", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mcu", + "name": "Mambila, Cameroon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "myg", + "name": "Manta", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mcn", + "name": "Masana", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "msv", + "name": "Maslam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mfh", + "name": "Matal", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "dkx", + "name": "Mazagway-Hidi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nza", + "name": "Mbembe, Tigon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mbo", + "name": "Mbo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mtk", + "name": "Mbo\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xmb", + "name": "Mbonga", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bgu", + "name": "Mbongno", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xmd", + "name": "Mbudum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mqb", + "name": "Mbuko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mlb", + "name": "Mbule", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mdd", + "name": "Mbum", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "byv", + "name": "Medumba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mfj", + "name": "Mefele", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mfd", + "name": "Mendankwe-Nkwen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "xmg", + "name": "Mengaka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mct", + "name": "Mengisa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mea", + "name": "Menka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "meq", + "name": "Merey", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "iyo", + "name": "Mesaka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mgo", + "name": "Meta\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nfu", + "name": "Mfumte", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "hna", + "name": "Mina", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mmu", + "name": "Mmaala", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bfm", + "name": "Mmen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mfk", + "name": "Mofu, North", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mif", + "name": "Mofu-Gudur", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bri", + "name": "Mokpwe", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mlw", + "name": "Moloko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mru", + "name": "Mono", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mpi", + "name": "Mpade", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mcx", + "name": "Mpiemo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mgg", + "name": "Mpumpong", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kqx", + "name": "Mser", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "muo", + "name": "Mubako", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "boe", + "name": "Mundabli", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mua", + "name": "Mundang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mnf", + "name": "Mundani", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mhk", + "name": "Mungaka", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mij", + "name": "Mungbam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mse", + "name": "Musey", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mug", + "name": "Musgu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "muy", + "name": "Muyang", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bzv", + "name": "Naami", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ngv", + "name": "Nagumi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mff", + "name": "Naki", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ncr", + "name": "Nchane", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nnz", + "name": "Nda\u2019nda\u2019", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gke", + "name": "Ndai", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nml", + "name": "Ndemli", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ndr", + "name": "Ndoola", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nbv", + "name": "Ngamambo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "sba", + "name": "Ngambay", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nge", + "name": "Ngemba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ngj", + "name": "Ngie", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nnh", + "name": "Ngiemboon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "jgo", + "name": "Ngomba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nla", + "name": "Ngombale", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nsh", + "name": "Ngoshie", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nwe", + "name": "Ngwe", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ngn", + "name": "Ngwo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nmr", + "name": "Nimbari", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "sgi", + "name": "Nizaa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "njj", + "name": "Njen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "njy", + "name": "Njyem", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nkc", + "name": "Nkongho", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lem", + "name": "Nomaande", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nhu", + "name": "Noone", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "baf", + "name": "Nubaca", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yas", + "name": "Nugunu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nvo", + "name": "Nyokon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nzy", + "name": "Nzakambay", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "nja", + "name": "Nzanyi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "obl", + "name": "Oblo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "oku", + "name": "Oku", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bdu", + "name": "Oroko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ost", + "name": "Osatu", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pmn", + "name": "Pam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pnz", + "name": "Pana", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pbi", + "name": "Parkwa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pfe", + "name": "Pere", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "lme", + "name": "P\u00e9v\u00e9", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "wes", + "name": "Pidgin, Cameroon", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pny", + "name": "Pinyin", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "pmm", + "name": "Pol", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "kvj", + "name": "Psikye", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ndi", + "name": "Samba Leko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "asj", + "name": "Sari", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "swq", + "name": "Sharwa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bbw", + "name": "Supapya", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "sox", + "name": "Swo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ngy", + "name": "Tibea", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "tik", + "name": "Tikar", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "tiv", + "name": "Tiv", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "toz", + "name": "To", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "tsh", + "name": "Tsuvan", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bag", + "name": "Tuki", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "tvu", + "name": "Tunen", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ttf", + "name": "Tuotomb", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "tui", + "name": "Tupuri", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "twn", + "name": "Twendi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "usk", + "name": "Usaghade", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mlr", + "name": "Vame", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "vem", + "name": "Vemgo-Mabas", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bav", + "name": "Vengo", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ver", + "name": "Verre", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "vut", + "name": "Vute", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "mfi", + "name": "Wandala", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "www", + "name": "Wawa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "weh", + "name": "Weh", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bqm", + "name": "Wumboko", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "bse", + "name": "Wushi", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "udl", + "name": "Wuzlam", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yam", + "name": "Yamba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yat", + "name": "Yambeta", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yav", + "name": "Yangben", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yko", + "name": "Yasa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ybb", + "name": "Yemba", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "yei", + "name": "Yeni", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ybl", + "name": "Yukuben", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "zhw", + "name": "Zhoa", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "ziz", + "name": "Zizilivakan", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "gnd", + "name": "Zulgo-Gemzek", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }, { + "id": "zuy", + "name": "Zumaya", + "region": 2, + "font": { + "family": "AndikaAfr", + "source": ["AndikaAfr-R.ttf"] + } + }] + }] +}); diff --git a/web/src/test/auto/resources/query-mock-results/sil_euro_latin@no_sv.js.fixture b/web/src/test/auto/resources/query-mock-results/sil_euro_latin@no_sv.js.fixture new file mode 100644 index 0000000000000000000000000000000000000000..307196a6518812041e55213992668abc981dcc60 --- /dev/null +++ b/web/src/test/auto/resources/query-mock-results/sil_euro_latin@no_sv.js.fixture @@ -0,0 +1,52 @@ +keyman.register({ + "options": { + "context": "keyboard", + "dateFormat": "standard", + "device": "any", + "keyboardBaseUri": "https://s.keyman.com/keyboard/", + "fontBaseUri": "https://s.keyman.com/font/deploy/", + "keyboardid": "sil_euro_latin@no,sil_euro_latin@sv", + "keyboardVersion": "current" + }, + "keyboard": [{ + "id": "sil_euro_latin", + "name": "EuroLatin (SIL)", + "filename": "sil_euro_latin/2.0.3/sil_euro_latin-2.0.3.js", + "version": "2.0.3", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/sil/sil_euro_latin", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "no", + "name": "Norwegian", + "region": 4, + "font": { + "family": "LatinWeb", + "source": ["DejaVuSans.ttf", "DejaVuSans.mobileconfig"] + } + }] + }, { + "id": "sil_euro_latin", + "name": "EuroLatin (SIL)", + "filename": "sil_euro_latin/2.0.3/sil_euro_latin-2.0.3.js", + "version": "2.0.3", + "source": "https://github.com/keymanapp/keyboards/tree/master/release/sil/sil_euro_latin", + "devices": { + "phone": 2, + "tablet": 2, + "desktop": 2 + }, + "languages": [{ + "id": "sv", + "name": "Swedish", + "region": 4, + "font": { + "family": "LatinWeb", + "source": ["DejaVuSans.ttf", "DejaVuSans.mobileconfig"] + } + }] + }] +}); \ No newline at end of file diff --git a/web/src/test/manual/embed/android-harness/build.sh b/web/src/test/manual/embed/android-harness/build.sh index 0445022e5c9103edad0d21fc633a53346559485b..f8e7975d592ff5ee7b4e683be34624d30dc489fb 100755 --- a/web/src/test/manual/embed/android-harness/build.sh +++ b/web/src/test/manual/embed/android-harness/build.sh @@ -8,21 +8,52 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../../../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE -THIS_DIR="$(dirname "$THIS_SCRIPT")" +cd "$THIS_SCRIPT_PATH" -mkdir -p "$THIS_DIR/host" -cp -R "$KEYMAN_ROOT/android/KMEA/app/src/main/assets/"* "$THIS_DIR/host/" -cp "$KEYMAN_ROOT/web/build/app/embed/debug/keyman.js" "$THIS_DIR/host/keymanandroid.js" -cp "$KEYMAN_ROOT/web/build/app/embed/debug/keyman.js.map" "$THIS_DIR/host/keyman.js.map" -cp "$KEYMAN_ROOT/web/build/app/embed/debug/osk/kmwosk.css" "$THIS_DIR/host/kmwosk.css" -cp "$KEYMAN_ROOT/web/build/app/embed/debug/osk/globe-hint.css" "$THIS_DIR/host/globe-hint.css" -cp "$KEYMAN_ROOT/web/build/app/embed/debug/osk/keymanweb-osk.ttf" "$THIS_DIR/host/keymanweb-osk.ttf" +builder_describe "Builds a debug-host page simulating Keyman Android's WebView setup for KMW use" \ + "@/common/web/keyman-version" \ + "@/web/src/app/webview" \ + "@/common/web/sentry-manager" \ + clean configure build -# We shouldn't rely on a prior Android build for these files! We can retrieve them separately. -cp "$KEYMAN_ROOT/common/web/sentry-manager/build/index.js" "$THIS_DIR/host/keyman-sentry.js" -cp "$KEYMAN_ROOT/node_modules/es6-shim/es6-shim.min.js" "$THIS_DIR/host/es6-shim.min.js" +builder_describe_outputs \ + configure /node_modules \ + build /web/src/test/manual/embed/android-harness/host/keyboard.html -# Test keyboards +builder_parse "$@" -cp "$KEYMAN_ROOT/web/src/test/manual/web/web_context_tests.js" "$THIS_DIR/host/" -# android\KMEA\app\src\main\assets\keyboard.html \ No newline at end of file +### CLEAN ACTIONS + +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +if builder_start_action build; then + mkdir -p "host/osk" + cp -R "$KEYMAN_ROOT/android/KMEA/app/src/main/assets/"* "host/" + cp "$KEYMAN_ROOT/web/build/app/webview/debug/keymanweb-webview.js" "host/keymanweb-webview.js" + cp "$KEYMAN_ROOT/web/build/app/webview/debug/keymanweb-webview.js.map" "host/keymanweb-webview.js.map" + cp "$KEYMAN_ROOT/web/build/app/resources/osk/ajax-loader.gif" "host/ajax-loader.gif" + cp "$KEYMAN_ROOT/web/build/app/resources/osk/kmwosk.css" "host/kmwosk.css" + cp "$KEYMAN_ROOT/web/build/app/resources/osk/globe-hint.css" "host/globe-hint.css" + cp "$KEYMAN_ROOT/web/build/app/resources/osk/keymanweb-osk.ttf" "host/keymanweb-osk.ttf" + + # We shouldn't rely on a prior Android build for these files! We can retrieve them separately. + cp "$KEYMAN_ROOT/common/web/sentry-manager/build/lib/index.js" "host/keyman-sentry.js" + cp "$KEYMAN_ROOT/node_modules/es6-shim/es6-shim.min.js" "host/es6-shim.min.js" + cp "$KEYMAN_ROOT/node_modules/@sentry/browser/build/bundle.min.js" "host/sentry.min.js" + + # Test keyboards + + cp "$KEYMAN_ROOT/web/src/test/manual/web/web_context_tests.js" "host/" + # android\KMEA\app\src\main\assets\keyboard.html + builder_finish_action success build +fi \ No newline at end of file diff --git a/web/src/test/manual/web/attachment-api/index.html b/web/src/test/manual/web/attachment-api/index.html index 0ee6896af425d0d964f1d1f7ec2c866695dc89a9..f904d25f08e4b631f55473c5f38f979bd3a06a26 100644 --- a/web/src/test/manual/web/attachment-api/index.html +++ b/web/src/test/manual/web/attachment-api/index.html @@ -23,7 +23,7 @@ - + - + + - + diff --git a/web/src/test/manual/web/build-visual-keyboard/index.html b/web/src/test/manual/web/build-visual-keyboard/index.html index a572a638a7373e3a84814ff8d5248e783b6b4d54..2c3e1d6c0aa02477f6bac5e6f27a947e1bc3b0be 100644 --- a/web/src/test/manual/web/build-visual-keyboard/index.html +++ b/web/src/test/manual/web/build-visual-keyboard/index.html @@ -24,7 +24,7 @@ - + - + diff --git a/web/src/test/manual/web/caps-lock-layer-3620/index.html b/web/src/test/manual/web/caps-lock-layer-3620/index.html index 0c8981a72581b5f097056bb1b5e02b59e715aabe..c5b328189f62aef858a86ec3c6122bd8b585d881 100644 --- a/web/src/test/manual/web/caps-lock-layer-3620/index.html +++ b/web/src/test/manual/web/caps-lock-layer-3620/index.html @@ -17,7 +17,7 @@ - + - + + - + + - + + - + + - + diff --git a/web/src/test/manual/web/desktop-ui/float.html b/web/src/test/manual/web/desktop-ui/float.html index 049fa9d316b0d377e3fa2bee87e7c840028aa707..338a66a160c8d07db1d31eed2bbd4a3e961703fe 100644 --- a/web/src/test/manual/web/desktop-ui/float.html +++ b/web/src/test/manual/web/desktop-ui/float.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/desktop-ui/toggle.html b/web/src/test/manual/web/desktop-ui/toggle.html index 6aa99b1c5abb6f9934bc8f4223c7b8dc75a4de4f..857e98466c337f0248cbb8e2367ab547e718e303 100644 --- a/web/src/test/manual/web/desktop-ui/toggle.html +++ b/web/src/test/manual/web/desktop-ui/toggle.html @@ -23,7 +23,7 @@ - + - + @@ -41,7 +41,7 @@ + - + diff --git a/web/src/test/manual/web/empty-row/index.html b/web/src/test/manual/web/empty-row/index.html index 460d544b64528bbc0668c58ae36296e89718da21..02953493e91a9231c2e299c1c0c802c16008090f 100644 --- a/web/src/test/manual/web/empty-row/index.html +++ b/web/src/test/manual/web/empty-row/index.html @@ -23,7 +23,7 @@ - + - + + - + + #KeymanWebControl {width:50%;min-width:600px;} + + + + - - - + + - + keyman.init({ + attachType:'auto' + }); + + - - + + - - + +

KeymanWeb: 'Test case' for proper implementation of the removeKeyboards API function

- -
+

This version is used to test the removeKeyboards API function for proper argument handling.

Automatically-loaded keyboard IDs:

    @@ -71,34 +70,34 @@


    -
+

Add a keyboard by keyboard ID:

- - + +

Remove a keyboard by keyboard ID:

- +

Return to testing home page

- - + + - - + * + *** + --> diff --git a/web/src/test/manual/web/issue103/index.html b/web/src/test/manual/web/issue103/index.html index c02d9312ed49c0d9f672fc5b25cea6f01ccc9487..35b006e34721cc3e4c0ac5d1273ed0b954df909e 100644 --- a/web/src/test/manual/web/issue103/index.html +++ b/web/src/test/manual/web/issue103/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue115/index.html b/web/src/test/manual/web/issue115/index.html index 5c9ffeb3afb0b7630e3565f2bcd2393634a8adcb..cd9afcb8fa00258eb353669fc173e32abaddb090 100644 --- a/web/src/test/manual/web/issue115/index.html +++ b/web/src/test/manual/web/issue115/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue116/index.html b/web/src/test/manual/web/issue116/index.html index b8b49e1375e17758af2fe77edf29639338f62a87..d81fb947c04ba2c81fa34efbfba19fcf6018b1e7 100644 --- a/web/src/test/manual/web/issue116/index.html +++ b/web/src/test/manual/web/issue116/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue1332/index.html b/web/src/test/manual/web/issue1332/index.html index 4b8715a7f170494626f20a98925cdd1b8a5f726e..8bd8c403aa4af93ff275f2d4f850e0add033fef6 100644 --- a/web/src/test/manual/web/issue1332/index.html +++ b/web/src/test/manual/web/issue1332/index.html @@ -23,7 +23,7 @@ - + - + + - + diff --git a/web/src/test/manual/web/issue266/index.html b/web/src/test/manual/web/issue266/index.html index aca186b178488f480c0c6277c506e6f967449ce9..56e52016d765ca9c752ca85638692ffa2e6e7918 100644 --- a/web/src/test/manual/web/issue266/index.html +++ b/web/src/test/manual/web/issue266/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue271/index.html b/web/src/test/manual/web/issue271/index.html index 8c055c4015f06d62862f21a7040e93ed8754d835..fa969195a138f80723073ff1ec7ea7297fc1bae1 100644 --- a/web/src/test/manual/web/issue271/index.html +++ b/web/src/test/manual/web/issue271/index.html @@ -23,7 +23,7 @@ - + - + @@ -57,10 +57,10 @@
  • us, en
  • french, fr
  • -
  • european2, no
  • -
  • european2, sv
  • +
  • sil_euro_latin, no
  • +
  • sil_euro_latin, sv
  • hebrew, he
  • -
  • dzongkha, dzo
  • +
  • dzongkha, dz
  • lao_2008_basic, lo

diff --git a/web/src/test/manual/web/issue29/index.html b/web/src/test/manual/web/issue29/index.html index 92fed6a8c1ca8a658e2e5cdb2cc5b4d8c5dbf555..24a226326ca54147df38c301f00aa732754c62a4 100644 --- a/web/src/test/manual/web/issue29/index.html +++ b/web/src/test/manual/web/issue29/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue2924/header.js b/web/src/test/manual/web/issue2924/header.js index 8e3ecb745ab7c4a9ba8a6c85d200c03b6a6f89b1..dcd8083c7dc52e8a10883c0bf56702ef3a8e2094 100644 --- a/web/src/test/manual/web/issue2924/header.js +++ b/web/src/test/manual/web/issue2924/header.js @@ -35,7 +35,7 @@ window.addEventListener('load', function() { ? window.location.href.substr(0, window.location.href.lastIndexOf('/')+1) : window.location.href; - keyman.modelManager.register({ + keyman.addModel({ id: 'nrc.en.mtnt', languages: ['en'], path: pageRef + 'dmg.dv.test.model.js' diff --git a/web/src/test/manual/web/issue2924/index.html b/web/src/test/manual/web/issue2924/index.html index 61a1be97cde25f9efe1b20a3c5043ca536f0b96f..12396420c24bf2a13f6feecc31d409e7e8fb245b 100644 --- a/web/src/test/manual/web/issue2924/index.html +++ b/web/src/test/manual/web/issue2924/index.html @@ -6,8 +6,8 @@ KeymanWeb Issue 2924 - Variable Stores and Predictive Text - - + + diff --git a/web/src/test/manual/web/issue3701/index.html b/web/src/test/manual/web/issue3701/index.html index ca2729133abb2df3d439af5c8dd54eedeccbc6b4..4b621678cc699a8ee002a4364c308434f4aca961 100644 --- a/web/src/test/manual/web/issue3701/index.html +++ b/web/src/test/manual/web/issue3701/index.html @@ -23,7 +23,7 @@ - + - + + - + + - + diff --git a/web/src/test/manual/web/issue5312-touch-alias-optimization/index.html b/web/src/test/manual/web/issue5312-touch-alias-optimization/index.html deleted file mode 100644 index 682e279ab6b37293354289e1e6f62c55cf9667a5..0000000000000000000000000000000000000000 --- a/web/src/test/manual/web/issue5312-touch-alias-optimization/index.html +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - - - - - - - - - - KeymanWeb Sample Page - Unminified Source - - - - - - - - - - - - - - - - - - - - -

KeymanWeb Sample Page - Touch-Alias Element Optimization

- -
- -
- -

Text performance with this element:

- -

Text was generated by the Lorem Ipsum Generator.

- -

Return to testing home page

-
- - -
-

--End of Document--

- - - - - diff --git a/web/src/test/manual/web/issue5455/index.html b/web/src/test/manual/web/issue5455/index.html index 0ba1f0f84a97b84fe623ac9b0de88ebb6e945c24..77ea7dbb216bb6c67864433bc4caf5d1399c8bcd 100644 --- a/web/src/test/manual/web/issue5455/index.html +++ b/web/src/test/manual/web/issue5455/index.html @@ -23,7 +23,7 @@ - + - + + - + + - + diff --git a/web/src/test/manual/web/issue63/index.html b/web/src/test/manual/web/issue63/index.html index 43842dc3e9cad571b93f2850f700e70b9a9a877a..55cee63c97995cd2e9f6a064b7e54f6fe7e3abfc 100644 --- a/web/src/test/manual/web/issue63/index.html +++ b/web/src/test/manual/web/issue63/index.html @@ -23,7 +23,7 @@ - + - + diff --git a/web/src/test/manual/web/issue917-context-and-notany/index.html b/web/src/test/manual/web/issue917-context-and-notany/index.html index 1db3b10c4dbf0b3204094b195b64a018c5fea3e0..251edb9142a6c04df491e22c081b0aca7867ed73 100644 --- a/web/src/test/manual/web/issue917-context-and-notany/index.html +++ b/web/src/test/manual/web/issue917-context-and-notany/index.html @@ -23,7 +23,7 @@ - + - + + - + + - + diff --git a/web/src/test/manual/web/keyboard-errors/timeout.js b/web/src/test/manual/web/keyboard-errors/timeout.js index 2af6bf907a3a953dcc3ccce76a13c64e23001c29..d4e21179dfcd6905d02b10d06bd4aeb9a7f708a5 100644 --- a/web/src/test/manual/web/keyboard-errors/timeout.js +++ b/web/src/test/manual/web/keyboard-errors/timeout.js @@ -1,10 +1,21 @@ (function() { var me = document.currentScript; console.log(me); - document.body.addEventListener('load', function(e) { + + var onload = me.onload; + if(onload) { + me.onload = null; + } + + document.body.addEventListener('load', function(e) { // Prevent the element's onload event from firing if(e.srcElement == me || e.target == me) { e.cancelBubble = true; } }, {capture: true}); + + window.setTimeout(function () { + // Restores the function after a slight delay. + me.onload = onload; + }, 1); })(); \ No newline at end of file diff --git a/web/src/test/manual/web/mnemonic/index.html b/web/src/test/manual/web/mnemonic/index.html index 65331615aa5d7eeea99e761b4d5b7f24ac070c51..e0b0f0de79be9402cfe76a55b3a2bf330dacb82e 100644 --- a/web/src/test/manual/web/mnemonic/index.html +++ b/web/src/test/manual/web/mnemonic/index.html @@ -20,7 +20,7 @@ - + - + diff --git a/web/src/test/manual/web/options-with-save/index.html b/web/src/test/manual/web/options-with-save/index.html index 0f91ea354a2f080a232168d55b1205ba29a88aa5..9f607fd11f47c3b7ccd90cc52186de36f85ce960 100644 --- a/web/src/test/manual/web/options-with-save/index.html +++ b/web/src/test/manual/web/options-with-save/index.html @@ -20,7 +20,7 @@ - + - + diff --git a/web/src/test/manual/web/osk-event-buttons/index.html b/web/src/test/manual/web/osk-event-buttons/index.html index 3c88e222017a7797530e2fd385aaa0aa66836519..92dafcdb4a9ec84152176d7e25f168a3e0f57f87 100644 --- a/web/src/test/manual/web/osk-event-buttons/index.html +++ b/web/src/test/manual/web/osk-event-buttons/index.html @@ -23,7 +23,7 @@ - + - + + - + + + + + + + + +

KeymanWeb OSK Module Testing

+

+ Note: if hosted via an IIS setup, you may need to add the 'mjs' extension under + "MIME Types" for it to properly serve related module-based JS files. +

+ +
+ +
+
+
+

Keyboard

+
+ + + +
+
+ + +
+

Layer

+
+ + +
+
+
+ +
+
+

OSK Mode

+
+ + + +
+
+
+

Target Device

+
+ + + + + +
+
+
+ +
+
+ +

Active Element

+
+ + + +
+
+
+ +

Visibility

+
+ + + +
+
+ Disabled (touch-based) +
+
+
+ +
+
+ +

Active Element

+
+ + + +
+
+
+ +

Visibility

+
+ + + +
+
+ Disabled (touch-based) +
+
+
+ +
+
+

OSK Sizing

+
Width:
+
Height:
+ +
+
+
+ +
+ +
+ +

Textbox target:

+ + +

Input target:

+ + +
+ +

Event log:

+ + + + +
+ +

Return to testing home page

+ + +
+

--End of Document--

+ + + + + \ No newline at end of file diff --git a/web/src/test/manual/web/osk/scratchspace/setup.mjs b/web/src/test/manual/web/osk/scratchspace/setup.mjs new file mode 100644 index 0000000000000000000000000000000000000000..ec3ae0dcac58e90b1aeae2bd8edeb22f7502bbbf --- /dev/null +++ b/web/src/test/manual/web/osk/scratchspace/setup.mjs @@ -0,0 +1,48 @@ +import * as KeymanOSK from '../../../../../../build/engine/osk/lib/index.mjs'; +import Device from '../../../../../../build/engine/device-detect/lib/index.mjs'; +import { loadKeyboardFromPath, loadKeyboardsFromStubs } from '../kbdLoader.mjs'; + +// NOTE: we must actually set these in this manner to have accessibility outside of +// this script tag! +window.modules = { + osk: KeymanOSK, + device: Device +}; + +let prefix = '../../'; +const stubs = [ + {id:'us',name:'English',languages:{id:'en',name:'English'}, filename:(prefix + 'us-1.0.js')}, + {id:'obolo_chwerty_6351',name:'obolo_chwerty_6351',languages:{id:'en',name:'English'}, + filename:(prefix + 'obolo_chwerty_6351.js')}, + {id:'lao_2008_basic',name:'Lao Basic', languages: {id:'lo',name:'Lao',region:'Asia'}, + filename:(prefix + 'lao_2008_basic-1.2.js')} +] + +window.keyboards = loadKeyboardsFromStubs(stubs).then((keyboards) => { + window.keyboards = keyboards; +}); + +let device = new Device(); +device.detect(); + +if(device.formFactor == 'phone') { + document.body.classList.add('phone'); +} + +if(device.touchable) { + document.body.classList.add('touch'); +} + +window.HOST_DEVICE = device.coreSpec; +window.targetDevice = new modules.osk.DeviceSpec('safari', 'phone', 'ios', true); + +window.BaseConfiguration = { + isEmbedded: false, + // Relative path to the resources' original versions. + pathConfig: { + resources: '../../../../../resources', + fonts: '' + } +}; + +window.keyboardLoader = loadKeyboardFromPath; \ No newline at end of file diff --git a/web/src/test/manual/web/platform/index.html b/web/src/test/manual/web/platform/index.html index c91e667e52c9c9f12188f00a59415ee7df15f4ad..c528862ead5d09aacbedcaa8709f956ebf58af7b 100644 --- a/web/src/test/manual/web/platform/index.html +++ b/web/src/test/manual/web/platform/index.html @@ -23,7 +23,7 @@ - + - + + - + + - + + - + @@ -51,6 +51,7 @@ addKeyboards('french','@he'); kmw.init({ + attachType: 'auto', useAlerts: alertType }).then(function() { loadKeyboards(); diff --git a/web/src/test/manual/web/rotation-events/index.html b/web/src/test/manual/web/rotation-events/index.html index 6e884f505a248541b8a546e15f759a1b71c0618e..c07b3300c9426d230712a366794ab4c4231752b1 100644 --- a/web/src/test/manual/web/rotation-events/index.html +++ b/web/src/test/manual/web/rotation-events/index.html @@ -14,7 +14,7 @@ - + - + diff --git a/web/src/test/manual/web/scrolling/index.html b/web/src/test/manual/web/scrolling/index.html deleted file mode 100644 index 138e5a43c115bbcb8a26913d0edc06d26d02f2ce..0000000000000000000000000000000000000000 --- a/web/src/test/manual/web/scrolling/index.html +++ /dev/null @@ -1,97 +0,0 @@ - - - - - - - - - - - - - - - KeymanWeb Testing Page - Predictive Text: robust testing - - - - - - - - - - - - - - - - -

KeymanWeb Sample Page - Scrolling Touch Elements

-
- -
- -

Type in your language in this text area:

- - -

Return to testing home page

-
- - - - - - - diff --git a/web/src/test/manual/web/sentry-integration/index.html b/web/src/test/manual/web/sentry-integration/index.html index 9dbf167b88bb85606aa9960f9498da6c37d77c40..51d4635c67bf959349467a2cc85e6bbd0ebd0d5c 100644 --- a/web/src/test/manual/web/sentry-integration/index.html +++ b/web/src/test/manual/web/sentry-integration/index.html @@ -23,7 +23,7 @@ - + - + - + diff --git a/web/src/test/manual/web/spacebar-text/index.html b/web/src/test/manual/web/spacebar-text/index.html index ee1053be0caf7affc0f0dea3e1559a94f537a7a3..2db281323bd6567c3411f6c5d5bbbd22eca7705d 100644 --- a/web/src/test/manual/web/spacebar-text/index.html +++ b/web/src/test/manual/web/spacebar-text/index.html @@ -17,7 +17,7 @@ - + - + + - + + - + + - + + - + + @@ -34,9 +34,9 @@ var kmw=window.keyman; kmw.init({ attachType:'auto' - }).then(function() { - kmw.addKeyboards(); - }); + }).then(function() { + kmw.addKeyboards(); + }); function runRenderer() { diff --git a/web/src/tools/testing/bulk_rendering/renderer_core.ts b/web/src/tools/testing/bulk_rendering/renderer_core.ts index adcbc45069f8eb81f4595edd97296eb068899e51..3348b72f16715d89c8dd481d61cbfc8fb656a938 100644 --- a/web/src/tools/testing/bulk_rendering/renderer_core.ts +++ b/web/src/tools/testing/bulk_rendering/renderer_core.ts @@ -1,296 +1,297 @@ // Includes KeymanWeb's Device class, as it's quite a useful resource for KMW-related projects. - -type KeyboardMap = {[id: string]: any}; - -namespace com.keyman.renderer { - export class BatchRenderer { - static divMaster: HTMLDivElement; - static dummy: HTMLInputElement; - static captureStream: any; - static video: any; - static boundingRect: DOMRect; - static allLayers = false; - static keyboardMatch: RegExp; - - async startCapture() { - let captureStream = null; - - try { - captureStream = await (navigator.mediaDevices as any).getDisplayMedia({ - audio: false, - video: { - width: screen.width, - height: screen.height, - frameRate: 60, - } - }); - } catch(err) { - console.error("Error: " + err); - } - - const video = document.createElement("video"); - video.srcObject = captureStream; - video.play(); - - const result = await new Promise((resolve, reject) => { - video.onloadedmetadata = function () { - resolve(); +import { Device } from 'keyman/engine/device-detect'; + +import type { KeymanEngine } from 'keyman/app/browser'; +import type { FloatingOSKView } from 'keyman/engine/osk'; + +declare var keyman: KeymanEngine; + +type KeyboardData = ReturnType; +type KeyboardMap = {[id: string]: KeyboardData}; + +export class BatchRenderer { + static divMaster: HTMLDivElement; + static dummy: HTMLInputElement; + static captureStream: any; + static video: any; + static boundingRect: DOMRect; + static allLayers = false; + static keyboardMatch: RegExp; + + async startCapture() { + let captureStream = null; + + try { + captureStream = await (navigator.mediaDevices as any).getDisplayMedia({ + audio: false, + video: { + width: screen.width, + height: screen.height, + frameRate: 60, } }); - - return {captureStream: captureStream, video: video}; + } catch(err) { + console.error("Error: " + err); } - // Filters the keyboard array to ensure only a single entry remains, rather than an entry per language. - private filterKeyboards(): KeyboardMap { - let keyman = window['keyman']; + const video = document.createElement("video"); + video.srcObject = captureStream; + video.play(); - let kbds = keyman['getKeyboards'](); + const result = await new Promise((resolve, reject) => { + video.onloadedmetadata = function () { + resolve(); + } + }); - let keyboardMap = []; + return {captureStream: captureStream, video: video}; + } - for(var i = 0; i < kbds.length; i++) { - let id: string = kbds[i]['InternalName']; - if(!id.match(BatchRenderer.keyboardMatch)) continue; - if(id.match(/^Keyboard_bod/)) continue; // bod keyboards currently don't load on Chrome and break the test run; they need rebuild. - if(keyboardMap[id]) { - continue; - } else { - keyboardMap[id] = kbds[i]; - } - } + // Filters the keyboard array to ensure only a single entry remains, rather than an entry per language. + private filterKeyboards(): KeyboardMap { + let kbds = keyman.getKeyboards(); - return keyboardMap; + let keyboardMap = {}; + + for(var i = 0; i < kbds.length; i++) { + let id: string = kbds[i].InternalName; + if(!id.match(BatchRenderer.keyboardMatch)) continue; + if(id.match(/^Keyboard_bod/)) continue; // bod keyboards currently don't load on Chrome and break the test run; they need rebuild. + if(keyboardMap[id]) { + continue; + } else { + keyboardMap[id] = kbds[i]; + } } - private render(ele: HTMLElement, isMobile?: boolean): Promise { - const capture = async () => { - const result = await new Promise((resolve, reject) => { - // from: https://github.com/kasprownik/electron-screencapture/blob/master/index.js - const canvas = document.createElement('canvas'); - canvas.width = BatchRenderer.boundingRect.width; - canvas.height = BatchRenderer.boundingRect.height; - const context = canvas.getContext('2d'); - // see: https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement - context.drawImage(BatchRenderer.video, - BatchRenderer.boundingRect.left, - BatchRenderer.boundingRect.top, - canvas.width, canvas.height, - 0, 0, canvas.width, canvas.height); - - const frame = canvas.toDataURL("image/png"); - let imgOut = document.createElement('img'); - imgOut.src = frame; - - resolve(imgOut); - }); + return keyboardMap; + } - return result; - } + private render(ele: HTMLElement, isMobile?: boolean): Promise { + const capture = async () => { + const result = await new Promise((resolve, reject) => { + // from: https://github.com/kasprownik/electron-screencapture/blob/master/index.js + const canvas = document.createElement('canvas'); + canvas.width = BatchRenderer.boundingRect.width; + canvas.height = BatchRenderer.boundingRect.height; + const context = canvas.getContext('2d'); + // see: https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement + context.drawImage(BatchRenderer.video, + BatchRenderer.boundingRect.left, + BatchRenderer.boundingRect.top, + canvas.width, canvas.height, + 0, 0, canvas.width, canvas.height); + + const frame = canvas.toDataURL("image/png"); + let imgOut = document.createElement('img'); + imgOut.src = frame; + + resolve(imgOut); + }); - return capture(); + return result; } - createKeyboardHeader(kbd, loaded: boolean): HTMLDivElement { - let divHeader = document.createElement('div'); - let eleName = document.createElement('h2'); + return capture(); + } - eleName.textContent = 'ID: ' + kbd['InternalName']; - divHeader.appendChild(eleName); + createKeyboardHeader(kbd: KeyboardData, loaded: boolean): HTMLDivElement { + let divHeader = document.createElement('div'); + let eleName = document.createElement('h2'); - let eleDescription = document.createElement('p'); + eleName.textContent = 'ID: ' + kbd.InternalName; + divHeader.appendChild(eleName); - if(loaded) { + let eleDescription = document.createElement('p'); - eleDescription.appendChild(document.createTextNode('Name: ' + kbd['Name'])); - eleDescription.appendChild(document.createElement('br')); - eleDescription.appendChild(document.createTextNode('Font: ' + window['keyman'].core.activeKeyboard._legacyLayoutSpec.F)); + if(loaded) { - } else { - eleDescription.appendChild(document.createTextNode('Unable to load this keyboard!')); - } + eleDescription.appendChild(document.createTextNode('Name: ' + kbd.Name)); + eleDescription.appendChild(document.createElement('br')); + eleDescription.appendChild(document.createTextNode('Font: ' + keyman.core.activeKeyboard['_legacyLayoutSpec'].F)); - divHeader.appendChild(eleDescription); - - return divHeader; + } else { + eleDescription.appendChild(document.createTextNode('Unable to load this keyboard!')); } - private processKeyboard(kbd) { - let keyman = window['keyman']; - let p: Promise = keyman.setActiveKeyboard(kbd['InternalName']); - let isMobile = keyman.util.device.formFactor != 'desktop'; + divHeader.appendChild(eleDescription); - // Establish common keyboard header info. - let divSummary = document.createElement('div'); - // Establishes a linkable target for this keyboard's data. - divSummary.id = "summary-" + kbd['InternalName']; + return divHeader; + } - BatchRenderer.divMaster.insertAdjacentElement('afterbegin', divSummary); + private processKeyboard(kbd: KeyboardData) { + let p: Promise = keyman.setActiveKeyboard(kbd.InternalName); + let isMobile = keyman.config.hostDevice.formFactor != 'desktop'; - // A nice, closure-friendly reference for use in our callbacks. - let renderer = this; + // Establish common keyboard header info. + let divSummary = document.createElement('div'); + // Establishes a linkable target for this keyboard's data. + divSummary.id = "summary-" + kbd.InternalName; + BatchRenderer.divMaster.insertAdjacentElement('afterbegin', divSummary); - // Once the keyboard's loaded, we can really get started. - return p.then(function() { - let box: HTMLDivElement = keyman.osk._Box; + // A nice, closure-friendly reference for use in our callbacks. + let renderer = this; - BatchRenderer.boundingRect = box.getBoundingClientRect(); + // Once the keyboard's loaded, we can really get started. + return p.then(function() { + let box: HTMLDivElement = keyman.osk._Box; - // Appromixate handling for non-fullscreen mode runs. - // Adjusts for the browser window's title bars, address bars, etc. - const screenOffsetY = window.outerHeight - window.innerHeight; - BatchRenderer.boundingRect.y += screenOffsetY; + BatchRenderer.boundingRect = box.getBoundingClientRect(); - divSummary.appendChild(renderer.createKeyboardHeader(kbd, true)); + // Appromixate handling for non-fullscreen mode runs. + // Adjusts for the browser window's title bars, address bars, etc. + const screenOffsetY = window.outerHeight - window.innerHeight; + BatchRenderer.boundingRect.y += screenOffsetY; - let divRenders = document.createElement('div'); - divSummary.appendChild(divRenders); + divSummary.appendChild(renderer.createKeyboardHeader(kbd, true)); - // Uses 'private' APIs that may be subject to change in the future. Keep it updated! - var layers; - if(isMobile) { - layers = keyman.osk.vkbd.layerGroup.layers; - } else { - // The desktop OSK will be overpopulated, with a number of blank layers to display in most cases. - // We instead rely upon the KLS definition to ensure we keep the renders sparse. - layers = keyman.core.activeKeyboard._legacyLayoutSpec.KLS; - } + let divRenders = document.createElement('div'); + divSummary.appendChild(divRenders); - let renderLayer = function(i: number) { - return new Promise(function(resolve) { - // (Private API) Directly sets the keyboard layer within KMW, then uses .show to force-display it. - if(keyman.osk.vkbd) { - keyman.core.keyboardProcessor.layerId = Object.keys(layers)[i]; - } else { - console.error("Error - keyman.osk.vkbd is undefined!"); - } - // Make sure the active element's still set! - renderer.setActiveDummy(); - keyman.osk.show(true); - - (document as any).fonts.ready.then(function() { - window.setTimeout(function() { - renderer.render(box, isMobile).then(function(imgEle: HTMLImageElement) { - let eleLayer = document.createElement('div'); - let eleLayerId = document.createElement('p'); - eleLayerId.textContent = 'Layer ID: ' + Object.keys(layers)[i]; - - eleLayer.appendChild(eleLayerId); - eleLayer.appendChild(imgEle); - eleLayer.appendChild(document.createElement('br')); - - divRenders.appendChild(eleLayer); - resolve(i); - }); - }, 100); - }); + // Uses 'private' APIs that may be subject to change in the future. Keep it updated! + var layers; + if(isMobile) { + layers = keyman.osk.vkbd.layerGroup.layers; + } else { + // The desktop OSK will be overpopulated, with a number of blank layers to display in most cases. + // We instead rely upon the KLS definition to ensure we keep the renders sparse. + // + // _legacyLayoutSpec is technically private, but it's what we've been using, so... yeah. + layers = keyman.core.activeKeyboard['_legacyLayoutSpec'].KLS; + } + + let renderLayer = function(i: number) { + return new Promise(function(resolve) { + // (Private API) Directly sets the keyboard layer within KMW, then uses .show to force-display it. + if(keyman.osk.vkbd) { + keyman.core.keyboardProcessor.layerId = Object.keys(layers)[i]; + } else { + console.error("Error - keyman.osk.vkbd is undefined!"); + } + // Make sure the active element's still set! + renderer.setActiveDummy(); + keyman.osk.show(true); + + (document as any).fonts.ready.then(function() { + window.setTimeout(function() { + renderer.render(box, isMobile).then(function(imgEle: HTMLImageElement) { + let eleLayer = document.createElement('div'); + let eleLayerId = document.createElement('p'); + eleLayerId.textContent = 'Layer ID: ' + Object.keys(layers)[i]; + + eleLayer.appendChild(eleLayerId); + eleLayer.appendChild(imgEle); + eleLayer.appendChild(document.createElement('br')); + + divRenders.appendChild(eleLayer); + resolve(i); + }); + }, 100); }); - }; + }); + }; + + // The resulting Promise will only call it's `.then()` once all of this keyboard's renders have been completed. + return renderer.arrayPromiseIteration(renderLayer, Object.keys(layers).length); + }).catch(function() { + console.log("Failed to load the \"" + kbd.InternalName + "\" keyboard for rendering!"); + divSummary.appendChild(renderer.createKeyboardHeader(kbd, false)); + return Promise.resolve(); + }); + } - // The resulting Promise will only call it's `.then()` once all of this keyboard's renders have been completed. - return renderer.arrayPromiseIteration(renderLayer, Object.keys(layers).length); - }).catch(function() { - console.log("Failed to load the \"" + kbd['InternalName'] + "\" keyboard for rendering!"); - divSummary.appendChild(renderer.createKeyboardHeader(kbd, false)); + // Synchronously performs asynchronous operations across a loop, one at a time. + // Necessary due to the nature of KMW OSK rendering. + private arrayPromiseIteration(promiseGenerator: (i: number) => Promise, length: number): Promise { + let iteration = function(index: number): Promise { + if(index < length) { + var promise = promiseGenerator(index); + return promise.then(function(index: number) { + return iteration(++index); + }) + } else { return Promise.resolve(); - }); - } - - // Synchronously performs asynchronous operations across a loop, one at a time. - // Necessary due to the nature of KMW OSK rendering. - private arrayPromiseIteration(promiseGenerator: (i: number) => Promise, length: number): Promise { - let iteration = function(index: number): Promise { - if(index < length) { - var promise = promiseGenerator(index); - return promise.then(function(index: number) { - return iteration(++index); - }) - } else { - return Promise.resolve(); - } } - - return iteration(0); } - fillDeviceNotes() { - let description = document.createElement('p'); - let device = new com.keyman.Device(); - device.detect(); + return iteration(0); + } - description.appendChild(document.createTextNode('Browser: ' + device.browser)); - description.appendChild(document.createElement('br')); - description.appendChild(document.createTextNode('OS: ' + device.OS)); - description.appendChild(document.createElement('br')); - description.appendChild(document.createTextNode('Form factor: ' + device.formFactor)); - description.appendChild(document.createElement('br')); - description.appendChild(document.createTextNode('Touchable: ' + device.touchable)); + fillDeviceNotes() { + let description = document.createElement('p'); + let device = new Device(); + device.detect(); - document.getElementById('deviceNotes').appendChild(description); - } + description.appendChild(document.createTextNode('Browser: ' + device.browser)); + description.appendChild(document.createElement('br')); + description.appendChild(document.createTextNode('OS: ' + device.OS)); + description.appendChild(document.createElement('br')); + description.appendChild(document.createTextNode('Form factor: ' + device.formFactor)); + description.appendChild(document.createElement('br')); + description.appendChild(document.createTextNode('Touchable: ' + device.touchable)); - setActiveDummy() { - window['keyman'].domManager.activeElement = BatchRenderer.dummy; - } + document.getElementById('deviceNotes').appendChild(description); + } - async run(allLayers, filter) { - BatchRenderer.allLayers = allLayers; - BatchRenderer.keyboardMatch = new RegExp('^Keyboard_('+filter+')', 'i'); - if(window['keyman']) { - let keyman = window['keyman']; - - let cc = await this.startCapture(); - BatchRenderer.captureStream = cc.captureStream; - BatchRenderer.video = cc.video; - - // Establish a 'dummy' element to bypass the 'nothing's active' check KMW usually uses. - BatchRenderer.dummy = document.createElement('input'); - window['keyman'].attachToControl(BatchRenderer.dummy); - this.setActiveDummy(); - - BatchRenderer.divMaster = document.getElementById('renderList'); - if(BatchRenderer.divMaster.childElementCount > 0) { - console.log("Prior bulk-renderer run detected. Terminating execution."); - return; - } + setActiveDummy() { + keyman.setActiveElement(BatchRenderer.dummy, true); + } - // We want the renderer to control where the keyboard is displayed. - // Also bypasses another 'fun' OSK complication. - if(keyman.util.device.formFactor == 'desktop') { - keyman.osk.userPositioned = true; - } + async run(allLayers, filter) { + BatchRenderer.allLayers = allLayers; + BatchRenderer.keyboardMatch = new RegExp('^Keyboard_('+filter+')', 'i'); + if(window['keyman']) { + let cc = await this.startCapture(); + BatchRenderer.captureStream = cc.captureStream; + BatchRenderer.video = cc.video; + + // Establish a 'dummy' element to bypass the 'nothing's active' check KMW usually uses. + BatchRenderer.dummy = document.createElement('input'); + keyman.attachToControl(BatchRenderer.dummy); + this.setActiveDummy(); + + BatchRenderer.divMaster = document.getElementById('renderList'); + if(BatchRenderer.divMaster.childElementCount > 0) { + console.log("Prior bulk-renderer run detected. Terminating execution."); + return; + } - // Assumes that the keyboards have been preloaded for us. - let kbds = this.filterKeyboards(); + // We want the renderer to control where the keyboard is displayed. + // Also bypasses another 'fun' OSK complication. + if(keyman.config.hostDevice.formFactor == 'desktop') { + (keyman.osk as FloatingOSKView).userPositioned = true; + } - console.log("Unique keyboard ids detected: " + Object.keys(kbds).length); + // Assumes that the keyboards have been preloaded for us. + let kbds = this.filterKeyboards(); - let renderer = this; + console.log("Unique keyboard ids detected: " + Object.keys(kbds).length); - let keyboardIterator = function(i) { - return new Promise(function(resolve) { - renderer.processKeyboard(kbds[Object.keys(kbds)[i]]).then(function () { - //console.log("Keyboard " + i + " processed!"); - resolve(i); - }); + let renderer = this; + + let keyboardIterator = function(i) { + return new Promise(function(resolve) { + renderer.processKeyboard(kbds[Object.keys(kbds)[i]]).then(function () { + //console.log("Keyboard " + i + " processed!"); + resolve(i); }); - }; + }); + }; - this.arrayPromiseIteration(keyboardIterator, Object.keys(kbds).length).then(function() { - // Once all renders are done, we can now tidy the page up and prep it for final display + potential file-saving. + this.arrayPromiseIteration(keyboardIterator, Object.keys(kbds).length).then(function() { + // Once all renders are done, we can now tidy the page up and prep it for final display + potential file-saving. - // This will go at the top of the page when finished, but not when actively rendering. - // We want to leave as much space visible as possible when actively rendering keyboards - // so that auto-scrolling isn't an issue. - renderer.fillDeviceNotes(); - }); - } else { - console.error("KeymanWeb not detected!"); - } + // This will go at the top of the page when finished, but not when actively rendering. + // We want to leave as much space visible as possible when actively rendering keyboards + // so that auto-scrolling isn't an issue. + renderer.fillDeviceNotes(); + }); + } else { + console.error("KeymanWeb not detected!"); } } +} - (function(){ - window['kmw_renderer'] = new com.keyman.renderer.BatchRenderer(); - })(); -} \ No newline at end of file +(function(){ + window['kmw_renderer'] = new BatchRenderer(); +})(); \ No newline at end of file diff --git a/web/src/tools/testing/bulk_rendering/tsconfig.json b/web/src/tools/testing/bulk_rendering/tsconfig.json index b200d751f079d1a8e6c194fdd240e7db9578e142..12d77f538969b443a381da5389b5feb7760021a3 100644 --- a/web/src/tools/testing/bulk_rendering/tsconfig.json +++ b/web/src/tools/testing/bulk_rendering/tsconfig.json @@ -3,11 +3,18 @@ "compilerOptions": { "allowJs": true, + "module": "es6", + "moduleResolution": "Node16", + "allowSyntheticDefaultImports": true, "inlineSources": true, - "module": "none", - "outFile": "../../../../build/tools/testing/bulk_rendering/bulk_render.js", "sourceMap": true, - "target": "es5" + "sourceRoot": "keyman", + "baseUrl": "./", + "outDir": "../../../../build/tools/testing/bulk_rendering/obj/", + "tsBuildInfoFile": "../../../../build/tools/testing/bulk_rendering/obj/tsconfig.tsbuildinfo", + "target": "es5", + "rootDir": ".", + "composite": true }, "files": [ @@ -15,9 +22,7 @@ ], "references": [ - { "path": "../../../../../common/web/utils", "prepend": true }, - { "path": "../../../../../common/web/keyman-version", "prepend": true }, - { "path": "../../../../../common/web/lm-message-types" }, - { "path": "../../../engine/device-detect", "prepend": true} + { "path": "../../../app/browser" }, + { "path": "../../../engine/device-detect" } ] } diff --git a/web/src/tools/testing/recorder/browserDriver.ts b/web/src/tools/testing/recorder/browserDriver.ts index 55402f5afb6a302d3994406cdb147745018b687e..53b16ab53d98068372120b507de2aa718343afd5 100644 --- a/web/src/tools/testing/recorder/browserDriver.ts +++ b/web/src/tools/testing/recorder/browserDriver.ts @@ -1,93 +1,102 @@ -namespace KMWRecorder { - export class BrowserDriver { - static readonly physicalEventClass: string = "KeyboardEvent"; - static readonly physicalEventType: string = "keydown"; - - static readonly oskEventClass: string = "MouseEvent"; - static readonly oskDownMouseType: string = "mousedown"; - static readonly oskUpMouseType: string = "mouseup"; - static readonly oskDownTouchType: string = "touchstart"; - static readonly oskUpTouchType: string = "touchend"; - - /** The destination element for use in simulation. */ - private target: HTMLElement; - - constructor(target: HTMLElement) { - this.target = target; - } +import { + InputEventSpec, + InputEventSpecSequence, + OSKInputEventSpec, + PhysicalInputEventSpec +} from "@keymanapp/recorder-core"; + +import { type KeymanEngine } from 'keyman/app/browser'; + +declare var keyman: KeymanEngine; + +export class BrowserDriver { + static readonly physicalEventClass: string = "KeyboardEvent"; + static readonly physicalEventType: string = "keydown"; + + static readonly oskEventClass: string = "MouseEvent"; + static readonly oskDownMouseType: string = "mousedown"; + static readonly oskUpMouseType: string = "mouseup"; + static readonly oskDownTouchType: string = "touchstart"; + static readonly oskUpTouchType: string = "touchend"; + + /** The destination element for use in simulation. */ + private target: HTMLElement; + + constructor(target: HTMLElement) { + this.target = target; + } - simulateEvent(eventSpec: InputEventSpec) { - switch(eventSpec.type) { - case "key": - this.simulateHardwareEvent(eventSpec as PhysicalInputEventSpec); - break; - case "osk": - this.simulateOSKEvent(eventSpec as OSKInputEventSpec); - break; - } + simulateEvent(eventSpec: InputEventSpec) { + switch(eventSpec.type) { + case "key": + this.simulateHardwareEvent(eventSpec as PhysicalInputEventSpec); + break; + case "osk": + this.simulateOSKEvent(eventSpec as OSKInputEventSpec); + break; } + } + + simulateHardwareEvent(eventSpec: PhysicalInputEventSpec) { + // Yep, not KeyboardEvent. "keyCode" is nasty-bugged in Chrome and unusable if initializing through KeyboardEvent. + let event = new Event(BrowserDriver.physicalEventType); + event['key'] = eventSpec.key; + event['code'] = eventSpec.code; + event['keyCode'] = eventSpec.keyCode; + event['location'] = eventSpec.location; + event['getModifierState'] = eventSpec.getModifierState.bind(eventSpec); + + this.target.dispatchEvent(event); + } - simulateHardwareEvent(eventSpec: PhysicalInputEventSpec) { - // Yep, not KeyboardEvent. "keyCode" is nasty-bugged in Chrome and unusable if initializing through KeyboardEvent. - let event = new Event(BrowserDriver.physicalEventType); - event['key'] = eventSpec.key; - event['code'] = eventSpec.code; - event['keyCode'] = eventSpec.keyCode; - event['location'] = eventSpec.location; - event['getModifierState'] = eventSpec.getModifierState.bind(eventSpec); + simulateOSKEvent(eventSpec: OSKInputEventSpec) { + let target = this.target; + let oskKeyElement = document.getElementById(eventSpec.keyID); - this.target.dispatchEvent(event); + if(!oskKeyElement) { + console.error('Could not find OSK key "' + eventSpec.keyID + '"!'); + // The following lines will throw an appropriate-enough error. + return; } - simulateOSKEvent(eventSpec: OSKInputEventSpec) { - let target = this.target; - let oskKeyElement = document.getElementById(eventSpec.keyID); - - if(!oskKeyElement) { - console.error('Could not find OSK key "' + eventSpec.keyID + '"!'); - // The following lines will throw an appropriate-enough error. - return; - } - - // To be safe, we replicate the MouseEvent similarly to the keystroke event. - var downEvent; - var upEvent; - if((window['keyman'] as any).util.device.touchable) { - downEvent = new Event(BrowserDriver.oskDownTouchType); - upEvent = new Event(BrowserDriver.oskUpTouchType); - downEvent['touches'] = [{"target": oskKeyElement}]; - upEvent['touches'] = [{"target": oskKeyElement}]; - downEvent['changedTouches'] = [{"target": oskKeyElement}]; - upEvent['changedTouches'] = [{"target": oskKeyElement}]; - } else { - downEvent = new Event(BrowserDriver.oskDownMouseType); - upEvent = new Event(BrowserDriver.oskUpMouseType); - downEvent['relatedTarget'] = target; - upEvent['relatedTarget'] = target; - // Mouse-click driven OSK use involves use of at least one mouse button. - downEvent['button'] = upEvent['button'] = 0; - downEvent['buttons'] = 1; - upEvent['buttons'] = 0; - } - - oskKeyElement.dispatchEvent(downEvent); - oskKeyElement.dispatchEvent(upEvent); + // To be safe, we replicate the MouseEvent similarly to the keystroke event. + var downEvent; + var upEvent; + if(keyman.config.hostDevice.touchable) { + downEvent = new Event(BrowserDriver.oskDownTouchType); + upEvent = new Event(BrowserDriver.oskUpTouchType); + downEvent['touches'] = [{"target": oskKeyElement}]; + upEvent['touches'] = [{"target": oskKeyElement}]; + downEvent['changedTouches'] = [{"target": oskKeyElement}]; + upEvent['changedTouches'] = [{"target": oskKeyElement}]; + } else { + downEvent = new Event(BrowserDriver.oskDownMouseType); + upEvent = new Event(BrowserDriver.oskUpMouseType); + downEvent['relatedTarget'] = target; + upEvent['relatedTarget'] = target; + // Mouse-click driven OSK use involves use of at least one mouse button. + downEvent['button'] = upEvent['button'] = 0; + downEvent['buttons'] = 1; + upEvent['buttons'] = 0; } - // Execution of a test sequence depends on the testing environment; integrated - // testing requires browser-specific code. - simulateSequence(sequence: InputEventSpecSequence): string { - let ele = this.target; + oskKeyElement.dispatchEvent(downEvent); + oskKeyElement.dispatchEvent(upEvent); + } + + // Execution of a test sequence depends on the testing environment; integrated + // testing requires browser-specific code. + simulateSequence(sequence: InputEventSpecSequence): string { + let ele = this.target; - for(var i=0; i < sequence.inputs.length; i++) { - this.simulateEvent(sequence.inputs[i]); - } + for(var i=0; i < sequence.inputs.length; i++) { + this.simulateEvent(sequence.inputs[i]); + } - if(ele instanceof HTMLInputElement || ele instanceof HTMLTextAreaElement) { - return ele.value; - } else { - return ele.textContent; - } + if(ele instanceof HTMLInputElement || ele instanceof HTMLTextAreaElement) { + return ele.value; + } else { + return ele.textContent; } } } \ No newline at end of file diff --git a/web/src/tools/testing/recorder/browserProctor.ts b/web/src/tools/testing/recorder/browserProctor.ts index 74b02449b298cae67ffcd4673884b8389906455e..8a4d4f6b860db11ebbaf77f1fe9c6fbeeab41eea 100644 --- a/web/src/tools/testing/recorder/browserProctor.ts +++ b/web/src/tools/testing/recorder/browserProctor.ts @@ -1,80 +1,97 @@ -/// // In theory, we could totally split Scribe off into its own, separate micro-module. // Only BrowserDriver.ts and its prerequisites are needed during actual integrated unit testing. /// -namespace KMWRecorder { - type AssertCallback = (s1: any, s2: any, msg?: string) => void; +import { type DeviceSpec } from "@keymanapp/web-utils"; - function resetElement(ele: HTMLElement): void { - if(ele instanceof HTMLInputElement || ele instanceof HTMLTextAreaElement) { - (window['keyman'] as any).resetContext(); - ele.value = ""; - } else { - (window['keyman'] as any).resetContext(); - ele.textContent = ""; - } - } +import { type OutputTarget } from "@keymanapp/keyboard-processor"; - /** - * Runs Recorder-generated tests in a browser. - * - * Future notes: further abstraction needed; much of this code will be in common with Node. - */ - export class BrowserProctor extends Proctor { - target: HTMLElement; - usingOSK: boolean; +import { type KeymanEngine } from 'keyman/app/browser'; - constructor(target: HTMLElement, device: com.keyman.utils.DeviceSpec, usingOSK: boolean, assert: AssertCallback) { - super(device, assert); +declare var keyman: KeymanEngine; - this.target = target; - this.usingOSK = usingOSK; - } +import { + InputEventSpec, + InputEventSpecSequence, + KeyboardTest, + Proctor, + RecordedKeystrokeSequence, + TestSequence, + TestSet +} from "@keymanapp/recorder-core"; +import { BrowserDriver } from "./browserDriver.js"; - // Performs browser-specific global test prep. - beforeAll() { - let ele = this.target; - (window['keyman'] as any).setActiveElement(ele['base'] ? ele['base'] : ele); - } - before() { - resetElement(this.target); - } +type AssertCallback = (s1: any, s2: any, msg?: string) => void; - compatibleWithSuite(testSuite: KeyboardTest): boolean { - // So far, the only thing using Recorder code always specifies the necessary data to construct - // DOM events for use in integrated testing. - return true; - } +function resetElement(ele: HTMLElement): void { + if(ele instanceof HTMLInputElement || ele instanceof HTMLTextAreaElement) { + keyman.resetContext(); + ele.value = ""; + } else { + keyman.resetContext(); + ele.textContent = ""; + } +} - // We only need to filter test cases when performing tests in an integrated - // environment. - matchesTestSet(testSet: TestSet) { - return testSet.isValidForDevice(this.device, this.usingOSK); - } +/** + * Runs Recorder-generated tests in a browser. + * + * Future notes: further abstraction needed; much of this code will be in common with Node. + */ +export class BrowserProctor extends Proctor { + target: HTMLElement; + usingOSK: boolean; - // Execution of a test sequence depends on the testing environment; this handles - // the browser-specific aspects. - simulateSequence(sequence: TestSequence, outputTarget?: com.keyman.text.OutputTarget): string { - let driver = new BrowserDriver(this.target); + constructor(target: HTMLElement, device: DeviceSpec, usingOSK: boolean, assert: AssertCallback) { + super(device, assert); - // For the version 10.0 spec - if(sequence instanceof InputEventSpecSequence) { - return driver.simulateSequence(sequence); + this.target = target; + this.usingOSK = usingOSK; + } - // For the version 14.0+ spec - } else if(sequence instanceof RecordedKeystrokeSequence) { - let inputSpecs: InputEventSpec[] = []; + // Performs browser-specific global test prep. + beforeAll() { + let ele = this.target; + (window['keyman'] as any).setActiveElement(ele['base'] ? ele['base'] : ele); + } - for(let i=0; i < sequence.inputs.length; i++) { - inputSpecs[i] = sequence.inputs[i].inputEventSpec; - } + before() { + resetElement(this.target); + } + + compatibleWithSuite(testSuite: KeyboardTest): boolean { + // So far, the only thing using Recorder code always specifies the necessary data to construct + // DOM events for use in integrated testing. + return true; + } + + // We only need to filter test cases when performing tests in an integrated + // environment. + matchesTestSet(testSet: TestSet) { + return testSet.isValidForDevice(this.device, this.usingOSK); + } - // Converts the sequence back to version 10.0, since it's very well made for browser simulation. - let eventSpecSequence = new InputEventSpecSequence(inputSpecs, sequence.output, sequence.msg); - return driver.simulateSequence(eventSpecSequence); + // Execution of a test sequence depends on the testing environment; this handles + // the browser-specific aspects. + simulateSequence(sequence: TestSequence, outputTarget?: OutputTarget): string { + let driver = new BrowserDriver(this.target); + + // For the version 10.0 spec + if(sequence instanceof InputEventSpecSequence) { + return driver.simulateSequence(sequence); + + // For the version 14.0+ spec + } else if(sequence instanceof RecordedKeystrokeSequence) { + let inputSpecs: InputEventSpec[] = []; + + for(let i=0; i < sequence.inputs.length; i++) { + inputSpecs[i] = sequence.inputs[i].inputEventSpec; } + + // Converts the sequence back to version 10.0, since it's very well made for browser simulation. + let eventSpecSequence = new InputEventSpecSequence(inputSpecs, sequence.output, sequence.msg); + return driver.simulateSequence(eventSpecSequence); } } } \ No newline at end of file diff --git a/web/src/tools/testing/recorder/build-bundler.js b/web/src/tools/testing/recorder/build-bundler.js new file mode 100644 index 0000000000000000000000000000000000000000..d74fbecd2bd4b6831dec41aa02af7d42bf5b6e39 --- /dev/null +++ b/web/src/tools/testing/recorder/build-bundler.js @@ -0,0 +1,15 @@ +import esbuild from 'esbuild'; + +await esbuild.build({ + bundle: true, + sourcemap: true, + format: "esm", + nodePaths: ['../../../../../node_modules'], + entryPoints: { + 'index': '../../../../build/tools/testing/recorder/obj/index.js', + }, + outdir: '../../../../build/tools/testing/recorder/lib/', + outExtension: { '.js': '.mjs' }, + tsconfig: './tsconfig.json', + target: "es5" +}); \ No newline at end of file diff --git a/web/src/tools/testing/recorder/build.sh b/web/src/tools/testing/recorder/build.sh index dfe0d600d7e560aad0b5cf81bb8a6c10f8e03dd4..6b5cb18342668eee08e4938148103a1c810ecc61 100755 --- a/web/src/tools/testing/recorder/build.sh +++ b/web/src/tools/testing/recorder/build.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # # Compile KeymanWeb's dev & test tool modules -# -set -eu ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary @@ -10,6 +8,8 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE +SUBPROJECT_NAME=tools/testing/recorder +. "$KEYMAN_ROOT/web/common.inc.sh" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" # This script runs from its own folder @@ -27,28 +27,10 @@ builder_describe "Builds the Keyman Engine for Web's test-sequence recording too builder_describe_outputs \ configure /node_modules \ - build /web/build/tools/testing/recorder/index.js + build /web/build/$SUBPROJECT_NAME/lib/index.mjs builder_parse "$@" -### CONFIGURE ACTIONS - -if builder_start_action configure; then - verify_npm_setup - builder_finish_action success configure -fi - -### CLEAN ACTIONS - -if builder_start_action clean; then - rm -rf ../../../../build/tools/testing/recorder/ - builder_finish_action success clean -fi - -### BUILD ACTIONS - -if builder_start_action build; then - tsc -b $THIS_SCRIPT_PATH/tsconfig.json - - builder_finish_action success build -fi \ No newline at end of file +builder_run_action configure verify_npm_setup +builder_run_action clean rm -rf ../../../../build/$SUBPROJECT_NAME/ +builder_run_action build compile $SUBPROJECT_NAME \ No newline at end of file diff --git a/web/src/tools/testing/recorder/index.html b/web/src/tools/testing/recorder/index.html index 5f7dfe19efb50da42d7403362c18debc3d63917a..f4a68ba76902102e7d46e1fbd34f4af4c245fd8f 100644 --- a/web/src/tools/testing/recorder/index.html +++ b/web/src/tools/testing/recorder/index.html @@ -46,23 +46,17 @@ } - - - - - + + + +