Massive refactor largely to support Unicode on Mac

This does a bunch of crazy stuff:

- The ability to set a unicode mode (right now only Linux+ibus or
MacOS-RALT) in the keymap. This will be changeable at runtime soon, to
allow a single keyboard to be able to send table flips and whatever
other crazy stuff on any OS the board is plugged into (something that's
not currently doable on QMK, so yay us?)

- As part of the above, there is now just one user-facing macro for
unicode codepoint submission,
`kmk.common.macros.unicode.unicode_sequence`. Users should never use the
platform-specific macros, partly because they just outright won't work.
There's all sorts of fun stuff in these methods now, thank goodness
MicroPython supports the `yield from` construct.

- Keycode (these should really be renamed Keysym or something) objects
that are intended to not be pressed, or not be released. Right now these
properties are completely ignored if not part of a macro, and it's
probably sane to keep it that way. This was necessary to support MacOS's
"hold RALT while typing the codepoint characters" flow.

- Other refactor-y bits, like moving macro support to `kmk/common`
rather than sitting at the top level of the tree. One day `kmk/common`
may make sense to surface at top level `kmk/`, but that's a discussion
for another day.

kmk/common/consts.py

@@ -113,3 +113,9 @@ class DiodeOrientation:
 
     COLUMNS = 0
     ROWS = 1
+
+
+class UnicodeModes:
+    NOOP = 0
+    LINUX = IBUS = 1
+    MACOS = OSX = RALT = 2

kmk/common/event_defs.py

@@ -16,13 +16,14 @@ MACRO_COMPLETE_EVENT = const(8)
 logger = logging.getLogger(__name__)
 
 
-def init_firmware(keymap, row_pins, col_pins, diode_orientation):
+def init_firmware(keymap, row_pins, col_pins, diode_orientation, unicode_mode):
     return {
         'type': INIT_FIRMWARE_EVENT,
         'keymap': keymap,
         'row_pins': row_pins,
         'col_pins': col_pins,
         'diode_orientation': diode_orientation,
+        'unicode_mode': unicode_mode,
     }
 
 
@@ -77,10 +78,9 @@ def hid_report_event():
     }
 
 
-def macro_complete_event(macro):
+def macro_complete_event():
     return {
         'type': MACRO_COMPLETE_EVENT,
-        'macro': macro,
     }
 
 
@@ -126,12 +126,13 @@ def matrix_changed(new_matrix):
                 except ImportError:
                     logger.warning('Tried to reset to bootloader, but not supported on this chip?')
 
-        while get_state().macros_pending:
-            macro = get_state().macros_pending[0]
+        with get_state() as new_state:
+            if new_state.macro_pending:
+                macro = new_state.macro_pending
 
-            for event in macro():
-                dispatch(event)
+                for event in macro(new_state):
+                    dispatch(event)
 
-            dispatch(macro_complete_event(macro))
+                dispatch(macro_complete_event())
 
     return _key_pressed

kmk/common/internal_state.py

@@ -1,14 +1,14 @@
 import logging
 import sys
 
-from kmk.common.consts import DiodeOrientation
+from kmk.common.consts import DiodeOrientation, UnicodeModes
 from kmk.common.event_defs import (HID_REPORT_EVENT, INIT_FIRMWARE_EVENT,
                                    KEY_DOWN_EVENT, KEY_UP_EVENT,
                                    KEYCODE_DOWN_EVENT, KEYCODE_UP_EVENT,
                                    MACRO_COMPLETE_EVENT, NEW_MATRIX_EVENT)
 from kmk.common.internal_keycodes import process_internal_key_event
 from kmk.common.keycodes import FIRST_KMK_INTERNAL_KEYCODE, Keycodes
-from kmk.macros import KMKMacro
+from kmk.common.macros import KMKMacro
 
 
 class ReduxStore:
@@ -54,7 +54,8 @@ class ReduxStore:
 class InternalState:
     modifiers_pressed = frozenset()
     keys_pressed = frozenset()
-    macros_pending = []
+    macro_pending = None
+    unicode_mode = UnicodeModes.NOOP
     keymap = []
     row_pins = []
     col_pins = []
@@ -77,6 +78,7 @@ class InternalState:
             'keys_pressed': self.keys_pressed,
             'modifiers_pressed': self.modifiers_pressed,
             'active_layers': self.active_layers,
+            'unicode_mode': self.unicode_mode,
         }
 
         if verbose:
@@ -164,7 +166,7 @@ def kmk_reducer(state=None, action=None, logger=None):
         if isinstance(changed_key, KMKMacro):
             if changed_key.keyup:
                 return state.update(
-                    macros_pending=state.macros_pending + [changed_key.keyup],
+                    macro_pending=changed_key.keyup,
                 )
 
             return state
@@ -194,7 +196,7 @@ def kmk_reducer(state=None, action=None, logger=None):
         if isinstance(changed_key, KMKMacro):
             if changed_key.keydown:
                 return state.update(
-                    macros_pending=state.macros_pending + [changed_key.keydown],
+                    macro_pending=changed_key.keydown,
                 )
 
             return state
@@ -216,6 +218,7 @@ def kmk_reducer(state=None, action=None, logger=None):
             row_pins=action['row_pins'],
             col_pins=action['col_pins'],
             diode_orientation=action['diode_orientation'],
+            unicode_mode=action['unicode_mode'],
             matrix=[
                 [False for c in action['col_pins']]
                 for r in action['row_pins']
@@ -230,12 +233,7 @@ def kmk_reducer(state=None, action=None, logger=None):
         return state
 
     if action['type'] == MACRO_COMPLETE_EVENT:
-        return state.update(
-            macros_pending=[
-                m for m in state.macros_pending
-                if m != action['macro']
-            ],
-        )
+        return state.update(macro_pending=None)
 
     # On unhandled events, log and do not mutate state
     logger.warning('Unhandled event! Returning state unmodified.')

kmk/common/keycodes.py

@@ -14,17 +14,36 @@ LayerKeycode = namedtuple('LayerKeycode', ('code', 'layer'))
 
 
 class Keycode:
-    def __init__(self, code, has_modifiers=None):
+    def __init__(self, code, has_modifiers=None, no_press=False, no_release=False):
         self.code = code
         self.has_modifiers = has_modifiers
+        # cast to bool() in case we get a None value
+        self.no_press = bool(no_press)
+        self.no_release = bool(no_press)
+
+    def __call__(self, no_press=None, no_release=None):
+        if no_press is None and no_release is None:
+            return self
+
+        return Keycode(
+            code=self.code,
+            has_modifiers=self.has_modifiers,
+            no_press=no_press,
+            no_release=no_release,
+        )
 
 
-class ModifierKeycode:
-    def __init__(self, code):
-        self.code = code
+class ModifierKeycode(Keycode):
+    def __call__(self, modified_code=None, no_press=None, no_release=None):
+        if modified_code is None and no_press is None and no_release is None:
+            return self
 
-    def __call__(self, modified_code):
-        new_keycode = Keycode(modified_code.code, {self.code})
+        new_keycode = Keycode(
+            modified_code.code,
+            {self.code},
+            no_press=no_press,
+            no_release=no_release,
+        )
 
         if modified_code.has_modifiers:
             new_keycode.has_modifiers |= modified_code.has_modifiers

kmk/common/macros/simple.py

@@ -0,0 +1,17 @@
+from kmk.common.event_defs import (hid_report_event, keycode_down_event,
+                                   keycode_up_event)
+from kmk.common.macros import KMKMacro
+
+
+def simple_key_sequence(seq):
+    def _simple_key_sequence(state):
+        for key in seq:
+            if not getattr(key, 'no_press', None):
+                yield keycode_down_event(key)
+                yield hid_report_event()
+
+            if not getattr(key, 'no_release', None):
+                yield keycode_up_event(key)
+                yield hid_report_event()
+
+    return KMKMacro(keydown=_simple_key_sequence)

kmk/common/macros/unicode.py

@@ -0,0 +1,45 @@
+from kmk.common.consts import UnicodeModes
+from kmk.common.event_defs import (hid_report_event, keycode_down_event,
+                                   keycode_up_event)
+from kmk.common.keycodes import Common, Modifiers
+from kmk.common.macros import KMKMacro
+from kmk.common.macros.simple import simple_key_sequence
+
+IBUS_KEY_COMBO = Modifiers.KC_LCTRL(Modifiers.KC_LSHIFT(Common.KC_U))
+
+
+def generate_codepoint_keysym_seq(codepoint):
+    return [
+        getattr(Common, 'KC_{}'.format(codepoint_fragment.upper()))
+        for codepoint_fragment in codepoint
+    ]
+
+
+def unicode_sequence(codepoints):
+    def _unicode_sequence(state):
+        if state.unicode_mode == UnicodeModes.IBUS:
+            yield from _ibus_unicode_sequence(codepoints, state)
+        elif state.unicode_mode == UnicodeModes.RALT:
+            yield from _ralt_unicode_sequence(codepoints, state)
+
+    return KMKMacro(keydown=_unicode_sequence)
+
+
+def _ralt_unicode_sequence(codepoints, state):
+    for codepoint in codepoints:
+        yield keycode_down_event(Modifiers.RALT(no_release=True))
+        yield from simple_key_sequence(generate_codepoint_keysym_seq(codepoint)).keydown(state)
+        yield keycode_up_event(Modifiers.RALT(no_press=True))
+
+
+def _ibus_unicode_sequence(codepoints, state):
+    for codepoint in codepoints:
+        yield keycode_down_event(IBUS_KEY_COMBO)
+        yield hid_report_event()
+        yield keycode_up_event(IBUS_KEY_COMBO)
+        yield hid_report_event()
+
+        seq = generate_codepoint_keysym_seq(codepoint)
+        seq.append(Common.KC_ENTER)
+
+        yield from simple_key_sequence(seq).keydown(state)

kmk/entrypoints/handwire/pyboard.py

@@ -1,6 +1,7 @@
 import sys
 from logging import DEBUG
 
+from kmk.common.consts import UnicodeModes
 from kmk.firmware import Firmware
 from kmk.micropython.pyb_hid import HIDHelper
 
@@ -8,12 +9,18 @@ from kmk.micropython.pyb_hid import HIDHelper
 def main():
     from kmk_keyboard_user import cols, diode_orientation, keymap, rows
 
+    try:
+        from kmk_keyboard_user import unicode_mode
+    except Exception:
+        unicode_mode = UnicodeModes.NOOP
+
     try:
         firmware = Firmware(
             keymap=keymap,
             row_pins=rows,
             col_pins=cols,
             diode_orientation=diode_orientation,
+            unicode_mode=unicode_mode,
             hid=HIDHelper,
             log_level=DEBUG,
         )

kmk/firmware.py

@@ -12,7 +12,8 @@ except ImportError:
 class Firmware:
     def __init__(
         self, keymap, row_pins, col_pins,
-        diode_orientation, hid=None, log_level=logging.NOTSET,
+        diode_orientation, unicode_mode=None,
+        hid=None, log_level=logging.NOTSET,
     ):
         logger = logging.getLogger(__name__)
         logger.setLevel(log_level)
@@ -36,6 +37,7 @@ class Firmware:
             row_pins=row_pins,
             col_pins=col_pins,
             diode_orientation=diode_orientation,
+            unicode_mode=unicode_mode,
         ))
 
     def _subscription(self, state, action):

kmk/macros/simple.py

@@ -1,29 +0,0 @@
-from kmk.common.event_defs import (hid_report_event, keycode_down_event,
-                                   keycode_up_event)
-from kmk.common.keycodes import Common, Modifiers
-from kmk.macros import KMKMacro
-
-
-def simple_key_sequence(seq):
-    def _simple_key_sequence():
-        for key in seq:
-            yield keycode_down_event(key)
-            yield hid_report_event()
-            yield keycode_up_event(key)
-            yield hid_report_event()
-
-    return KMKMacro(keydown=_simple_key_sequence)
-
-
-def ibus_unicode_sequence(codepoints):
-    seq = []
-
-    for codepoint in codepoints:
-        seq.append(Modifiers.KC_LCTRL(Modifiers.KC_LSHIFT(Common.KC_U)))
-
-        for codepoint_fragment in codepoint:
-            seq.append(getattr(Common, 'KC_{}'.format(codepoint_fragment.upper())))
-
-        seq.append(Common.KC_ENTER)
-
-    return simple_key_sequence(seq)

user_keymaps/klardotsh/threethree_matrix_pyboard.py

@@ -1,15 +1,17 @@
 import machine
 
-from kmk.common.consts import DiodeOrientation
+from kmk.common.consts import DiodeOrientation, UnicodeModes
 from kmk.common.keycodes import KC
+from kmk.common.macros.simple import simple_key_sequence
+from kmk.common.macros.unicode import unicode_sequence
 from kmk.entrypoints.handwire.pyboard import main
-from kmk.macros.simple import ibus_unicode_sequence, simple_key_sequence
 
 p = machine.Pin.board
 cols = (p.X10, p.X11, p.X12)
 rows = (p.X1, p.X2, p.X3)
 
 diode_orientation = DiodeOrientation.COLUMNS
+unicode_mode = UnicodeModes.LINUX
 
 MACRO_TEST_STRING = simple_key_sequence([
     KC.LSHIFT(KC.H),
@@ -26,7 +28,7 @@ MACRO_TEST_STRING = simple_key_sequence([
     KC.EXCLAIM,
 ])
 
-ANGRY_TABLE_FLIP = ibus_unicode_sequence([
+ANGRY_TABLE_FLIP = unicode_sequence([
     "28",
     "30ce",
     "ca0",