Skip to content

Plugins API

Every plugin is a plain Python module that follows a small contract rather than subclassing a base class. A module is loaded if it exposes a NAME string and a handle(cmd, core) function; the optional setup(core) hook runs once at startup, and COMMANDS/DESCRIPTION feed the help screen.

classDiagram
    class PluginContract {
        <<protocol>>
        +str NAME
        +str DESCRIPTION
        +list COMMANDS
        +setup(core)
        +handle(cmd, core)
    }
    class base
    class sleep
    class system
    class media
    class files
    class apps
    class browser
    class dictation
    class mousegrid
    class headtrack

    PluginContract <|.. base
    PluginContract <|.. sleep
    PluginContract <|.. system
    PluginContract <|.. media
    PluginContract <|.. files
    PluginContract <|.. apps
    PluginContract <|.. browser
    PluginContract <|.. dictation
    PluginContract <|.. mousegrid
    PluginContract <|.. headtrack

    note for base "zz_base.py — loads last; help and exit fallback"
    note for mousegrid "00_mousegrid.py — numeric prefix loads it early"
    note for headtrack "00_eyetrack.py — numeric prefix loads it early"

handle returns True when it consumed the command, False to signal the daemon to exit, or None to pass the command to the next plugin. Load order is alphabetical, so numeric prefixes (00_) load a plugin early and the zz_ prefix loads the base plugin last as the catch-all for help and exit.

base (zz_base)

plugins.zz_base

Base Plugin - Help and Exit.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/zz_base.py 
14
15
16
17
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

handle 

handle(cmd, core)

Handle the help and exit commands; return None to pass others through.

Returns False to signal the daemon to exit, True if help was shown, or None when the command is for another plugin.

Source code in src/plugins/zz_base.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def handle(cmd, core):
    """Handle the help and exit commands; return None to pass others through.

    Returns False to signal the daemon to exit, True if help was shown, or None when the
    command is for another plugin.
    """
    cmd_lower = cmd.lower().strip()

    # Exit - but NOT if it's "quit tracking" etc
    if "tracking" not in cmd_lower:
        if cmd_lower in ["exit", "quit", "goodbye", "bye"]:
            core.speak("Goodbye.")
            return False  # Signal to exit
        # Also match "jarvis quit" etc
        if any(cmd_lower.endswith(x) for x in [" exit", " quit"]):
            core.speak("Goodbye.")
            return False

    # Help
    if "help" in cmd_lower or "what can you do" in cmd_lower:
        show_help(core)
        return True

    return None  # Not handled

show_help 

show_help(core)

List every plugin's commands on the terminal.

Printed rather than logged: it is the direct reply to an explicit "help" request (the spoken reply tells the user to read the terminal), so it must appear regardless of the configured log verbosity.

Source code in src/plugins/zz_base.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def show_help(core):
    """List every plugin's commands on the terminal.

    Printed rather than logged: it is the direct reply to an explicit "help"
    request (the spoken reply tells the user to read the terminal), so it must
    appear regardless of the configured log verbosity.
    """
    print("\n=== Available Commands ===")  # noqa: T201
    for plugin in core.plugins:
        if hasattr(plugin, "COMMANDS"):
            print(f"\n{plugin.NAME}:")  # noqa: T201
            for cmd in plugin.COMMANDS:
                print(f"  • {cmd}")  # noqa: T201
    print()  # noqa: T201
    core.speak("Check the terminal for available commands.")

sleep

plugins.sleep

Sleep Plugin - Deactivate the assistant by voice.

"Go to sleep" (or "stop listening") releases the microphone and stops wake-word detection until the user reactivates EasySpeak from the GNOME tray indicator. It pairs with that panel icon, which only appears while the assistant is asleep (the one way back, since voice control is off until reactivated).

handle 

handle(cmd, core)

Deactivate the assistant on a sleep phrase; return None otherwise.

Source code in src/plugins/sleep.py 
23
24
25
26
27
28
29
30
31
32
33
34
def handle(cmd, core):
    """Deactivate the assistant on a sleep phrase; return None otherwise."""
    cmd_lower = cmd.lower().strip()
    if any(phrase in cmd_lower for phrase in SLEEP_PHRASES):
        # Announce only the attempt. The tray controller confirms once sleep
        # actually engages ("Reactivate me from the tray...") or, if it can't
        # reach the indicator, explains and stays awake — so we never promise a
        # tray that isn't there.
        core.speak("Going to sleep.")
        core.deactivate()
        return True
    return None

system

plugins.system

System Plugin - Volume, brightness, do not disturb.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/system.py 
38
39
40
41
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

volume_up 

volume_up(core)

Raise the volume one step.

Source code in src/plugins/system.py 
54
55
56
57
58
def volume_up(core):
    """Raise the volume one step."""
    _media_key(
        core, KEY_VOLUME_UP, ["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "10%+"]
    )

volume_down 

volume_down(core)

Lower the volume one step.

Source code in src/plugins/system.py 
61
62
63
64
65
def volume_down(core):
    """Lower the volume one step."""
    _media_key(
        core, KEY_VOLUME_DOWN, ["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "10%-"]
    )

volume_mute 

volume_mute(core)

Toggle mute on the default audio sink.

Source code in src/plugins/system.py 
68
69
70
def volume_mute(core):
    """Toggle mute on the default audio sink."""
    _media_key(core, KEY_MUTE, ["wpctl", "set-mute", "@DEFAULT_AUDIO_SINK@", "toggle"])

volume_max 

volume_max(core)

Jump near the top (85%, not a blast); no media key does this.

Source code in src/plugins/system.py 
73
74
75
def volume_max(core):
    """Jump near the top (85%, not a blast); no media key does this."""
    core.host_run(["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "85%"])

volume_min 

volume_min(core)

Drop low (15%, still audible — not a mute), set directly like volume_max.

Source code in src/plugins/system.py 
78
79
80
def volume_min(core):
    """Drop low (15%, still audible — not a mute), set directly like volume_max."""
    core.host_run(["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "15%"])

brightness_up 

brightness_up(core)

Raise screen brightness one step.

Source code in src/plugins/system.py 
83
84
85
86
87
88
89
def brightness_up(core):
    """Raise screen brightness one step."""
    _media_key(
        core,
        KEY_BRIGHTNESS_UP,
        [*_POWER_SCREEN, "org.gnome.SettingsDaemon.Power.Screen.StepUp"],
    )

brightness_down 

brightness_down(core)

Lower screen brightness one step.

Source code in src/plugins/system.py 
92
93
94
95
96
97
98
def brightness_down(core):
    """Lower screen brightness one step."""
    _media_key(
        core,
        KEY_BRIGHTNESS_DOWN,
        [*_POWER_SCREEN, "org.gnome.SettingsDaemon.Power.Screen.StepDown"],
    )

dnd_on 

dnd_on(core)

Enable do-not-disturb by hiding notification banners.

Source code in src/plugins/system.py 
101
102
103
104
105
def dnd_on(core):
    """Enable do-not-disturb by hiding notification banners."""
    core.host_run(
        ["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"]
    )

dnd_off 

dnd_off(core)

Disable do-not-disturb by showing notification banners again.

Source code in src/plugins/system.py 
108
109
110
111
112
def dnd_off(core):
    """Disable do-not-disturb by showing notification banners again."""
    core.host_run(
        ["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "true"]
    )

handle 

handle(cmd, core)

Route a volume/brightness/DND command; return None if none matched.

Source code in src/plugins/system.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def handle(cmd, core):
    """Route a volume/brightness/DND command; return None if none matched."""
    # Volume -- "louder"/"quieter" etc. work on their own, without "volume"/"sound".
    # Match whole words so "silent" doesn't fire on "silently", "softer" on "softest"...
    words = cmd.split()
    if "very" in words:
        if "loud" in words or "louder" in words:
            volume_max(core)
            return True
        if any(w in words for w in ("silent", "quiet", "quieter", "soft", "softer")):
            volume_min(core)
            return True
    louder = "louder" in words
    quieter = any(word in words for word in ("quieter", "softer", "silent"))
    # No spoken feedback for volume/mute: GNOME's native OSD and chime already
    # acknowledge the change (and a spoken reply would be inaudible once muted).
    if "volume" in cmd or "sound" in cmd or louder or quieter:
        if "up" in cmd or louder:
            volume_up(core)
            return True
        if "down" in cmd or quieter:
            volume_down(core)
            return True
        if "mute" in cmd or "unmute" in cmd:
            volume_mute(core)
            return True

    if "mute" in cmd:
        volume_mute(core)
        return True

    # Brightness
    if "brightness" in cmd or "screen" in cmd:
        if "up" in cmd or "brighter" in cmd:
            core.speak("Brighter.")
            brightness_up(core)
            return True
        if "down" in cmd or "dimmer" in cmd or "darker" in cmd:
            core.speak("Dimmer.")
            brightness_down(core)
            return True

    # Do Not Disturb
    if "do not disturb" in cmd or "dnd" in cmd or "notifications" in cmd:
        if "on" in cmd or "enable" in cmd:
            core.speak("Do not disturb on.")
            dnd_on(core)
            return True
        if "off" in cmd or "disable" in cmd:
            core.speak("Do not disturb off.")
            dnd_off(core)
            return True

    return None  # Not handled

media

plugins.media

Media Plugin - Playback controls via MPRIS.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/media.py 
16
17
18
19
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

get_media_players 

get_media_players(core)

Return the bus names of all running MPRIS media players.

Source code in src/plugins/media.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def get_media_players(core):
    """Return the bus names of all running MPRIS media players."""
    result = core.host_run(
        [
            "dbus-send",
            "--session",
            "--dest=org.freedesktop.DBus",
            "--type=method_call",
            "--print-reply",
            "/org/freedesktop/DBus",
            "org.freedesktop.DBus.ListNames",
        ]
    )
    return [
        line.split('"')[1]
        for line in result.stdout.split("\n")
        if "org.mpris.MediaPlayer2." in line
    ]

media_control 

media_control(action, core)

Send an MPRIS action (play/pause/next/previous) to every running player.

Returns False if no player is running or the action is unknown.

Source code in src/plugins/media.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def media_control(action, core):
    """Send an MPRIS action (play/pause/next/previous) to every running player.

    Returns False if no player is running or the action is unknown.
    """
    players = get_media_players(core)
    if not players:
        return False

    method = {
        "play": "Play",
        "pause": "Pause",
        "next": "Next",
        "previous": "Previous",
    }.get(action)

    if not method:
        return False

    for player in players:
        core.host_run(
            [
                "dbus-send",
                "--session",
                "--type=method_call",
                f"--dest={player}",
                "/org/mpris/MediaPlayer2",
                f"org.mpris.MediaPlayer2.Player.{method}",
            ]
        )
    return True

handle 

handle(cmd, core)

Map a playback command to an MPRIS action; return None if not media.

Source code in src/plugins/media.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def handle(cmd, core):
    """Map a playback command to an MPRIS action; return None if not media."""
    # "stop the music"/"stop playing" mean pause; bare "stop" isn't a media command.
    # Whole-word match so "stop tracking" (eye tracking) isn't caught by "track".
    words = cmd.split()
    stop_music = "stop" in words and any(
        word in words
        for word in ("music", "song", "playback", "track", "player", "playing", "play")
    )

    if "pause" in cmd or stop_music:
        core.speak("Paused.")
        media_control("pause", core)
        return True

    if "play" in cmd and "pause" not in cmd:
        core.speak("Playing.")
        media_control("play", core)
        return True

    if "next" in cmd or "skip" in cmd:
        core.speak("Next.")
        media_control("next", core)
        return True

    if "previous" in cmd or "back" in cmd:
        core.speak("Previous.")
        media_control("previous", core)
        return True

    return None  # Not handled

files

plugins.files

Files Plugin - Open folders in file manager.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/files.py 
27
28
29
30
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

open_folder 

open_folder(path, core)

Open a folder in the first available file manager; False if none found.

Source code in src/plugins/files.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def open_folder(path, core):
    """Open a folder in the first available file manager; False if none found."""
    expanded = os.path.expanduser(path)

    file_managers = [
        ("nautilus", [expanded]),
        ("dolphin", [expanded]),
        ("thunar", [expanded]),
        ("nemo", [expanded]),
        ("xdg-open", [expanded]),
    ]

    for fm, args in file_managers:
        result = core.host_run(["which", fm])
        if result.returncode == 0:
            core.host_run([fm, *args], background=True)
            return True
    return False

handle 

handle(cmd, core)

Open a known folder when the command names one; return None otherwise.

Source code in src/plugins/files.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
def handle(cmd, core):
    """Open a known folder when the command names one; return None otherwise."""
    for folder, path in FOLDERS.items():
        if folder in cmd and (
            "open" in cmd or "go to" in cmd or "show" in cmd or "browse" in cmd
        ):
            if open_folder(path, core):
                core.speak(f"Opening {folder}.")
            else:
                core.speak("No file manager found.")
            return True

    return None  # Not handled

apps

plugins.apps

Apps Plugin - Launch and close applications.

setup 

setup(c)

Store the core reference and resolve the user's default terminal.

Source code in src/plugins/apps.py 
75
76
77
78
79
def setup(c):
    """Store the core reference and resolve the user's default terminal."""
    global core
    core = c
    _register_terminal(c)

find_app 

find_app(name, core)

Find app - returns (type, id).

Source code in src/plugins/apps.py 
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def find_app(name, core):
    """Find app - returns (type, id)."""
    if name in FLATPAK_APPS:
        result = core.host_run(["flatpak", "info", FLATPAK_APPS[name]])
        if result.returncode == 0:
            return ("flatpak", FLATPAK_APPS[name])

    if name in LOCAL_APPS:
        return ("local", LOCAL_APPS[name])

    result = core.host_run(["which", name])
    if result.returncode == 0:
        return ("local", name)

    return (None, None)

launch_app 

launch_app(name, core)

Launch the named app (flatpak or local binary); False if not found.

Source code in src/plugins/apps.py 
163
164
165
166
167
168
169
170
171
172
def launch_app(name, core):
    """Launch the named app (flatpak or local binary); False if not found."""
    app_type, app_id = find_app(name, core)
    if app_type == "flatpak":
        core.host_run(["flatpak", "run", app_id], background=True)
        return True
    if app_type == "local":
        core.host_run([app_id], background=True)
        return True
    return False

close_app 

close_app(name, core)

Close the named app (flatpak kill or pkill); False if not found.

Source code in src/plugins/apps.py 
175
176
177
178
179
180
181
182
183
184
def close_app(name, core):
    """Close the named app (flatpak kill or pkill); False if not found."""
    app_type, app_id = find_app(name, core)
    if app_type == "flatpak":
        core.host_run(["flatpak", "kill", app_id])
        return True
    if app_type == "local":
        core.host_run(["pkill", "-f", app_id])
        return True
    return False

handle 

handle(cmd, core)

Open or close an app named in the command; return None if none matched.

Source code in src/plugins/apps.py 
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
def handle(cmd, core):
    """Open or close an app named in the command; return None if none matched."""
    all_apps = list(FLATPAK_APPS.keys()) + list(LOCAL_APPS.keys())

    for app in all_apps:
        if ("open" in cmd or "launch" in cmd) and app in cmd:
            if launch_app(app, core):
                core.speak(f"Opening {app}.")
            else:
                core.speak(f"{app} not installed.")
            return True

        if "close" in cmd and app in cmd:
            close_app(app, core)
            core.speak(f"Closing {app}.")
            return True

    return None  # Not handled

browser

plugins.browser

Browser Plugin - Qutebrowser voice control via IPC.

ensure_qutebrowser_config 

ensure_qutebrowser_config()

Append any missing required lines to qutebrowser's config.py.

Preserves everything else the user has written. Tolerates read-only configs (e.g. Nix Home-Manager symlinks into /nix/store): on a write failure we emit a polite note telling the user which lines to add themselves, rather than crashing startup.

Source code in src/plugins/browser.py 
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def ensure_qutebrowser_config():
    """Append any missing required lines to qutebrowser's config.py.

    Preserves everything else the user has written. Tolerates read-only configs (e.g.
    Nix Home-Manager symlinks into /nix/store): on a write failure we emit a polite note
    telling the user which lines to add themselves, rather than crashing startup.
    """
    cfg = Path.home() / ".config" / "qutebrowser" / "config.py"

    try:
        cfg.parent.mkdir(parents=True, exist_ok=True)
    except OSError as e:
        _note_missing_qb_lines(
            cfg,
            REQUIRED_QUTEBROWSER_LINES,
            reason=f"could not create {cfg.parent} ({e})",
        )
        return

    try:
        existing = cfg.read_text() if cfg.exists() else ""
    except OSError as e:
        _note_missing_qb_lines(
            cfg, REQUIRED_QUTEBROWSER_LINES, reason=f"could not read it ({e})"
        )
        return

    missing = [line for line in REQUIRED_QUTEBROWSER_LINES if line not in existing]
    if not missing:
        return

    separator = "" if not existing or existing.endswith("\n") else "\n"
    updated = existing + separator + "\n".join(missing) + "\n"

    try:
        cfg.write_text(updated)
    except OSError as e:
        _note_missing_qb_lines(cfg, missing, reason=f"is read-only ({e})")
        return

    verb = "wrote" if not existing else "updated"
    added = "; ".join(missing)
    logger.debug("%s %s (added: %s)", verb, cfg, added)

setup 

setup(c)

Store the core reference and write the required qutebrowser config.

Source code in src/plugins/browser.py 
216
217
218
219
220
def setup(c):
    """Store the core reference and write the required qutebrowser config."""
    global core
    core = c
    ensure_qutebrowser_config()

qb 

qb(command)

Send command to qutebrowser via IPC.

Source code in src/plugins/browser.py 
223
224
225
226
def qb(command):
    """Send command to qutebrowser via IPC."""
    logger.debug("  🌐 qutebrowser :%s", command)
    core.host_run(["qutebrowser", f":{command}"])

qb_open 

qb_open(url)

Open URL in qutebrowser.

Source code in src/plugins/browser.py 
229
230
231
def qb_open(url):
    """Open URL in qutebrowser."""
    core.host_run(["qutebrowser", url])

parse_hint_numbers 

parse_hint_numbers(cmd)

Extract hint numbers from spoken words.

Source code in src/plugins/browser.py 
234
235
236
237
238
239
def parse_hint_numbers(cmd):
    """Extract hint numbers from spoken words."""
    clean = re.sub(r"[.,!?\-]", " ", cmd.lower())
    words = clean.split()
    digits = [HINT_NUMBERS[word] for word in words if word in HINT_NUMBERS]
    return "".join(digits)

looks_like_hint 

looks_like_hint(cmd)

Check if command looks like a hint number (short, mostly digits/number words).

Source code in src/plugins/browser.py 
242
243
244
245
246
247
248
249
250
251
252
253
def looks_like_hint(cmd):
    """Check if command looks like a hint number (short, mostly digits/number words)."""
    clean = re.sub(r"[.,!?\-\s]", "", cmd.lower())
    # Must be short
    if len(clean) > 6:
        return False
    # Direct digits like "02", "92"
    if clean.replace("o", "0").isdigit():
        return True
    # Check if all words are number words
    words = cmd.lower().split()
    return len(words) <= 3 and all(w.strip(".,!?") in HINT_NUMBERS for w in words)

parse_hint_number 

parse_hint_number(cmd)

Parse spoken numbers into a hint string ('zero two' -> '02').

Source code in src/plugins/browser.py 
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def parse_hint_number(cmd):
    """Parse spoken numbers into a hint string ('zero two' -> '02')."""
    # Number word mappings
    NUM_WORDS = {
        "zero": "0",
        "oh": "0",
        "o": "0",
        "one": "1",
        "won": "1",
        "wan": "1",
        "two": "2",
        "to": "2",
        "too": "2",
        "tu": "2",
        "three": "3",
        "tree": "3",
        "free": "3",
        "four": "4",
        "for": "4",
        "fore": "4",
        "five": "5",
        "six": "6",
        "sex": "6",
        "seven": "7",
        "eight": "8",
        "ate": "8",
        "nine": "9",
        "nein": "9",
        # Tens
        "ten": "10",
        "eleven": "11",
        "twelve": "12",
        "thirteen": "13",
        "fourteen": "14",
        "fifteen": "15",
        "sixteen": "16",
        "seventeen": "17",
        "eighteen": "18",
        "nineteen": "19",
        "twenty": "2",
        "thirty": "3",
        "forty": "4",
        "fifty": "5",
        "sixty": "6",
        "seventy": "7",
        "eighty": "8",
        "ninety": "9",
    }

    result = []
    words = re.sub(r"[.,!?\-]", " ", cmd.lower()).split()

    for word in words:
        # Direct digit
        if word.isdigit():
            result.append(word)
        # Word to digit
        elif word in NUM_WORDS:
            result.append(NUM_WORDS[word])

    return "".join(result)

parse_spoken_url 

parse_spoken_url(spoken)

Convert spoken URL to actual URL.

'claude dot ai' -> 'https://claude.ai'.

Source code in src/plugins/browser.py 
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
def parse_spoken_url(spoken):
    """Convert spoken URL to actual URL.

    'claude dot ai' -> 'https://claude.ai'.
    """
    url = spoken.lower().strip()

    # Replace spoken elements
    url = url.replace(" dot ", ".")
    url = url.replace(" slash ", "/")
    url = url.replace(" colon ", ":")
    url = url.replace(" dash ", "-")
    url = url.replace(" hyphen ", "-")
    url = url.replace(" underscore ", "_")

    # Remove remaining spaces
    url = url.replace(" ", "")

    # Add https:// if no protocol
    if not url.startswith("http://") and not url.startswith("https://"):
        url = "https://" + url

    return url

listen_for_hint 

listen_for_hint(core)

Listen for hint number after showing hints.

Source code in src/plugins/browser.py 
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
def listen_for_hint(core):
    """Listen for hint number after showing hints."""
    logger.info("  🔢 Say hint number (e.g. 'zero two'), 'exit links' to cancel")

    # Small delay to let hints render
    time.sleep(0.3)

    # Clear audio buffer
    with contextlib.suppress(Exception):
        core.stream.read(core.stream.get_read_available(), exception_on_overflow=False)

    # Wait for speech
    first = core.wait_for_speech(timeout=10)
    if not first:
        logger.info("  ⏱ Timeout - hints cancelled")
        qb("mode-leave")
        logger.debug("  [listen_for_hint returning - timeout]")
        return

    audio = first + core.record_until_silence()
    cmd = core.transcribe(
        audio, prompt="zero one two three four five six seven eight nine"
    )

    if not cmd:
        logger.debug("  [listen_for_hint - no transcription, waiting again]")
        # Try one more time
        first = core.wait_for_speech(timeout=5)
        if first:
            audio = first + core.record_until_silence()
            cmd = core.transcribe(
                audio, prompt="zero one two three four five six seven eight nine"
            )
        if not cmd:
            qb("mode-leave")
            logger.debug("  [listen_for_hint returning - no transcription]")
            return

    cmd_lower = cmd.lower().strip(".,!? ")
    logger.debug("  ← %s", cmd_lower)

    # Cancel
    if cmd_lower in [
        "exit links",
        "exit link",
        "cancel",
        "nevermind",
        "stop",
        "close",
        "exit",
    ]:
        qb("mode-leave")
        logger.info("  ✗ Hints cancelled")
        logger.debug("  [listen_for_hint returning - cancelled]")
        return

    # Parse hint number
    hint = parse_hint_number(cmd_lower)

    if hint:
        logger.debug("  🔤 Hint: '%s' → '%s'", cmd_lower, hint)
        qb(f"hint-follow {hint}")
        # Wait for page to load, then clear any stuck state
        time.sleep(1.0)
        qb("fake-key <Escape>")
    else:
        # Try phonetic fallback
        hint = parse_hint_numbers(cmd_lower)
        if hint:
            logger.debug("  🔤 Phonetic: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            # Wait for page to load, then clear any stuck state
            time.sleep(1.0)
            qb("fake-key <Escape>")
        else:
            # Not a hint - might be a browser command, pass it through
            logger.debug("  ↪ Not a hint, trying as command: '%s'", cmd_lower)
            qb("mode-leave")
            handle_browser_command(cmd_lower, core)

    logger.debug("  [listen_for_hint returning - complete]")

handle 

handle(cmd, core)

Enter browser mode on a browser command; return None otherwise.

A matching command launches qutebrowser (if needed) and runs the continuous browser-mode loop; reserved global commands (sleep/quit) are passed through. Outside the explicit "open browser", navigation commands are only acted on when a browser is already running, so ambiguous words can't open one.

Source code in src/plugins/browser.py 
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
def handle(cmd, core):
    """Enter browser mode on a browser command; return None otherwise.

    A matching command launches qutebrowser (if needed) and runs the continuous
    browser-mode loop; reserved global commands (sleep/quit) are passed through. Outside
    the explicit "open browser", navigation commands are only acted on when a browser is
    already running, so ambiguous words can't open one.
    """
    cmd_lower = cmd.lower().strip(".,!? ")

    # Global sleep/quit commands belong to other plugins; if we matched them as
    # browser commands we would open qutebrowser instead. Let them through.
    if _is_reserved_global(cmd_lower):
        return None

    # --- Enter browser mode (explicit) ---
    if cmd_lower in ["browser", "browser mode", "open browser", "launch browser"]:
        core.host_run(["qutebrowser"], background=True)
        browser_mode(core)
        return True

    if not _qutebrowser_running(core):
        return None

    # --- Single browser commands → enters browser mode ---
    try:
        result = handle_browser_command(cmd_lower, core)
        if result:
            logger.info("  → Entering browser mode...")
            browser_mode(core)
            return True
    except Exception:
        logger.exception("  ! Browser error")
        return True

    return None

browser_mode 

browser_mode(core)

Continuous listening for browser commands.

Source code in src/plugins/browser.py 
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
def browser_mode(core):
    """Continuous listening for browser commands."""
    core.speak("Browser")
    logger.info("=== BROWSER MODE ACTIVE ===")
    logger.info("Say commands directly. 'exit browser' to leave.")

    while True:
        with contextlib.suppress(Exception):
            core.stream.read(
                core.stream.get_read_available(), exception_on_overflow=False
            )

        first = core.wait_for_speech(timeout=30)
        if not first:
            continue

        audio = first + core.record_until_silence()
        cmd = core.transcribe(audio)

        if not cmd:
            continue

        cmd_lower = cmd.lower().strip(".,!? ")
        logger.debug("  [browser] %s", cmd_lower)

        # Exit browser mode - require explicit phrase
        if cmd_lower in [
            "exit browser",
            "leave browser",
            "stop browser",
            "quit browser",
            "close browser",
        ]:
            logger.info("=== BROWSER MODE EXIT ===")
            return

        # Grid triggers - escape to grid mode
        grid_triggers = {"grid", "grit", "grip", "mouse", "pointer", "cursor"}
        if any(w in cmd_lower for w in grid_triggers):
            logger.info("=== BROWSER MODE EXIT → GRID ===")
            core.route_command(cmd_lower)
            return

        # Handle browser command
        if not handle_browser_command(cmd_lower, core):
            logger.debug("  ? Unknown: %s", cmd_lower)

handle_browser_command 

handle_browser_command(cmd_lower, core)

Execute a single in-browser command; None if it isn't recognised.

Source code in src/plugins/browser.py 
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
def handle_browser_command(cmd_lower, core):
    """Execute a single in-browser command; None if it isn't recognised."""
    # --- Hints ---
    if cmd_lower in [
        "numbers",
        "number",
        "hints",
        "hint",
        "show numbers",
        "show hints",
        "links",
        "link",
        "blanks",
        "blinks",
        "lynx",
        "lings",
        "lanes",
        "licks",
        "clicks",
    ]:
        qb("hint")
        listen_for_hint(core)
        return True

    if cmd_lower in [
        "numbers new",
        "number new",
        "hints new",
        "new numbers",
        "links new",
        "link new",
        "blanks new",
        "blinks new",
        "lynx new",
    ]:
        qb("hint links tab")
        listen_for_hint(core)
        return True

    # --- Navigation ---
    if cmd_lower in ["back", "go back", "previous page"]:
        qb("back")
        return True

    if cmd_lower in ["forward", "go forward", "next page"]:
        qb("forward")
        return True

    if cmd_lower in ["reload", "refresh", "reload page"]:
        qb("reload")
        return True

    if cmd_lower in ["stop", "stop loading"]:
        qb("stop")
        return True

    # --- Scrolling ---
    if cmd_lower in ["scroll down", "down"]:
        qb(f"jseval -q {SCROLL_DOWN_JS}")
        return True

    if cmd_lower in ["scroll up", "up"]:
        qb(f"jseval -q {SCROLL_UP_JS}")
        return True

    if "page" in cmd_lower and "down" in cmd_lower:
        qb(f"jseval -q {PAGE_DOWN_JS}")
        return True

    if "page" in cmd_lower and "up" in cmd_lower:
        qb(f"jseval -q {PAGE_UP_JS}")
        return True

    if cmd_lower in ["top", "go to top", "scroll to top"]:
        qb(f"jseval -q {SCROLL_TOP_JS}")
        return True

    if cmd_lower in ["bottom", "go to bottom", "scroll to bottom"]:
        qb(f"jseval -q {SCROLL_BOTTOM_JS}")
        return True

    # --- Tabs ---
    # Switch to specific tab by number
    if cmd_lower.startswith("tab "):
        tab_part = cmd_lower.replace("tab ", "").strip()
        tab_num = parse_hint_number(tab_part)
        if tab_num and tab_num.isdigit():
            qb(f"tab-focus {tab_num}")
            return True

    if cmd_lower in ["new tab", "open tab"]:
        qb("open -t about:blank")
        return True

    if cmd_lower in ["close tab", "close this tab"]:
        qb("tab-close")
        return True

    if cmd_lower in ["next tab", "tab right"]:
        qb("tab-next")
        return True

    if cmd_lower in ["last tab", "previous tab", "tab left"]:
        qb("tab-prev")
        return True

    if cmd_lower in ["undo tab", "restore tab", "reopen tab"]:
        qb("undo")
        return True

    # --- Find ---
    if cmd_lower.startswith("find "):
        query = cmd_lower.replace("find ", "", 1).strip()
        if query:
            qb(f"search {query}")
            return True

    if cmd_lower in ["find next", "next match"]:
        qb("search-next")
        return True

    if cmd_lower in ["find previous", "previous match"]:
        qb("search-prev")
        return True

    # --- Escape ---
    if cmd_lower in ["escape", "cancel", "nevermind"]:
        qb("mode-leave")
        return True

    # --- Bookmarks ---
    # Save current page as quickmark
    if (
        "bookmark this" in cmd_lower or "save this" in cmd_lower
    ) and " as " in cmd_lower:
        name = cmd_lower.split(" as ")[-1].strip()
        if name:
            core.speak(f"Saved as {name}.")
            qb(f"quickmark-save {name}")
            return True

    # Load quickmark (user-saved)
    if cmd_lower.startswith(("go to ", "open ")):
        target = cmd_lower.replace("go to ", "").replace("open ", "").strip()

        # Check predefined bookmarks first
        for site, url in BOOKMARKS.items():
            if site == target:
                core.speak(f"Opening {site}.")
                qb_open(url)
                return True

        # Try as spoken URL (contains "dot")
        if "dot" in target or "." in target:
            url = parse_spoken_url(target)
            core.speak(f"Opening {url}.")
            qb_open(url)
            return True

        # Don't catch generic "open X" - let other plugins handle it
        # Only use quickmark for explicit "go to X"
        if cmd_lower.startswith("go to "):
            qb(f"quickmark-load {target}")
            return True

        return None  # Let another plugin handle "open X"

    # --- Search ---
    if cmd_lower.startswith(("search ", "search for ")):
        query = cmd_lower.replace("search for ", "").replace("search ", "").strip()
        if query:
            url = f"https://duckduckgo.com/?q={query.replace(' ', '+')}"
            core.speak(f"Searching for {query}.")
            qb_open(url)
            return True

    # --- Phonetic hint selection (LAST - only if nothing else matched) ---
    # Only try hint parsing if it actually looks like a hint
    if looks_like_hint(cmd_lower):
        # Direct digit input (e.g., "02", "92", "0-2")
        stripped = re.sub(r"[^0-9a-z]", "", cmd_lower)
        if stripped.replace("o", "0").isdigit():
            hint = stripped.replace("o", "0")
            logger.debug("  🔤 Direct digits: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            return True

        # Try phonetic parsing
        hint = parse_hint_numbers(cmd_lower)
        if hint and hint.isdigit():
            logger.debug("  🔤 Phonetic parsed: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            return True

    return None

dictation

plugins.dictation

Dictation Plugin - Voice to text via AT-SPI.

ensure_gnome_accessibility 

ensure_gnome_accessibility()

Enable GNOME's toolkit-accessibility for the AT-SPI bridge.

Silently skipped if gsettings isn't on PATH or the schema isn't installed (i.e. user isn't on GNOME). Warns if it's present but the flip fails. Tells the user to re-login when newly enabled.

Source code in src/plugins/dictation.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def ensure_gnome_accessibility():
    """Enable GNOME's toolkit-accessibility for the AT-SPI bridge.

    Silently skipped if gsettings isn't on PATH or the schema isn't installed (i.e. user
    isn't on GNOME). Warns if it's present but the flip fails. Tells the user to
    re-login when newly enabled.
    """
    if shutil.which("gsettings") is None:
        return
    schema, key = "org.gnome.desktop.interface", "toolkit-accessibility"
    try:
        current = subprocess.run(
            ["gsettings", "get", schema, key],
            capture_output=True,
            text=True,
            check=False,
        )
        if current.returncode != 0:
            return  # schema missing — not really on GNOME
        if current.stdout.strip() == "true":
            return
        result = subprocess.run(
            ["gsettings", "set", schema, key, "true"],
            capture_output=True,
            text=True,
            check=False,
        )
        if result.returncode == 0:
            logger.info(
                "enabled GNOME toolkit-accessibility — "
                "log out and back in for dictation to work"
            )
        else:
            logger.warning(
                "could not enable GNOME toolkit-accessibility; dictation will not work"
            )
    except OSError:
        pass

setup 

setup(c)

Store the core reference and enable the GNOME accessibility bridge.

Source code in src/plugins/dictation.py 
73
74
75
76
77
78
79
80
81
82
83
def setup(c):
    """Store the core reference and enable the GNOME accessibility bridge."""
    global core
    core = c
    ensure_gnome_accessibility()
    # Offer dictation to the keyboard (silent) activation path too: core runs
    # this while the hotkey combo is held, with no wake word spoken.
    if hasattr(c, "register_push_to_talk"):
        c.register_push_to_talk(
            lambda should_continue: run_push_to_talk(c, should_continue)
        )

atspi_python 

atspi_python()

Path to the interpreter that runs the AT-SPI helper.

The helper needs PyGObject and the AT-SPI typelib, which the app's own venv usually lacks. EASYSPEAK_ATSPI_PYTHON lets the packaging point at an interpreter that has them (the Nix flake sets it); otherwise we fall back to the system python3, where distro packages like python3-gi typically live.

Source code in src/plugins/dictation.py 
 99
100
101
102
103
104
105
106
107
def atspi_python():
    """Path to the interpreter that runs the AT-SPI helper.

    The helper needs PyGObject and the AT-SPI typelib, which the app's own venv usually
    lacks. `EASYSPEAK_ATSPI_PYTHON` lets the packaging point at an interpreter that has
    them (the Nix flake sets it); otherwise we fall back to the system `python3`, where
    distro packages like `python3-gi` typically live.
    """
    return os.environ.get("EASYSPEAK_ATSPI_PYTHON") or "python3"

insert_text 

insert_text(text)

Insert text via AT-SPI.

Returns one of INSERTED, NO_FOCUS or BACKEND_ERROR so the caller can give feedback that matches the real cause instead of always blaming focus.

Source code in src/plugins/dictation.py 
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def insert_text(text):
    """Insert text via AT-SPI.

    Returns one of INSERTED, NO_FOCUS or BACKEND_ERROR so the caller can give feedback
    that matches the real cause instead of always blaming focus.
    """
    cmd = [atspi_python(), ATSPI_HELPER, text]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
    except OSError as exc:
        logger.warning("dictation backend could not start: %s", exc)
        return BACKEND_ERROR

    if "OK" in result.stdout:
        return INSERTED
    if "NO_BACKEND" in result.stdout or result.returncode != 0:
        logger.warning(
            "dictation backend unavailable (PyGObject/AT-SPI missing): %s",
            result.stderr.strip() or "no detail",
        )
        return BACKEND_ERROR
    return NO_FOCUS

format_text 

format_text(text)

Convert spoken punctuation to actual punctuation.

Source code in src/plugins/dictation.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def format_text(text):
    """Convert spoken punctuation to actual punctuation."""
    text = text.strip()

    # Strip ALL punctuation Whisper auto-adds; only explicit commands add it back
    text = re.sub(r"[.,!?;:]+", "", text)
    text = text.strip()

    # Punctuation replacements - order matters!
    replacements = [
        # Editing commands
        (r"\s*backspace\s*", "\b"),
        (r"\s*back space\s*", "\b"),
        (r"\s*delete\s*", "\b"),
        (r"\s*space\s*", " "),
        (r"\s*tab\s*", "\t"),
        # Sentence breaks - add space after
        (r"\s*,?\s*new sentence\s*", ". "),
        (r"\s*,?\s*next sentence\s*", ". "),
        (r"\s*,?\s*new paragraph\s*", "\n\n"),
        (r"\s*,?\s*next paragraph\s*", "\n\n"),
        (r"\s*,?\s*new para\s*", "\n\n"),
        (r"\s*,?\s*new line\s*", "\n"),
        (r"\s*,?\s*newline\s*", "\n"),
        (r"\s*,?\s*you line\s*", "\n"),
        (r"\s*,?\s*line break\s*", "\n"),
        (r"\s*,?\s*enter\s*", "\n"),
        # Punctuation - include common mishearings
        (r"\s*,?\s*comma\s*", ", "),
        (r"\s*,?\s*karma\s*", ", "),
        (r"\s*,?\s*kama\s*", ", "),
        (r"\s*,?\s*carma\s*", ", "),
        (r"\s*,?\s*calm a\s*", ", "),
        (r"\s*,?\s*calm him\s*", ", "),
        (r"\s*,?\s*calm up\s*", ", "),
        (r"\s*,?\s*come a\s*", ", "),
        (r"\s*,?\s*coma\s*", ", "),
        (r"\s*,?\s*calmer\s*", ", "),
        (r",\s*\.", ","),  # Fix comma followed by period
        (r"\s*,?\s*period\s*", ". "),
        (r"\s*,?\s*full stop\s*", ". "),
        (r"\s*,?\s*\.\s*\.+", "."),  # Multiple periods to one
        (r"\s*,?\s*question mark\s*", "? "),
        (r"\s*,?\s*exclamation mark\s*", "! "),
        (r"\s*,?\s*exclamation point\s*", "! "),
        (r"\s*,?\s*colon\s*", ": "),
        (r"\s*,?\s*semicolon\s*", "; "),
        (r"\s*,?\s*semi colon\s*", "; "),
        (r"\s*,?\s*dash\s*", " - "),
        (r"\s*,?\s*hyphen\s*", "-"),
        (r"\s*,?\s*apostrophe\s*", "'"),
        (r"\s*,?\s*open quote\s*", ' "'),
        (r"\s*,?\s*close quote\s*", '" '),
        (r"\s*,?\s*quote\s*", '"'),
        (r"\s*,?\s*open paren\s*", " ("),
        (r"\s*,?\s*close paren\s*", ") "),
        # Common words/symbols
        (r"\s*,?\s*at sign\s*", "@"),
        (r"\s*,?\s*ampersand\s*", "&"),
        (r"\s*,?\s*dollar sign\s*", "$"),
        (r"\s*,?\s*percent sign\s*", "%"),
        (r"\s*,?\s*percent\s*", "%"),
        (r"\s*,?\s*hashtag\s*", "#"),
        (r"\s*,?\s*hash\s*", "#"),
        (r"\s*,?\s*asterisk\s*", "*"),
        (r"\s*,?\s*star\s*", "*"),
        (r"\s*,?\s*underscore\s*", "_"),
        (r"\s*,?\s*slash\s*", "/"),
        (r"\s*,?\s*backslash\s*", "\\\\"),
    ]

    for pattern, replacement in replacements:
        text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)

    # Capitalize after sentence endings
    def capitalize_after(match):
        return match.group(1) + match.group(2).upper()

    text = re.sub(r"([.!?]\s+)([a-z])", capitalize_after, text)

    # Capitalize first letter
    if text:
        text = text[0].upper() + text[1:]

    # Clean up extra spaces
    text = re.sub(r" +", " ", text)
    text = re.sub(r" ([.,!?:;])", r"\1", text)

    # Fix double periods
    text = re.sub(r"\.+", ".", text)

    return text.strip()

run_push_to_talk 

run_push_to_talk(core, should_continue)

Dictate while the activation keys are held (the silent-activation path).

Mirrors the voice notes loop but is gated on should_continue — a predicate that is True while the keys remain held — instead of a spoken "stop notes": each utterance captured from core is formatted and inserted until the keys are released. The capture waits re-check should_continue so releasing stops dictation promptly.

Source code in src/plugins/dictation.py 
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
def run_push_to_talk(core, should_continue):
    """Dictate while the activation keys are held (the silent-activation path).

    Mirrors the voice `notes` loop but is gated on `should_continue` — a predicate that
    is True while the keys remain held — instead of a spoken "stop notes": each
    utterance captured from `core` is formatted and inserted until the keys are
    released. The capture waits re-check `should_continue` so releasing stops dictation
    promptly.
    """
    logger.info("🎙️ Push-to-talk dictation — release to end")
    while should_continue():
        core.flush_stream()
        first = core.wait_for_speech(
            timeout=PTT_LISTEN_TIMEOUT, should_continue=should_continue
        )
        if not first:
            continue
        audio = first + core.record_until_silence(should_continue=should_continue)
        text = core.transcribe(audio, prompt=DICTATION_PROMPT)
        if not text:
            continue
        if _dictate_utterance(core, text.strip().lower()):
            return

handle 

handle(cmd, core)

Enter dictation mode on a whole-word "note"/"notes"; return None otherwise.

Matching whole words (not substrings) keeps unrelated words like "notebook" or "noted" from triggering it. While in dictation mode it loops, transcribing speech and inserting it into the focused field until "stop notes" is heard.

Source code in src/plugins/dictation.py 
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
def handle(cmd, core):
    """Enter dictation mode on a whole-word "note"/"notes"; return None otherwise.

    Matching whole words (not substrings) keeps unrelated words like "notebook" or
    "noted" from triggering it. While in dictation mode it loops, transcribing speech
    and inserting it into the focused field until "stop notes" is heard.
    """
    words = cmd.split()
    if ("notes" in words or "note" in words) and "stop" not in words:
        core.speak("Dictation")

        logger.info("🎙️ Dictation mode - say 'stop notes' to end")

        while True:
            # Clear buffer and wait for speech
            core.stream.read(
                core.stream.get_read_available(), exception_on_overflow=False
            )
            first = core.wait_for_speech(timeout=30)

            if not first:
                logger.debug("   (waiting...)")
                continue

            audio = first + core.record_until_silence()
            text = core.transcribe(audio, prompt=DICTATION_PROMPT)

            if not text:
                continue

            text = text.strip().lower()
            logger.debug("   Raw: %s", text)

            # Check for exit - include mishearings
            if any(
                x in text
                for x in [
                    "stop notes",
                    "stop note",
                    "end notes",
                    "exit notes",
                    "stop nurts",
                    "stop nots",
                    "stop nuts",
                    "stopnotes",
                    "done notes",
                    "finish notes",
                    "close notes",
                    "closed notes",
                ]
            ):
                core.speak("Done")
                return True

            # Format and insert
            if _dictate_utterance(core, text):
                return True

    return None

mousegrid (00_mousegrid)

plugins.00_mousegrid

Mouse Grid Plugin - Voice-controlled mouse via GNOME Shell D-Bus extension.

Features: - Chained numbers: "3 7 5" zooms three times in one utterance - Repetition: "nudge up 5", "scroll down 3" - Drag support: "mark" to start, "drag" to end

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/00_mousegrid.py 
105
106
107
108
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

host_run 

host_run(cmd)

Run a command and capture its output (the plugin's own subprocess seam).

A missing executable returns a failed result (returncode 1) rather than raising, so callers treat an absent gdbus—e.g. on a headless host such as WSL with no GNOME session—as a plain D-Bus failure. Without this the atexit cleanup would raise FileNotFoundError on exit.

Source code in src/plugins/00_mousegrid.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def host_run(cmd):
    """Run a command and capture its output (the plugin's own subprocess seam).

    A missing executable returns a failed result (returncode 1) rather than
    raising, so callers treat an absent `gdbus`—e.g. on a headless host such as
    WSL with no GNOME session—as a plain D-Bus failure. Without this the atexit
    cleanup would raise FileNotFoundError on exit.
    """
    try:
        return subprocess.run(cmd, capture_output=True, text=True)
    except FileNotFoundError as exc:
        logger.debug("Command not found: %s", exc)
        return subprocess.CompletedProcess(
            cmd, returncode=1, stdout="", stderr=str(exc)
        )

dbus_call 

dbus_call(method, *args)

Call a method on the grid extension over D-Bus; True on success.

Source code in src/plugins/00_mousegrid.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def dbus_call(method, *args):
    """Call a method on the grid extension over D-Bus; True on success."""
    cmd = [
        "gdbus",
        "call",
        "--session",
        "--dest",
        "org.gnome.Shell",
        "--object-path",
        "/org/easyspeak/Desktop",
        "--method",
        f"org.easyspeak.Desktop.{method}",
    ]
    cmd.extend(str(a) for a in args)
    result = host_run(cmd)
    return result.returncode == 0

get_screen_size 

get_screen_size()

Get screen size from GNOME Shell extension (accurate for Wayland).

Source code in src/plugins/00_mousegrid.py 
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
def get_screen_size():
    """Get screen size from GNOME Shell extension (accurate for Wayland)."""
    result = host_run(
        [
            "gdbus",
            "call",
            "--session",
            "--dest",
            "org.gnome.Shell",
            "--object-path",
            "/org/easyspeak/Desktop",
            "--method",
            "org.easyspeak.Desktop.GetScreenSize",
        ]
    )

    if result.returncode == 0:
        # Parse output like "((1920, 1200),)"

        match = re.search(r"\((\d+),\s*(\d+)\)", result.stdout)
        if match:
            return int(match.group(1)), int(match.group(2))

    # Fallback to /sys/class/drm
    for card in [
        "card0-eDP-1",
        "card1-eDP-1",
        "card0-DP-1",
        "card1-DP-1",
        "card0-HDMI-A-1",
        "card1-HDMI-A-1",
    ]:
        result = host_run(["cat", f"/sys/class/drm/{card}/modes"])
        if result.returncode == 0 and result.stdout.strip():
            match = re.match(r"(\d+)x(\d+)", result.stdout.strip().split("\n")[0])
            if match:
                return int(match.group(1)), int(match.group(2))
    return 1920, 1080

cleanup 

cleanup()

Hide the grid on interpreter exit so no overlay is left on screen.

Source code in src/plugins/00_mousegrid.py 
186
187
188
def cleanup():
    """Hide the grid on interpreter exit so no overlay is left on screen."""
    dbus_call("Hide")

parse_number_sequence 

parse_number_sequence(text)

Extract ALL numbers from text as a sequence.

'3 7 5' -> [3, 7, 5].

Source code in src/plugins/00_mousegrid.py 
194
195
196
197
198
199
200
201
202
203
204
205
def parse_number_sequence(text):
    """Extract ALL numbers from text as a sequence.

    '3 7 5' -> [3, 7, 5].
    """
    # Replace word numbers with digits
    text_lower = text.lower()
    for word, num in WORD_TO_NUM.items():
        text_lower = re.sub(rf"\b{word}\b", str(num), text_lower)

    # Extract all single digits (grid zones are 1-9)
    return [int(char) for char in text_lower if char.isdigit() and char != "0"]

parse_count 

parse_count(text)

Extract a repeat count (for nudge/scroll).

Returns single number or 1.

Source code in src/plugins/00_mousegrid.py 
208
209
210
211
212
213
214
215
216
217
218
219
220
def parse_count(text):
    """Extract a repeat count (for nudge/scroll).

    Returns single number or 1.
    """
    text_lower = text.lower()
    for word, num in WORD_TO_NUM.items():
        text_lower = re.sub(rf"\b{word}\b", str(num), text_lower)

    match = re.search(r"\b(\d+)\b", text_lower)
    if match:
        return min(int(match.group(1)), 20)
    return 1

parse_direction 

parse_direction(text)

Return the direction (up/down/left/right) named in text, or None.

Source code in src/plugins/00_mousegrid.py 
223
224
225
226
227
228
229
def parse_direction(text):
    """Return the direction (up/down/left/right) named in text, or None."""
    for direction, words in DIRECTIONS.items():
        for word in words:
            if word in text:
                return direction
    return None

get_center 

get_center()

Return the (x, y) center of the current grid cell, or None if no grid.

Source code in src/plugins/00_mousegrid.py 
232
233
234
235
236
237
def get_center():
    """Return the (x, y) center of the current grid cell, or None if no grid."""
    if not grid_bounds:
        return None
    x, y, w, h = grid_bounds
    return x + w // 2, y + h // 2

show_grid 

show_grid()

Show the full-screen grid overlay (no-op if it is already active).

Source code in src/plugins/00_mousegrid.py 
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
def show_grid():
    """Show the full-screen grid overlay (no-op if it is already active)."""
    global grid_active, grid_bounds, screen_size

    if grid_active:
        return

    screen_size = get_screen_size()
    grid_bounds = (0, 0, screen_size[0], screen_size[1])

    if dbus_call("Show", screen_size[0], screen_size[1]):
        grid_active = True
        core.speak("Grid")
        logger.info("Grid shown (%sx%s)", screen_size[0], screen_size[1])
    else:
        logger.warning("Failed to show grid - is extension enabled?")

close_grid 

close_grid()

Hide the grid overlay and clear its bounds.

Source code in src/plugins/00_mousegrid.py 
261
262
263
264
265
266
def close_grid():
    """Hide the grid overlay and clear its bounds."""
    global grid_active, grid_bounds
    dbus_call("Hide")
    grid_active = False
    grid_bounds = None

update_grid 

update_grid(zone)

Zoom to a single zone.

Returns True if successful.

Source code in src/plugins/00_mousegrid.py 
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
def update_grid(zone):
    """Zoom to a single zone.

    Returns True if successful.
    """
    global grid_bounds

    if not grid_active or not grid_bounds:
        return False

    x, y, w, h = grid_bounds
    zone_w, zone_h = w // 3, h // 3

    zone_map = {
        1: (0, 0),
        2: (1, 0),
        3: (2, 0),  # top row
        4: (0, 1),
        5: (1, 1),
        6: (2, 1),  # middle row
        7: (0, 2),
        8: (1, 2),
        9: (2, 2),  # bottom row
    }

    if zone not in zone_map:
        return False

    col, row = zone_map[zone]
    new_x = x + col * zone_w
    new_y = y + row * zone_h

    # Only clamp if actually going off-screen (shouldn't happen, but safety)
    new_x = max(new_x, 0)
    new_y = max(new_y, 0)
    if new_x + zone_w > screen_size[0]:
        new_x = screen_size[0] - zone_w
    if new_y + zone_h > screen_size[1]:
        new_y = screen_size[1] - zone_h

    grid_bounds = (new_x, new_y, zone_w, zone_h)

    cx, cy = get_center()
    logger.debug("  → Zone %s: %sx%s center=(%s, %s)", zone, zone_w, zone_h, cx, cy)
    return dbus_call("Update", new_x, new_y, zone_w, zone_h)

process_zones 

process_zones(zones)

Process a sequence of zones.

'three seven five' zooms 3 times.

Source code in src/plugins/00_mousegrid.py 
316
317
318
319
320
321
322
def process_zones(zones):
    """Process a sequence of zones.

    'three seven five' zooms 3 times.
    """
    for zone in zones:
        update_grid(zone)

do_click 

do_click(click_type='click')

Click at the current cell center and close the grid; False if no grid.

Source code in src/plugins/00_mousegrid.py 
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
def do_click(click_type="click"):
    """Click at the current cell center and close the grid; False if no grid."""
    global grid_bounds, grid_active, last_bounds

    center = get_center()
    if not center:
        return False

    cx, cy = center
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None

    method_map = {
        "click": "Click",
        "double": "DoubleClick",
        "right": "RightClick",
        "middle": "MiddleClick",
    }

    return dbus_call(method_map.get(click_type, "Click"), cx, cy)

do_scroll 

do_scroll(direction, count=1)

Scroll count steps at the current cell center; False if no grid.

Source code in src/plugins/00_mousegrid.py 
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
def do_scroll(direction, count=1):
    """Scroll `count` steps at the current cell center; False if no grid."""
    global grid_bounds, grid_active, last_bounds

    center = get_center()
    if not center:
        return False

    cx, cy = center
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None

    logger.debug("  → Scroll %s x%s", direction, count)
    return dbus_call("Scroll", cx, cy, direction, count)

nudge_grid 

nudge_grid(direction, count=1)

Shift the current cell count steps in a direction; False if no grid.

Source code in src/plugins/00_mousegrid.py 
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def nudge_grid(direction, count=1):
    """Shift the current cell `count` steps in a direction; False if no grid."""
    global grid_bounds

    if not grid_bounds:
        return False

    x, y, w, h = grid_bounds
    amount = NUDGE_AMOUNT * count

    if direction == "up":
        y = max(0, y - amount)
    elif direction == "down":
        y = min(screen_size[1] - h, y + amount)
    elif direction == "left":
        x = max(0, x - amount)
    elif direction == "right":
        x = min(screen_size[0] - w, x + amount)

    grid_bounds = (x, y, w, h)
    cx, cy = get_center()
    logger.debug("  → Nudge %s x%s: center=(%s, %s)", direction, count, cx, cy)
    return dbus_call("Update", x, y, w, h)

start_drag 

start_drag()

Press at the current cell center and reset the grid to full screen.

Leaves a drag in progress so a later end_drag releases at the new target. False if no grid is active.

Source code in src/plugins/00_mousegrid.py 
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
def start_drag():
    """Press at the current cell center and reset the grid to full screen.

    Leaves a drag in progress so a later [`end_drag`][plugins.00_mousegrid.end_drag]
    releases at the new target. False if no grid is active.
    """
    global drag_start, grid_bounds
    center = get_center()
    if not center:
        return False
    drag_start = center
    logger.debug("  → Drag start: %s", center)
    dbus_call("StartDrag", center[0], center[1])

    # Reset grid to full screen so user can navigate to destination
    grid_bounds = (0, 0, screen_size[0], screen_size[1])
    dbus_call("Update", 0, 0, screen_size[0], screen_size[1])
    logger.debug("  → Grid reset - navigate to drop location")
    return True

end_drag 

end_drag()

Release a drag and close the grid.

Releases a drag started by start_drag. False if no drag is in progress or no grid is active.

Source code in src/plugins/00_mousegrid.py 
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
def end_drag():
    """Release a drag and close the grid.

    Releases a drag started by [`start_drag`][plugins.00_mousegrid.start_drag].
    False if no drag is in progress or no grid is active.
    """
    global drag_start, grid_bounds, grid_active, last_bounds
    if not drag_start:
        logger.debug("  → No drag in progress")
        return False
    center = get_center()
    if not center:
        return False
    logger.debug("  → Drag end: %s", center)
    dbus_call("EndDrag", center[0], center[1])
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None
    drag_start = None
    return True

handle 

handle(cmd, core)

Show or reopen the grid on a trigger word, then enter grid mode.

Returns None for commands that don't open the grid.

Source code in src/plugins/00_mousegrid.py 
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def handle(cmd, core):
    """Show or reopen the grid on a trigger word, then enter grid mode.

    Returns None for commands that don't open the grid.
    """
    global grid_active, grid_bounds, last_bounds, screen_size

    cmd_lower = cmd.lower().strip()

    # "Again" - reopen at last position
    if any(w in cmd_lower for w in ["again", "repeat", "reopen"]) and last_bounds:
        screen_size = get_screen_size()
        grid_bounds = last_bounds
        grid_active = True
        dbus_call("Show", screen_size[0], screen_size[1])
        dbus_call("Update", *last_bounds)
        core.speak("Grid")
        logger.info("Grid reopened at last position")
        listen_for_grid_commands(core)
        return True

    # Show grid
    if (
        any(w in cmd_lower for w in GRID_TRIGGERS)
        and "close" not in cmd_lower
        and "hide" not in cmd_lower
    ):
        show_grid()
        listen_for_grid_commands(core)
        return True

    return None

listen_for_grid_commands 

listen_for_grid_commands(core)

Continuous listening while grid active.

Source code in src/plugins/00_mousegrid.py 
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
def listen_for_grid_commands(core):
    """Continuous listening while grid active."""
    global grid_active, drag_start

    logger.info("Grid mode: say numbers (e.g. '3 7 5'), nudge, scroll, click, close")

    while grid_active:
        with contextlib.suppress(Exception):
            core.stream.read(
                core.stream.get_read_available(), exception_on_overflow=False
            )

        first = core.wait_for_speech(timeout=10)
        if not first:
            continue

        audio = first + core.record_until_silence()
        cmd = core.transcribe(
            audio,
            prompt=(
                "one two three four five six seven eight nine click double "
                "right scroll nudge up down left right close cancel mark drag"
            ),
        )
        if not cmd:
            continue

        cmd_lower = cmd.lower().strip()
        logger.debug("  ← %s", cmd_lower)

        # === Exit ===
        if any(
            w in cmd_lower
            for w in [
                "close",
                "cancel",
                "escape",
                "exit",
                "hide",
                "stop",
                "done",
                "quit",
            ]
        ):
            close_grid()
            logger.info("Grid closed")
            return

        # === Drag ===
        if "mark" in cmd_lower:
            start_drag()
            continue
        if "drag" in cmd_lower and drag_start:
            end_drag()
            return

        # === Scroll ===
        if "scroll" in cmd_lower:
            direction = parse_direction(cmd_lower)
            if direction:
                do_scroll(direction, parse_count(cmd_lower))
                return
            continue

        # === Clicks ===
        has_click = "click" in cmd_lower

        if "double" in cmd_lower:
            do_click("double")
            return
        if "middle" in cmd_lower:
            do_click("middle")
            return
        if has_click and any(
            w in cmd_lower for w in ["right", "write", "rite", "wright"]
        ):
            do_click("right")
            return
        if has_click:
            do_click("click")
            return

        # === Nudge ===
        direction = parse_direction(cmd_lower)
        if direction and not has_click:
            nudge_grid(direction, parse_count(cmd_lower))
            continue

        # === Zone numbers (chained) ===
        zones = parse_number_sequence(cmd_lower)
        if zones:
            logger.debug("  → Zones: %s", zones)
            process_zones(zones)
            continue

headtrack (00_eyetrack)

plugins.00_eyetrack

Head Tracking Plugin - Cursor control via SixDRepNet.

Uses a 6D rotation representation for smooth head pose estimation.

OneEuroFilter 

OneEuroFilter(freq=30.0, min_cutoff=1.0, beta=0.007, d_cutoff=1.0)

One-Euro filter for smoothing noisy signals.

Adaptive: smooth when the signal is still, responsive when it moves.

Initialise the filter's cutoff/responsiveness parameters.

Source code in src/plugins/00_eyetrack.py 
47
48
49
50
51
52
53
54
def __init__(self, freq=30.0, min_cutoff=1.0, beta=0.007, d_cutoff=1.0):
    """Initialise the filter's cutoff/responsiveness parameters."""
    self.freq = freq
    self.min_cutoff = min_cutoff
    self.beta = beta
    self.d_cutoff = d_cutoff
    self.x_prev = None
    self.dx_prev = 0.0

__call__ 

__call__(x)

Filter the next sample x and return the smoothed value.

Source code in src/plugins/00_eyetrack.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __call__(self, x):
    """Filter the next sample `x` and return the smoothed value."""
    if self.x_prev is None:
        self.x_prev = x
        return x

    # Derivative
    dx = (x - self.x_prev) * self.freq

    # Smooth derivative
    a_d = self._alpha(self.d_cutoff)
    dx_smooth = a_d * dx + (1 - a_d) * self.dx_prev
    self.dx_prev = dx_smooth

    # Adaptive cutoff based on speed
    cutoff = self.min_cutoff + self.beta * abs(dx_smooth)

    # Smooth value
    a = self._alpha(cutoff)
    x_smooth = a * x + (1 - a) * self.x_prev
    self.x_prev = x_smooth

    return x_smooth

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/00_eyetrack.py 
86
87
88
89
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

host_run 

host_run(cmd)

Run a command and capture its output (the plugin's own subprocess seam).

Source code in src/plugins/00_eyetrack.py 
92
93
94
def host_run(cmd):
    """Run a command and capture its output (the plugin's own subprocess seam)."""
    return subprocess.run(cmd, capture_output=True, text=True)

dbus_call 

dbus_call(method, *args)

Call GNOME Shell extension for cursor movement.

Source code in src/plugins/00_eyetrack.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
def dbus_call(method, *args):
    """Call GNOME Shell extension for cursor movement."""
    cmd = [
        "gdbus",
        "call",
        "--session",
        "--dest",
        "org.gnome.Shell",
        "--object-path",
        "/org/easyspeak/Desktop",
        "--method",
        f"org.easyspeak.Desktop.{method}",
    ]
    cmd.extend(str(a) for a in args)
    result = host_run(cmd)
    return result.returncode == 0

get_screen_size 

get_screen_size()

Get screen size from GNOME Shell extension.

Source code in src/plugins/00_eyetrack.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
def get_screen_size():
    """Get screen size from GNOME Shell extension."""
    result = host_run(
        [
            "gdbus",
            "call",
            "--session",
            "--dest",
            "org.gnome.Shell",
            "--object-path",
            "/org/easyspeak/Desktop",
            "--method",
            "org.easyspeak.Desktop.GetScreenSize",
        ]
    )

    if result.returncode == 0:
        import re

        match = re.search(r"\((\d+),\s*(\d+)\)", result.stdout)
        if match:
            return int(match.group(1)), int(match.group(2))
    return 1920, 1080

run_tracking 

run_tracking()

Main tracking loop.

Source code in src/plugins/00_eyetrack.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
def run_tracking():
    """Main tracking loop."""
    global tracking_active, stop_event, cursor_x, cursor_y, frozen

    import cv2
    from sixdrepnet import SixDRepNet

    logger.info("Loading SixDRepNet model...")
    model = SixDRepNet(gpu_id=-1)  # CPU mode

    SCREEN_W, SCREEN_H = get_screen_size()
    logger.debug("Screen: %sx%s", SCREEN_W, SCREEN_H)

    cap = cv2.VideoCapture(1)  # Integrated webcam
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
    cap.set(cv2.CAP_PROP_FPS, 30)

    if not cap.isOpened():
        logger.error("Cannot open webcam!")
        tracking_active = False
        return

    # One-euro filters for each axis
    # min_cutoff: lower = smoother (try 0.5-2.0)
    # beta: higher = more responsive to fast moves (try 0.001-0.01)
    filter_yaw = OneEuroFilter(freq=30.0, min_cutoff=0.8, beta=0.005)
    filter_pitch = OneEuroFilter(freq=30.0, min_cutoff=0.8, beta=0.005)

    # Rolling average buffers for stable output
    BUFFER_SIZE = 15
    yaw_buffer = []
    pitch_buffer = []

    # Velocity tracking for movement gating
    prev_yaw = None
    prev_pitch = None
    VELOCITY_THRESHOLD = 0.3  # Degrees per frame - ignore movement below this

    # Calibration center
    center_yaw = None
    center_pitch = None

    # Settings
    SENSITIVITY_X = 40.0  # Pixels per degree
    SENSITIVITY_Y = 25.0
    DEAD_ZONE = 3.0  # Degrees - ignore small movements

    # Initialize cursor position
    cursor_x = SCREEN_W // 2
    cursor_y = SCREEN_H // 2

    logger.info("Head tracking started. Look at center of screen...")
    logger.info("Say 'recalibrate' to reset center, 'stop tracking' to end")

    calibration_frames = 0
    calibration_yaw = 0.0
    calibration_pitch = 0.0

    while tracking_active and not stop_event.is_set():
        ret, frame = cap.read()
        if not ret:
            continue

        frame = cv2.flip(frame, 1)  # Mirror

        pitch, yaw, _roll = model.predict(frame)

        if len(pitch) == 0:
            continue

        raw_pitch = -pitch[0]  # Invert: look up = cursor up
        raw_yaw = -yaw[0]  # Invert: look right = cursor right

        # Auto-calibrate on first 10 frames
        if center_yaw is None:
            calibration_yaw += raw_yaw
            calibration_pitch += raw_pitch
            calibration_frames += 1

            if calibration_frames >= 10:
                center_yaw = calibration_yaw / 10
                center_pitch = calibration_pitch / 10
                logger.info("Calibrated: center=(%.1f, %.1f)", center_yaw, center_pitch)
            continue

        # Apply one-euro filter to raw values
        smooth_yaw = filter_yaw(raw_yaw)
        smooth_pitch = filter_pitch(raw_pitch)

        # Add to rolling average buffers
        yaw_buffer.append(smooth_yaw)
        pitch_buffer.append(smooth_pitch)
        if len(yaw_buffer) > BUFFER_SIZE:
            yaw_buffer.pop(0)  # pragma: no cover
            pitch_buffer.pop(0)  # pragma: no cover

        # Use averaged values
        avg_yaw = sum(yaw_buffer) / len(yaw_buffer)
        avg_pitch = sum(pitch_buffer) / len(pitch_buffer)

        # Velocity gating - ignore tiny movements (noise)
        if prev_yaw is not None:
            vel_yaw = abs(avg_yaw - prev_yaw)
            vel_pitch = abs(avg_pitch - prev_pitch)

            # If head is basically still, skip this frame
            if vel_yaw < VELOCITY_THRESHOLD and vel_pitch < VELOCITY_THRESHOLD:
                prev_yaw = avg_yaw
                prev_pitch = avg_pitch
                time.sleep(0.033)
                continue

        prev_yaw = avg_yaw
        prev_pitch = avg_pitch

        # Offset from center
        offset_yaw = avg_yaw - center_yaw
        offset_pitch = avg_pitch - center_pitch

        # Dead zone
        if abs(offset_yaw) < DEAD_ZONE:
            offset_yaw = 0
        else:
            offset_yaw = (
                offset_yaw - DEAD_ZONE if offset_yaw > 0 else offset_yaw + DEAD_ZONE
            )

        if abs(offset_pitch) < DEAD_ZONE:
            offset_pitch = 0
        else:
            offset_pitch = (
                offset_pitch - DEAD_ZONE
                if offset_pitch > 0
                else offset_pitch + DEAD_ZONE
            )

        # Exponential curve - small moves stay small, accelerates toward edges
        # Preserves sign, applies power curve to magnitude
        EXPO = 2.0  # Higher = more acceleration toward edges
        if offset_yaw != 0:
            sign = 1 if offset_yaw > 0 else -1
            offset_yaw = sign * (abs(offset_yaw) ** EXPO)
        if offset_pitch != 0:
            sign = 1 if offset_pitch > 0 else -1
            offset_pitch = sign * (abs(offset_pitch) ** EXPO)

        # Map to cursor position (reduced sensitivity since expo amplifies)
        target_x = SCREEN_W // 2 + (offset_yaw * SENSITIVITY_X / 3)
        target_y = SCREEN_H // 2 + (offset_pitch * SENSITIVITY_Y / 3)

        # Clamp target to screen FIRST
        target_x = max(0, min(SCREEN_W - 1, target_x))
        target_y = max(0, min(SCREEN_H - 1, target_y))

        # Edge dampening - slow down as we approach edges/corners
        EDGE_MARGIN = 150  # Pixels from edge where dampening kicks in
        MIN_DAMP = 0.3  # Minimum speed at very edge (0.3 = 30% speed)

        # Calculate distance from edges
        dist_left = target_x
        dist_right = SCREEN_W - 1 - target_x
        dist_top = target_y
        dist_bottom = SCREEN_H - 1 - target_y

        # Find closest edge distance
        closest_x = min(dist_left, dist_right)
        closest_y = min(dist_top, dist_bottom)

        # Dampening: 1.0 when far from edge, MIN_DAMP at edge
        damp_x = (
            1.0
            if closest_x > EDGE_MARGIN
            else MIN_DAMP + (1.0 - MIN_DAMP) * (closest_x / EDGE_MARGIN)
        )
        damp_y = (
            1.0
            if closest_y > EDGE_MARGIN
            else MIN_DAMP + (1.0 - MIN_DAMP) * (closest_y / EDGE_MARGIN)
        )

        # Apply dampening (blend toward target more slowly near edges)
        damp = min(damp_x, damp_y)  # Use tightest dampening for corners

        # Apply cursor position update (only if not frozen)
        if not frozen:
            # Apply edge dampening to movement
            target_x_damped = cursor_x + (target_x - cursor_x) * damp
            target_y_damped = cursor_y + (target_y - cursor_y) * damp

            cursor_x = target_x_damped
            cursor_y = target_y_damped

            # Strictly clamp to screen bounds
            cursor_x = max(5, min(SCREEN_W - 5, cursor_x))
            cursor_y = max(5, min(SCREEN_H - 5, cursor_y))

            # Move cursor via GNOME Shell extension
            dbus_call("MoveTo", int(cursor_x), int(cursor_y))

        # Throttle to reduce jitter
        time.sleep(0.033)  # ~30fps

    cap.release()
    tracking_active = False
    logger.info("Tracking stopped.")

start_tracking 

start_tracking()

Start the head-tracking thread; returns (started, message).

Source code in src/plugins/00_eyetrack.py 
348
349
350
351
352
353
354
355
356
357
358
359
360
def start_tracking():
    """Start the head-tracking thread; returns (started, message)."""
    global tracking_active, tracking_thread, stop_event, frozen

    if tracking_active:
        return False, "Already tracking"

    frozen = False
    stop_event.clear()
    tracking_active = True
    tracking_thread = threading.Thread(target=run_tracking, daemon=True)
    tracking_thread.start()
    return True, "Tracking"

stop_tracking 

stop_tracking()

Signal the tracking thread to stop; returns (stopped, message).

Source code in src/plugins/00_eyetrack.py 
363
364
365
366
367
368
369
def stop_tracking():
    """Signal the tracking thread to stop; returns (stopped, message)."""
    global tracking_active, stop_event
    stop_event.set()
    tracking_active = False
    time.sleep(0.2)
    return True, "Stopped"

recalibrate 

recalibrate()

Reset calibration - will auto-calibrate on next frames.

Source code in src/plugins/00_eyetrack.py 
372
373
374
375
376
377
378
379
380
def recalibrate():
    """Reset calibration - will auto-calibrate on next frames."""
    # center_yaw/pitch are in the tracking thread scope, so restart tracking
    if tracking_active:
        stop_tracking()
        time.sleep(0.3)
        start_tracking()
        return True, "Recalibrating"
    return False, "Not tracking"

handle 

handle(cmd, core)

Start/stop tracking or run a tracking command; None if not head-tracking.

Source code in src/plugins/00_eyetrack.py 
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
def handle(cmd, core):
    """Start/stop tracking or run a tracking command; None if not head-tracking."""
    global cursor_x, cursor_y
    cmd_lower = cmd.lower()

    # Start tracking
    if any(
        w in cmd_lower for w in ["start tracking", "begin tracking", "enable tracking"]
    ):
        success, msg = start_tracking()
        core.speak(msg)
        if success:
            listen_for_tracking_commands(core)
        return True

    # Stop tracking
    if any(
        w in cmd_lower
        for w in [
            "stop tracking",
            "end tracking",
            "close tracking",
            "quit tracking",
            "disable tracking",
            "tracking off",
            "stop track",
        ]
    ):
        success, msg = stop_tracking()
        core.speak(msg)
        return True

    # Recalibrate
    if "recalibrate" in cmd_lower or "calibrate" in cmd_lower:
        success, msg = recalibrate()
        core.speak(msg)
        return True

    return None

listen_for_tracking_commands 

listen_for_tracking_commands(core)

Continuous listening while tracking active.

Source code in src/plugins/00_eyetrack.py 
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
def listen_for_tracking_commands(core):
    """Continuous listening while tracking active."""
    global tracking_active, cursor_x, cursor_y, frozen

    SCREEN_W, SCREEN_H = get_screen_size()

    logger.info("Tracking mode: freeze, nudge, click, or stop tracking")

    while tracking_active:
        with contextlib.suppress(Exception):
            core.stream.read(
                core.stream.get_read_available(), exception_on_overflow=False
            )

        first = core.wait_for_speech(timeout=10)
        if not first:
            continue

        audio = first + core.record_until_silence()
        cmd = core.transcribe(
            audio,
            prompt=(
                "click double click right click freeze go nudge up down left "
                "right recalibrate stop tracking close cancel"
            ),
        )
        if not cmd:
            continue

        cmd_lower = cmd.lower().strip()
        logger.debug("  ← %s", cmd_lower)

        # Exit commands
        if any(
            w in cmd_lower
            for w in [
                "stop tracking",
                "end tracking",
                "close tracking",
                "stop",
                "cancel",
                "escape",
                "exit",
                "quit",
                "done",
            ]
        ):
            frozen = False
            stop_tracking()
            core.speak("Stopped")
            return

        # Freeze/Go
        if any(w in cmd_lower for w in ["freeze", "free", "rees", "frees"]):
            frozen = True
            logger.debug("  → Frozen at (%d, %d)", int(cursor_x), int(cursor_y))
            continue

        if cmd_lower in ["go", "go go", "unfreeze", "resume", "track"]:
            frozen = False
            logger.debug("  → Resumed")
            continue

        # Nudge (only when frozen)
        if frozen and any(
            w in cmd_lower for w in ["nudge", "move", "up", "down", "left", "right"]
        ):
            if any(w in cmd_lower for w in ["up", "north"]):
                cursor_y = max(0, cursor_y - NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["down", "south"]):
                cursor_y = min(SCREEN_H - 1, cursor_y + NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["left", "west"]):
                cursor_x = max(0, cursor_x - NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["right", "east", "write"]):
                cursor_x = min(SCREEN_W - 1, cursor_x + NUDGE_AMOUNT)
            dbus_call("MoveTo", int(cursor_x), int(cursor_y))
            continue

        # Recalibrate
        if "recalibrate" in cmd_lower or "calibrate" in cmd_lower:
            frozen = False
            recalibrate()
            core.speak("Recalibrating")
            continue

        # Click commands
        if "double" in cmd_lower:
            dbus_call("DoubleClick", int(cursor_x), int(cursor_y))
            continue

        if any(w in cmd_lower for w in ["right click", "right-click", "write click"]):
            dbus_call("RightClick", int(cursor_x), int(cursor_y))
            continue

        if any(
            w in cmd_lower
            for w in ["click", "select", "press", "kick", "quick", "flick"]
        ):
            dbus_call("Click", int(cursor_x), int(cursor_y))
            continue