Skip to content

Plugins API

Every plugin is a Python package (or a plain module) that follows a small contract rather than subclassing a base class. It is loaded if it exposes a NAME string and a handle(cmd, core) function; the optional setup(core) hook runs once at startup, COMMANDS/DESCRIPTION feed the help screen, and PRIORITY (default 50, lower first) sets its place in the routing order. A package's locale/<language>/ holds what it says, a gettext catalog, and what it listens for, a vocabulary.toml table; a plugin that has them binds _, its translator, and vocab, its vocabulary, at import (see Writing Plugins), and a plain module has neither. Click the diagram to enlarge it.

classDiagram
    class PluginContract {
        <<protocol>>
        +str NAME
        +str DESCRIPTION
        +int PRIORITY
        +list COMMANDS
        +translator _
        +Vocabulary vocab
        +setup(core)
        +handle(cmd, core)
    }
    class base {
        PRIORITY = 100
    }
    class mousegrid {
        PRIORITY = 1
    }
    class headtrack {
        PRIORITY = 0
    }
    class sleep
    class system
    class media
    class files
    class apps
    class browser
    class dictation

    PluginContract <|.. headtrack
    PluginContract <|.. mousegrid
    PluginContract <|.. apps
    PluginContract <|.. browser
    PluginContract <|.. dictation
    PluginContract <|.. files
    PluginContract <|.. media
    PluginContract <|.. sleep
    PluginContract <|.. system
    PluginContract <|.. base

    class Locale["locale/‹language›/"] {
        <<directory>>
        LC_MESSAGES/‹name›.po
        vocabulary.toml
    }
    Locale "1..*" --* "1" PluginContract
    style Locale fill:#80808026
    style headtrack stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style mousegrid stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style apps stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style browser stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style dictation stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style files stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style media stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style sleep stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style system stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5
    style base stroke:var(--md-mermaid-node-fg-color),stroke-dasharray:5

handle returns True when it consumed the command, False to signal the daemon to exit, or None to pass the command to the next plugin, in order of PRIORITY: the modes first, the base plugin last as the catch-all for help and exit.

base

plugins.base

Base Plugin - Help, About and Exit.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/base/__init__.py 
26
27
28
29
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

handle 

handle(cmd, core)

Handle the help, About and exit commands; return None to pass others through.

Returns False to signal the daemon to exit, True if the command was handled, or None when the command is for another plugin.

Source code in src/plugins/base/__init__.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def handle(cmd, core):
    """Handle the help, About and exit commands; return None to pass others through.

    Returns False to signal the daemon to exit, True if the command was handled, or
    None when the command is for another plugin.
    """
    cmd_lower = cmd.lower().strip()

    if vocab.says(cmd_lower, "require_wake_word"):
        core.require_wake_word = True
        core.speak(_("Modes now wait for the wake word."))
        return True

    if vocab.says(cmd_lower, "free_listening"):
        core.require_wake_word = False
        core.speak(_("Modes now take commands on their own."))
        return True

    # The exit word must end the command ("jarvis quit"), so "quit tracking"
    # is left to head tracking.
    if vocab.ends(cmd_lower, "exit"):
        core.speak(_("Goodbye."))
        return False  # Signal to exit

    if vocab.says(cmd_lower, "help"):
        show_help(core)
        return True

    if vocab.says(cmd_lower, "about"):
        about(cmd_lower, core)
        return True

    return None  # Not handled

about 

about(cmd, core)

Open the About dialog, or bring it to the front, or close it, as asked.

Source code in src/plugins/base/__init__.py 
67
68
69
70
71
72
73
74
75
76
def about(cmd, core):
    """Open the About dialog, or bring it to the front, or close it, as asked."""
    if vocab.says(cmd, "close"):
        if core.tray.close_about():
            core.speak(_("Closing the About dialog."))
        else:
            core.speak(_("The About dialog isn't open."))
    else:
        core.speak(_("Opening the About dialog."))
        core.tray.open_about()

show_help 

show_help(core)

List every plugin's commands on the terminal.

Printed rather than logged: it is the direct reply to an explicit "help" request (the spoken reply tells the user to read the terminal), so it must appear regardless of the configured log verbosity.

Source code in src/plugins/base/__init__.py 
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def show_help(core):
    """List every plugin's commands on the terminal.

    Printed rather than logged: it is the direct reply to an explicit "help"
    request (the spoken reply tells the user to read the terminal), so it must
    appear regardless of the configured log verbosity.
    """
    print("\n=== " + _("Available Commands") + " ===")  # noqa: T201
    for plugin in core.plugins:
        if hasattr(plugin, "COMMANDS"):
            print(f"\n{plugin.NAME}:")  # noqa: T201
            for cmd in plugin.COMMANDS:
                print(f"  • {cmd}")  # noqa: T201
    print()  # noqa: T201
    core.speak(_("Check the terminal for available commands."))

sleep

plugins.sleep

Sleep Plugin - Deactivate the assistant by voice.

"Go to sleep" (or "stop listening", "deactivate yourself") releases the microphone and stops wake-word detection until the user reactivates EasySpeak from the GNOME tray indicator or the Quick Settings toggle — whichever the extension is showing, the one way back since voice control is off until reactivated.

handle 

handle(cmd, core)

Deactivate the assistant on a sleep phrase; return None otherwise.

Source code in src/plugins/sleep/__init__.py 
23
24
25
26
27
28
29
30
def handle(cmd, core):
    """Deactivate the assistant on a sleep phrase; return None otherwise."""
    cmd_lower = cmd.lower().strip()
    if vocab.says(cmd_lower, "sleep"):
        core.speak(_("Voice control turned off."))
        core.deactivate()
        return True
    return None

system

plugins.system

System Plugin - Volume, brightness, do not disturb.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/system/__init__.py 
44
45
46
47
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

volume_up 

volume_up(core)

Raise the volume one step.

Source code in src/plugins/system/__init__.py 
60
61
62
63
64
def volume_up(core):
    """Raise the volume one step."""
    _media_key(
        core, KEY_VOLUME_UP, ["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "10%+"]
    )

volume_down 

volume_down(core)

Lower the volume one step.

Source code in src/plugins/system/__init__.py 
67
68
69
70
71
def volume_down(core):
    """Lower the volume one step."""
    _media_key(
        core, KEY_VOLUME_DOWN, ["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "10%-"]
    )

volume_mute 

volume_mute(core)

Toggle mute on the default audio sink.

Source code in src/plugins/system/__init__.py 
74
75
76
def volume_mute(core):
    """Toggle mute on the default audio sink."""
    _media_key(core, KEY_MUTE, ["wpctl", "set-mute", "@DEFAULT_AUDIO_SINK@", "toggle"])

volume_max 

volume_max(core)

Jump near the top (85%, not a blast); no media key does this.

Source code in src/plugins/system/__init__.py 
79
80
81
def volume_max(core):
    """Jump near the top (85%, not a blast); no media key does this."""
    core.host_run(["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "85%"])

volume_min 

volume_min(core)

Drop low (15%, still audible — not a mute), set directly like volume_max.

Source code in src/plugins/system/__init__.py 
84
85
86
def volume_min(core):
    """Drop low (15%, still audible — not a mute), set directly like volume_max."""
    core.host_run(["wpctl", "set-volume", "@DEFAULT_AUDIO_SINK@", "15%"])

brightness_up 

brightness_up(core)

Raise screen brightness one step.

Source code in src/plugins/system/__init__.py 
89
90
91
92
93
94
95
def brightness_up(core):
    """Raise screen brightness one step."""
    _media_key(
        core,
        KEY_BRIGHTNESS_UP,
        [*_POWER_SCREEN, "org.gnome.SettingsDaemon.Power.Screen.StepUp"],
    )

brightness_down 

brightness_down(core)

Lower screen brightness one step.

Source code in src/plugins/system/__init__.py 
 98
 99
100
101
102
103
104
def brightness_down(core):
    """Lower screen brightness one step."""
    _media_key(
        core,
        KEY_BRIGHTNESS_DOWN,
        [*_POWER_SCREEN, "org.gnome.SettingsDaemon.Power.Screen.StepDown"],
    )

dnd_on 

dnd_on(core)

Enable do-not-disturb by hiding notification banners.

Source code in src/plugins/system/__init__.py 
107
108
109
110
111
def dnd_on(core):
    """Enable do-not-disturb by hiding notification banners."""
    core.host_run(
        ["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"]
    )

dnd_off 

dnd_off(core)

Disable do-not-disturb by showing notification banners again.

Source code in src/plugins/system/__init__.py 
114
115
116
117
118
def dnd_off(core):
    """Disable do-not-disturb by showing notification banners again."""
    core.host_run(
        ["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "true"]
    )

handle 

handle(cmd, core)

Route a volume/brightness/DND command; return None if none matched.

Every word comes from the vocabulary table and is matched whole, so "silent" doesn't fire on "silently", "softer" on "softest", and "screenshot" isn't read as a screen command.

Source code in src/plugins/system/__init__.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
def handle(cmd, core):
    """Route a volume/brightness/DND command; return None if none matched.

    Every word comes from the vocabulary table and is matched whole, so "silent"
    doesn't fire on "silently", "softer" on "softest", and "screenshot" isn't read
    as a screen command.
    """
    says = lambda key: vocab.says(cmd, key)  # noqa: E731

    # Volume -- "louder"/"quieter" etc. work on their own, without "volume"/"sound".
    if says("very"):
        if says("loud"):
            volume_max(core)
            return True
        if says("quiet"):
            volume_min(core)
            return True
    louder, quieter = says("louder"), says("quieter")
    # No spoken feedback for volume/mute: GNOME's native OSD and chime already
    # acknowledge the change (and a spoken reply would be inaudible once muted).
    if says("volume") or louder or quieter:
        if says("up") or louder:
            volume_up(core)
            return True
        if says("down") or quieter:
            volume_down(core)
            return True
        if says("mute"):
            volume_mute(core)
            return True

    if says("mute"):
        volume_mute(core)
        return True

    if says("brightness"):
        if says("up") or says("brighter"):
            core.speak(_("Brighter."))
            brightness_up(core)
            return True
        if says("down") or says("dimmer"):
            core.speak(_("Dimmer."))
            brightness_down(core)
            return True

    dnd_named = says("dnd")
    if dnd_named or says("notifications"):
        turning_on, turning_off = says("on"), says("off")
        if not (turning_on or turning_off):
            return None  # a bare "notifications" says nothing about direction
        # Silencing notifications is do-not-disturb ON, so the notifications
        # vocabulary maps the other way round.
        if turning_on if dnd_named else turning_off:
            core.speak(_("Do not disturb on."))
            dnd_on(core)
        else:
            core.speak(_("Do not disturb off."))
            dnd_off(core)
        return True

    return None  # Not handled

media

plugins.media

Media Plugin - Playback controls via MPRIS.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/media/__init__.py 
37
38
39
40
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

get_media_players 

get_media_players(core)

Return the bus names of all running MPRIS media players.

Source code in src/plugins/media/__init__.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def get_media_players(core):
    """Return the bus names of all running MPRIS media players."""
    result = core.host_run(
        [
            "dbus-send",
            "--session",
            "--dest=org.freedesktop.DBus",
            "--type=method_call",
            "--print-reply",
            "/org/freedesktop/DBus",
            "org.freedesktop.DBus.ListNames",
        ]
    )
    return [
        line.split('"')[1]
        for line in result.stdout.split("\n")
        if "org.mpris.MediaPlayer2." in line
    ]

media_control 

media_control(action, core)

Send an MPRIS action (play/pause/next/previous) to every running player.

Returns False if no player is running or the action is unknown.

Source code in src/plugins/media/__init__.py 
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def media_control(action, core):
    """Send an MPRIS action (play/pause/next/previous) to every running player.

    Returns False if no player is running or the action is unknown.
    """
    players = get_media_players(core)
    if not players:
        return False

    method = {
        "play": "Play",
        "pause": "Pause",
        "next": "Next",
        "previous": "Previous",
    }.get(action)

    if not method:
        return False

    for player in players:
        core.host_run(
            [
                "dbus-send",
                "--session",
                "--type=method_call",
                f"--dest={player}",
                "/org/mpris/MediaPlayer2",
                f"org.mpris.MediaPlayer2.Player.{method}",
            ]
        )
    return True

handle 

handle(cmd, core)

Map a playback command to an MPRIS action; return None if not media.

The reply now follows the action rather than preceding it: this used to announce "Playing." and return handled even when no player was running, so a command that did nothing at all sounded like it had worked.

Source code in src/plugins/media/__init__.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def handle(cmd, core):
    """Map a playback command to an MPRIS action; return None if not media.

    The reply now follows the action rather than preceding it: this used to
    announce "Playing." and return handled even when no player was running, so a
    command that did nothing at all sounded like it had worked.
    """
    action = _action_for(cmd)
    if action is None:
        return None  # Not handled

    if not media_control(action, core):
        core.speak(_("No media player is running."))
        return True

    core.speak(FEEDBACK[action])
    return True

files

plugins.files

Files Plugin - Open folders in file manager.

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/files/__init__.py 
47
48
49
50
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

open_folder 

open_folder(path, core)

Open a folder in the user's default file manager; False if unavailable.

Delegates to xdg-open so whichever file manager the desktop is configured for is honoured, rather than hardcoding one. Launched with a clean environment (see core.host_run) so it doesn't inherit EasySpeak's library paths. Addresses like trash:/// pass through unchanged; a Path would collapse their slashes.

Source code in src/plugins/files/__init__.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def open_folder(path, core):
    """Open a folder in the user's default file manager; False if unavailable.

    Delegates to xdg-open so whichever file manager the desktop is configured
    for is honoured, rather than hardcoding one. Launched with a clean
    environment (see core.host_run) so it doesn't inherit EasySpeak's library
    paths. Addresses like `trash:///` pass through unchanged; a `Path` would
    collapse their slashes.
    """
    expanded = os.path.expanduser(path)  # noqa: PTH111
    if core.host_run(["which", "xdg-open"]).returncode != 0:
        return False
    core.host_run(["xdg-open", expanded], background=True, clean_env=True)
    return True

default_file_manager 

default_file_manager(core)

Return the process name of the configured default file manager, or None.

The same manager xdg-open opens folders with: xdg-mime names its desktop entry, and the entry's Exec line names the program, which is what pkill matches. A Flatpak's Exec runs flatpak, so its app id is taken instead, which is how the process list shows it.

Source code in src/plugins/files/__init__.py 
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def default_file_manager(core):
    """Return the process name of the configured default file manager, or None.

    The same manager `xdg-open` opens folders with: `xdg-mime` names its desktop
    entry, and the entry's Exec line names the program, which is what `pkill`
    matches. A Flatpak's Exec runs `flatpak`, so its app id is taken instead,
    which is how the process list shows it.
    """
    query = core.host_run(["xdg-mime", "query", "default", "inode/directory"])
    entry = query.stdout.strip() if query.returncode == 0 else ""
    if not entry:
        return None
    data_home = os.environ.get("XDG_DATA_HOME") or str(
        Path("~/.local/share").expanduser()
    )
    data_dirs = os.environ.get("XDG_DATA_DIRS") or "/usr/local/share:/usr/share"
    for directory in [data_home, *data_dirs.split(":")]:
        desktop = Path(directory) / "applications" / entry
        if not desktop.is_file():
            continue
        for line in desktop.read_text(encoding="utf-8", errors="replace").splitlines():
            if line.startswith("Exec="):
                words = line[len("Exec=") :].split()
                if not words:
                    return None
                program = Path(words[0]).name
                if program != "flatpak":
                    return program
                return next(
                    (w for w in words[1:] if "." in w and not w.startswith("-")), None
                )
    return None

close_file_manager 

close_file_manager(core)

Close the configured default file manager; False if none can be named.

Source code in src/plugins/files/__init__.py 
103
104
105
106
107
108
109
def close_file_manager(core):
    """Close the configured default file manager; False if none can be named."""
    program = default_file_manager(core)
    if program is None:
        return False
    core.host_run(["pkill", "-f", program])
    return True

handle 

handle(cmd, core)

Open a named folder or the file manager, or close the file manager.

Closing a folder ("close downloads") closes the file manager too, all of its windows: which one shows the folder can't be told reliably. None if neither matched.

Source code in src/plugins/files/__init__.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def handle(cmd, core):
    """Open a named folder or the file manager, or close the file manager.

    Closing a folder ("close downloads") closes the file manager too, all of its
    windows: which one shows the folder can't be told reliably. None if neither
    matched.
    """
    folder = vocab.which(cmd, "folders")
    if vocab.says(cmd, "close") and (vocab.says(cmd, "file_manager") or folder):
        if close_file_manager(core):
            core.speak(_("Closing the file manager."))
        else:
            core.speak(_("No file manager found."))
        return True

    if not vocab.says(cmd, "open"):
        return None

    if folder in FOLDERS:
        _open(FOLDERS[folder], folder, core)
        return True

    if vocab.says(cmd, "file_manager"):
        _open("~", "files", core)
        return True

    return None  # Not handled

apps

plugins.apps

Apps Plugin - Launch and close applications.

setup 

setup(c)

Store the core reference and resolve the user's default terminal.

Source code in src/plugins/apps/__init__.py 
119
120
121
122
123
def setup(c):
    """Store the core reference and resolve the user's default terminal."""
    global core
    core = c
    _register_terminal(c)

on_kde 

on_kde()

Whether the session is KDE Plasma.

Source code in src/plugins/apps/__init__.py 
190
191
192
def on_kde():
    """Whether the session is KDE Plasma."""
    return "KDE" in os.environ.get("XDG_CURRENT_DESKTOP", "").split(":")

find_app 

find_app(name, core)

Find an installed app - returns (type, id), or (None, None) if it's missing.

Source code in src/plugins/apps/__init__.py 
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def find_app(name, core):
    """Find an installed app - returns (type, id), or (None, None) if it's missing."""
    if on_kde():
        kde_app = next((app for app in KDE_APPS.get(name, []) if _has(app, core)), None)
        if kde_app:
            return ("local", kde_app)

    if name in FLATPAK_APPS and _has("flatpak", core):
        result = core.host_run(["flatpak", "info", FLATPAK_APPS[name]])
        if result.returncode == 0:
            return ("flatpak", FLATPAK_APPS[name])

    if name in LOCAL_APPS and _has(LOCAL_APPS[name], core):
        return ("local", LOCAL_APPS[name])

    if _has(name, core):
        return ("local", name)

    return (None, None)

launch_app 

launch_app(name, core)

Launch the named app (flatpak or local binary); False if not found.

Raises OSError if the app is there but can't be started.

Source code in src/plugins/apps/__init__.py 
216
217
218
219
220
221
222
223
224
225
226
227
228
def launch_app(name, core):
    """Launch the named app (flatpak or local binary); False if not found.

    Raises OSError if the app is there but can't be started.
    """
    app_type, app_id = find_app(name, core)
    if app_type == "flatpak":
        core.host_run(["flatpak", "run", app_id], background=True, clean_env=True)
        return True
    if app_type == "local":
        core.host_run([app_id], background=True, clean_env=True)
        return True
    return False

close_app 

close_app(name, core)

Close the named app (flatpak kill or pkill); False if not found.

Source code in src/plugins/apps/__init__.py 
231
232
233
234
235
236
237
238
239
240
def close_app(name, core):
    """Close the named app (flatpak kill or pkill); False if not found."""
    app_type, app_id = find_app(name, core)
    if app_type == "flatpak":
        core.host_run(["flatpak", "kill", app_id])
        return True
    if app_type == "local":
        core.host_run(["pkill", "-f", app_id])
        return True
    return False

handle 

handle(cmd, core)

Open or close an app named in the command; return None if none matched.

Source code in src/plugins/apps/__init__.py 
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
def handle(cmd, core):
    """Open or close an app named in the command; return None if none matched."""
    app = vocab.which(cmd, "apps")
    if app is None or (app not in FLATPAK_APPS and app not in LOCAL_APPS):
        return None

    if vocab.says(cmd, "open"):
        try:
            launched = launch_app(app, core)
        except OSError as exc:
            logger.warning("Couldn't start %s: %s", app, exc)
            core.speak(_("Sorry, I couldn't start {app}.").format(app=app))
            return True

        if launched:
            core.speak(_("Opening {app}.").format(app=app))
        else:
            core.speak(_("{app} not installed.").format(app=app))
        return True

    if vocab.says(cmd, "close"):
        close_app(app, core)
        core.speak(_("Closing {app}.").format(app=app))
        return True

    return None  # Not handled

browser

plugins.browser

Browser Plugin - Qutebrowser voice control via IPC.

adblock_method 

adblock_method()

Return the strongest ad-blocking method qutebrowser can actually run.

Ad overlays and newsletter interstitials are numbered like anything else, so they take hint numbers away from the page underneath and a spoken hint lands on a popup instead of the link the user meant. "both" adds Brave's ABP engine, which is what catches those overlays -- but it needs the adblock module, and without it qutebrowser complains on every single page load, which is worse than the ads. Host blocking is built in and needs nothing.

The module has to be importable by the interpreter that runs qutebrowser, which is often not the python3 on PATH: a virtualenv, a distro package, a Flatpak or a Nix build each ship their own. So this asks qutebrowser itself rather than probing a stray interpreter, which answered correctly only when the two happened to coincide. --version lists each optional module as adblock: <version> when present and adblock: no when not, so a leading digit means "both"; installing the module later is still picked up on the next start. That call imports Qt and so is slow rather than instant, so the 15-second timeout is a ceiling for a wedged process, not an expected wait -- past it, host blocking (which always works) is assumed.

Source code in src/plugins/browser/__init__.py 
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
def adblock_method():
    """Return the strongest ad-blocking method qutebrowser can actually run.

    Ad overlays and newsletter interstitials are numbered like anything else, so
    they take hint numbers away from the page underneath and a spoken hint lands
    on a popup instead of the link the user meant. "both" adds Brave's ABP engine,
    which is what catches those overlays -- but it needs the `adblock` module, and
    without it qutebrowser complains on every single page load, which is worse
    than the ads. Host blocking is built in and needs nothing.

    The module has to be importable by the interpreter that *runs* qutebrowser,
    which is often not the `python3` on `PATH`: a virtualenv, a distro package, a
    Flatpak or a Nix build each ship their own. So this asks qutebrowser itself
    rather than probing a stray interpreter, which answered correctly only when
    the two happened to coincide. `--version` lists each optional module as
    `adblock: <version>` when present and `adblock: no` when not, so a leading
    digit means "both"; installing the module later is still picked up on the
    next start. That call imports Qt and so is slow rather than instant, so the
    15-second timeout is a ceiling for a wedged process, not an expected wait --
    past it, host blocking (which always works) is assumed.
    """
    try:
        version = subprocess.run(
            ["qutebrowser", "--version"],
            capture_output=True,
            text=True,
            check=False,
            timeout=15,  # seconds
        )
    except (OSError, subprocess.TimeoutExpired):
        return "hosts"  # no qutebrowser, or it hung: host blocking always works
    available = re.search(r"^\s*adblock:\s*\d", version.stdout, re.MULTILINE)
    return "both" if available else "hosts"

required_qutebrowser_lines 

required_qutebrowser_lines()

The config lines EasySpeak needs, suited to what's installed.

Source code in src/plugins/browser/__init__.py 
220
221
222
223
224
225
def required_qutebrowser_lines():
    """The config lines EasySpeak needs, suited to what's installed."""
    return [
        *REQUIRED_QUTEBROWSER_LINES,
        f"c.content.blocking.method = '{adblock_method()}'",
    ]

set_config_line 

set_config_line(line, *, wanted)

Add or remove one line in qutebrowser's config.py, keyed on its setting.

Returns None when the config can't be written, so the caller can say why.

Source code in src/plugins/browser/__init__.py 
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
def set_config_line(line, *, wanted):
    """Add or remove one line in qutebrowser's config.py, keyed on its setting.

    Returns None when the config can't be written, so the caller can say why.
    """
    cfg = Path.home() / ".config" / "qutebrowser" / "config.py"
    try:
        cfg.parent.mkdir(parents=True, exist_ok=True)
        existing = cfg.read_text() if cfg.exists() else ""
        kept = [
            text
            for text in existing.splitlines()
            if _setting_name(text) != _setting_name(line)
        ]
        if wanted:
            kept.append(line)
        cfg.write_text("\n".join(kept) + "\n")
    except OSError as exc:
        logger.warning("Could not write %s: %s", cfg, exc)
        return None
    return wanted

ensure_qutebrowser_config 

ensure_qutebrowser_config()

Bring qutebrowser's config.py in line with what EasySpeak needs.

Preserves everything else the user has written. Tolerates read-only configs (e.g. Nix Home-Manager symlinks into /nix/store): on a write failure we emit a polite note telling the user which lines to add themselves, rather than crashing startup.

Source code in src/plugins/browser/__init__.py 
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
def ensure_qutebrowser_config():
    """Bring qutebrowser's config.py in line with what EasySpeak needs.

    Preserves everything else the user has written. Tolerates read-only configs (e.g.
    Nix Home-Manager symlinks into /nix/store): on a write failure we emit a polite note
    telling the user which lines to add themselves, rather than crashing startup.
    """
    cfg = Path.home() / ".config" / "qutebrowser" / "config.py"

    try:
        cfg.parent.mkdir(parents=True, exist_ok=True)
    except OSError as e:
        _note_missing_qb_lines(
            cfg,
            required_qutebrowser_lines(),
            reason=f"could not create {cfg.parent} ({e})",
        )
        return

    try:
        existing = cfg.read_text() if cfg.exists() else ""
    except OSError as e:
        _note_missing_qb_lines(
            cfg, required_qutebrowser_lines(), reason=f"could not read it ({e})"
        )
        return

    wanted = required_qutebrowser_lines()
    lines = existing.splitlines()
    changed = []

    for line in wanted:
        if line in lines:
            continue
        setting = _setting_name(line)
        index = next(
            (
                i
                for i, existing_line in enumerate(lines)
                if _setting_name(existing_line.strip()) == setting
            ),
            None,
        )
        if setting and index is not None:
            lines[index] = line
        else:
            lines.append(line)
        changed.append(line)

    if not changed:
        return

    try:
        cfg.write_text("\n".join(lines) + "\n")
    except OSError as e:
        _note_missing_qb_lines(cfg, changed, reason=f"is read-only ({e})")
        return

    verb = "wrote" if not existing else "updated"
    logger.debug("%s %s (%s)", verb, cfg, "; ".join(changed))

setup 

setup(c)

Store the core reference and write the required qutebrowser config.

Source code in src/plugins/browser/__init__.py 
350
351
352
353
354
355
356
def setup(c):
    """Store the core reference and write the required qutebrowser config."""
    global core
    core = c
    c.browser_page_js_stale = False
    c.in_browser_mode = False
    ensure_qutebrowser_config()

qb 

qb(command)

Send command to qutebrowser via IPC.

Source code in src/plugins/browser/__init__.py 
359
360
361
362
def qb(command):
    """Send command to qutebrowser via IPC."""
    logger.debug("  🌐 qutebrowser :%s", command)
    core.host_run(["qutebrowser", f":{command}"])

qb_open 

qb_open(url)

Open URL in qutebrowser.

Source code in src/plugins/browser/__init__.py 
365
366
367
def qb_open(url):
    """Open URL in qutebrowser."""
    core.host_run(["qutebrowser", url])

parse_hint_numbers 

parse_hint_numbers(cmd)

Extract hint numbers from spoken words.

Source code in src/plugins/browser/__init__.py 
370
371
372
373
374
375
def parse_hint_numbers(cmd):
    """Extract hint numbers from spoken words."""
    clean = re.sub(r"[.,!?\-]", " ", cmd.lower())
    words = clean.split()
    digits = [HINT_NUMBERS[word] for word in words if word in HINT_NUMBERS]
    return "".join(digits)

looks_like_hint 

looks_like_hint(cmd)

Check if command looks like a hint number (short, mostly digits/number words).

Source code in src/plugins/browser/__init__.py 
378
379
380
381
382
383
384
385
386
387
388
389
def looks_like_hint(cmd):
    """Check if command looks like a hint number (short, mostly digits/number words)."""
    clean = re.sub(r"[.,!?\-\s]", "", cmd.lower())
    # Must be short
    if len(clean) > 6:
        return False
    # Direct digits like "02", "92"
    if clean.replace("o", "0").isdigit():
        return True
    # Check if all words are number words
    words = cmd.lower().split()
    return len(words) <= 3 and all(w.strip(".,!?") in HINT_NUMBERS for w in words)

parse_hint_number 

parse_hint_number(cmd)

Parse spoken numbers into a hint string ('zero two' -> '02').

Source code in src/plugins/browser/__init__.py 
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
def parse_hint_number(cmd):
    """Parse spoken numbers into a hint string ('zero two' -> '02')."""
    # Number word mappings
    NUM_WORDS = {
        "zero": "0",
        "oh": "0",
        "o": "0",
        "one": "1",
        "won": "1",
        "wan": "1",
        "two": "2",
        "to": "2",
        "too": "2",
        "tu": "2",
        "three": "3",
        "tree": "3",
        "free": "3",
        "third": "3",
        "thee": "3",
        "four": "4",
        "for": "4",
        "fore": "4",
        "ford": "4",
        "forth": "4",
        "fourth": "4",
        "far": "4",
        "five": "5",
        "six": "6",
        "sex": "6",
        "seven": "7",
        "eight": "8",
        "ate": "8",
        "eighth": "8",
        "hate": "8",
        "nine": "9",
        "nein": "9",
        "ninth": "9",
        "mine": "9",
        # Tens
        "ten": "10",
        "eleven": "11",
        "twelve": "12",
        "thirteen": "13",
        "fourteen": "14",
        "fifteen": "15",
        "sixteen": "16",
        "seventeen": "17",
        "eighteen": "18",
        "nineteen": "19",
        "twenty": "2",
        "thirty": "3",
        "forty": "4",
        "fifty": "5",
        "sixty": "6",
        "seventy": "7",
        "eighty": "8",
        "ninety": "9",
    }

    result = []
    words = re.sub(r"[.,!?\-]", " ", cmd.lower()).split()

    for word in words:
        # Direct digit
        if word.isdigit():
            result.append(word)
        # Word to digit
        elif word in NUM_WORDS:
            result.append(NUM_WORDS[word])

    return "".join(result)

parse_tab_number 

parse_tab_number(cmd_lower)

The tab number a command names, or None if it doesn't name one.

Source code in src/plugins/browser/__init__.py 
479
480
481
482
483
484
485
486
487
def parse_tab_number(cmd_lower):
    """The tab number a command names, or None if it doesn't name one."""
    for prefix in TAB_PREFIXES:
        if not cmd_lower.startswith(prefix + " "):
            continue
        number = parse_hint_number(cmd_lower[len(prefix) :].strip())
        if number and number.isdigit():
            return number
    return None

parse_spoken_url 

parse_spoken_url(spoken)

Convert spoken URL to actual URL.

'claude dot ai' -> 'https://claude.ai'.

Source code in src/plugins/browser/__init__.py 
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
def parse_spoken_url(spoken):
    """Convert spoken URL to actual URL.

    'claude dot ai' -> 'https://claude.ai'.
    """
    url = spoken.lower().strip()

    # Replace spoken elements
    url = url.replace(" dot ", ".")
    url = url.replace(" slash ", "/")
    url = url.replace(" colon ", ":")
    url = url.replace(" dash ", "-")
    url = url.replace(" hyphen ", "-")
    url = url.replace(" underscore ", "_")

    # Remove remaining spaces
    url = url.replace(" ", "")

    # Add https:// if no protocol
    if not url.startswith("http://") and not url.startswith("https://"):
        url = "https://" + url

    return url

show_hints 

show_hints(core)

Show the numbered hints, reloading first if history navigation broke them.

Source code in src/plugins/browser/__init__.py 
541
542
543
544
def show_hints(core):
    """Show the numbered hints, reloading first if history navigation broke them."""
    _reload_if_page_js_is_stale(core)
    qb("hint")

scroll_page 

scroll_page(js, core)

Scroll the page, reloading first if history navigation broke its scripts.

Source code in src/plugins/browser/__init__.py 
547
548
549
550
def scroll_page(js, core):
    """Scroll the page, reloading first if history navigation broke its scripts."""
    _reload_if_page_js_is_stale(core)
    qb(f"jseval -q {js}")

listen_for_hint 

listen_for_hint(core, retries_left=3)

Listen for a hint number after showing hints.

retries_left bounds how many times a repeated "numbers" can keep the listener open, so a run of mishearings can't recurse without end.

Source code in src/plugins/browser/__init__.py 
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
def listen_for_hint(core, retries_left=3):
    """Listen for a hint number after showing hints.

    `retries_left` bounds how many times a repeated "numbers" can keep the
    listener open, so a run of mishearings can't recurse without end.
    """
    logger.info("  🔢 Say hint number (e.g. 'zero two'), 'exit links' to cancel")
    core.speak(_("Ready"))

    # Small delay to let hints render
    time.sleep(0.3)

    core.speech.drain()
    core.flush_stream()

    # Wait for speech
    first = core.wait_for_speech(timeout=10)
    if not first:
        if retries_left > 0:
            logger.debug("  ↻ No hints appeared; asking again")
            time.sleep(HINT_RETRY_DELAY)
            qb("fake-key <Escape>")
            show_hints(core)
            listen_for_hint(core, retries_left - 1)
            return
        logger.info("  ⏱ Timeout - hints cancelled")
        core.speak(_("Hints closed"))
        qb("fake-key <Escape>")
        logger.debug("  [listen_for_hint returning - timeout]")
        return

    audio = first + core.record_until_silence()
    cmd = core.transcribe(
        audio, prompt="zero one two three four five six seven eight nine"
    )

    if not cmd:
        logger.debug("  [listen_for_hint - no transcription, waiting again]")
        # Try one more time
        first = core.wait_for_speech(timeout=5)
        if first:
            audio = first + core.record_until_silence()
            cmd = core.transcribe(
                audio, prompt="zero one two three four five six seven eight nine"
            )
        if not cmd:
            qb("fake-key <Escape>")
            logger.debug("  [listen_for_hint returning - no transcription]")
            return

    cmd_lower = cmd.lower().strip(".,!? ")
    logger.debug("  ← %s", cmd_lower)

    if cmd_lower in HINT_TRIGGERS:
        if retries_left <= 0:
            logger.info("  ✗ Hints aren't appearing on this page")
            core.speak(_("No hints on this page"))
            qb("fake-key <Escape>")
            return
        logger.debug("  ↻ Showing hints again")
        qb("fake-key <Escape>")
        show_hints(core)
        listen_for_hint(core, retries_left - 1)
        return

    # Cancel
    if cmd_lower in [
        "exit links",
        "exit link",
        "cancel",
        "nevermind",
        "stop",
        "close",
        "exit",
    ]:
        qb("fake-key <Escape>")
        logger.info("  ✗ Hints cancelled")
        core.speak(_("Hints closed"))
        logger.debug("  [listen_for_hint returning - cancelled]")
        return

    # Parse hint number
    hint = parse_hint_number(cmd_lower)

    if hint:
        logger.debug("  🔤 Hint: '%s' → '%s'", cmd_lower, hint)
        qb(f"hint-follow {hint}")
        # Wait for page to load, then clear any stuck state
        time.sleep(1.0)
        qb("fake-key <Escape>")
    else:
        # Try phonetic fallback
        hint = parse_hint_numbers(cmd_lower)
        if hint:
            logger.debug("  🔤 Phonetic: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            # Wait for page to load, then clear any stuck state
            time.sleep(1.0)
            qb("fake-key <Escape>")
        else:
            logger.info("  ? Heard '%s', which isn't a hint number", cmd_lower)
            logger.debug("  ↪ Not a hint, trying as command: '%s'", cmd_lower)
            qb("fake-key <Escape>")
            handle_browser_command(cmd_lower, core)

    logger.debug("  [listen_for_hint returning - complete]")

handle 

handle(cmd, core)

Enter browser mode on a browser command; return None otherwise.

A matching command launches qutebrowser (if needed) and runs the continuous browser-mode loop; reserved global commands (sleep/quit) are passed through. Outside the explicit "open browser", navigation commands are only acted on when a browser is already running, so ambiguous words can't open one.

Source code in src/plugins/browser/__init__.py 
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
def handle(cmd, core):
    """Enter browser mode on a browser command; return None otherwise.

    A matching command launches qutebrowser (if needed) and runs the continuous
    browser-mode loop; reserved global commands (sleep/quit) are passed through. Outside
    the explicit "open browser", navigation commands are only acted on when a browser is
    already running, so ambiguous words can't open one.
    """
    cmd_lower = cmd.lower().strip(".,!? ")

    # Global sleep/quit commands belong to other plugins; if we matched them as
    # browser commands we would open qutebrowser instead. Let them through.
    if _is_reserved_global(cmd_lower):
        return None

    # Closing the application works from anywhere, and must not then step into
    # browser mode -- there would be no browser left to drive.
    if cmd_lower in CLOSE_BROWSER:
        if not _qutebrowser_running(core):
            core.speak(_("The browser isn't running."))
            return True
        qb("quit")
        core.speak(_("Closing browser."))
        return True

    # Already outside browser mode: acknowledge rather than fall through to the
    # "I didn't understand" path, which is what saying it twice used to get.
    if cmd_lower in LEAVE_BROWSER_MODE:
        return True

    # --- Enter browser mode (explicit) ---
    if cmd_lower in ["browser", "browser mode", "open browser", "launch browser"]:
        if core.in_browser_mode:
            return True  # already there; don't stack a second one
        if not _qutebrowser_running(core):
            try:
                core.host_run(["qutebrowser"], background=True, clean_env=True)
            except FileNotFoundError:
                logger.warning("qutebrowser isn't installed.")
                core.speak(_("Browser mode needs qutebrowser, which isn't installed."))
                return True
            except OSError as exc:
                logger.warning("Couldn't start qutebrowser: %s", exc)
                core.speak(_("Sorry, I couldn't start qutebrowser."))
                return True
        browser_mode(core)
        return True

    if not _qutebrowser_running(core):
        return None

    # --- Single browser commands → enters browser mode ---
    try:
        result = handle_browser_command(cmd_lower, core)
        if result:
            logger.info("  → Entering browser mode...")
            browser_mode(core)
            return True
    except Exception:
        logger.exception("  ! Browser error")
        core.speak(_("Sorry, that didn't work in the browser."))
        return True

    return None

browser_mode 

browser_mode(core)

Handle browser commands until the user leaves or the mode ends.

Core owns the listening loop (see listen_modal) so the tray keeps working while browser mode holds the microphone, and an unattended session ends on its own instead of leaving the wake word unreachable.

Source code in src/plugins/browser/__init__.py 
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
def browser_mode(core):
    """Handle browser commands until the user leaves or the mode ends.

    Core owns the listening loop (see
    [`listen_modal`][core.main.EasySpeak.listen_modal]) so the tray keeps working while
    browser mode holds the microphone, and an
    unattended session ends on its own instead of leaving the wake word unreachable.
    """
    core.speak(_("Browser"))
    logger.info("=== BROWSER MODE ACTIVE ===")
    logger.info("Say commands directly. 'exit browser' to leave.")

    core.in_browser_mode = True
    try:
        _run_browser_mode(core)
    finally:
        core.in_browser_mode = False

strip_filler 

strip_filler(cmd_lower)

Remove a leading filler word Whisper added to the front of a command.

Source code in src/plugins/browser/__init__.py 
821
822
823
824
825
826
def strip_filler(cmd_lower):
    """Remove a leading filler word Whisper added to the front of a command."""
    for filler in FILLER_PREFIXES:
        if cmd_lower.startswith(filler):
            return cmd_lower[len(filler) :].strip()
    return cmd_lower

handle_browser_command 

handle_browser_command(cmd_lower, core)

Execute a single in-browser command; None if it isn't recognised.

Source code in src/plugins/browser/__init__.py 
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
def handle_browser_command(cmd_lower, core):
    """Execute a single in-browser command; None if it isn't recognised."""
    cmd_lower = strip_filler(cmd_lower)

    # Said a beat after hint mode already closed; absorbing it beats answering
    # "I didn't understand" to a phrase this plugin owns.
    if cmd_lower in ("exit links", "exit link"):
        return True
    # --- Hints ---
    if cmd_lower in [
        "numbers",
        "number",
        "hints",
        "hint",
        "show numbers",
        "show hints",
        "links",
        "link",
        "blanks",
        "blinks",
        "lynx",
        "lings",
        "lanes",
        "licks",
        "clicks",
    ]:
        show_hints(core)
        listen_for_hint(core)
        return True

    if cmd_lower in [
        "numbers new",
        "number new",
        "hints new",
        "new numbers",
        "links new",
        "link new",
        "blanks new",
        "blinks new",
        "lynx new",
    ]:
        qb("hint links tab")
        listen_for_hint(core)
        return True

    # --- Navigation ---
    if cmd_lower in ["back", "go back", "previous page"]:
        qb("back")
        core.browser_page_js_stale = True
        return True

    if cmd_lower in ["forward", "go forward", "next page"]:
        qb("forward")
        core.browser_page_js_stale = True
        return True

    if cmd_lower in ["reload", "refresh", "reload page"]:
        qb("reload")
        return True

    if cmd_lower in ["stop", "stop loading"]:
        qb("stop")
        return True

    # --- Scrolling ---
    if cmd_lower in ["scroll down", "down"]:
        scroll_page(SCROLL_DOWN_JS, core)
        return True

    if cmd_lower in ["scroll up", "up"]:
        scroll_page(SCROLL_UP_JS, core)
        return True

    if "page" in cmd_lower and "down" in cmd_lower:
        qb(f"jseval -q {PAGE_DOWN_JS}")
        return True

    if "page" in cmd_lower and "up" in cmd_lower:
        qb(f"jseval -q {PAGE_UP_JS}")
        return True

    if cmd_lower in ["top", "go to top", "scroll to top"]:
        scroll_page(SCROLL_TOP_JS, core)
        return True

    if cmd_lower in ["bottom", "go to bottom", "scroll to bottom"]:
        scroll_page(SCROLL_BOTTOM_JS, core)
        return True

    # --- Tabs ---
    if cmd_lower in ["new tab", "open tab"]:
        qb("open -t about:blank")
        return True

    if cmd_lower in ["close tab", "close this tab"]:
        qb("tab-close")
        return True

    if cmd_lower in ["next tab", "tab right", "switch tab", "change tab"]:
        qb("tab-next")
        return True

    if cmd_lower in ["last tab", "previous tab", "tab left"]:
        qb("tab-prev")
        return True

    tab_num = parse_tab_number(cmd_lower)
    if tab_num is not None:
        if cmd_lower.startswith("close"):
            qb(f"tab-focus {tab_num}")
            qb("tab-close")
        else:
            qb(f"tab-focus {tab_num}")
        return True

    if cmd_lower in ["undo tab", "restore tab", "reopen tab"]:
        qb("undo")
        return True

    # --- Find ---
    if cmd_lower.startswith("find "):
        query = cmd_lower.replace("find ", "", 1).strip()
        if query:
            qb(f"search {query}")
            return True

    if cmd_lower in ["find next", "next match"]:
        qb("search-next")
        return True

    if cmd_lower in ["find previous", "previous match"]:
        qb("search-prev")
        return True

    # --- Escape ---
    if cmd_lower in ["escape", "cancel", "nevermind"]:
        qb("fake-key <Escape>")
        return True

    # --- Browser config toggles ---
    if cmd_lower in ["software rendering", "fix rendering", "fix the display"]:
        return _apply_config_line(
            core, SOFTWARE_RENDERING_LINE, _("Software rendering on"), wanted=True
        )

    if cmd_lower in ["hardware rendering", "restore rendering"]:
        return _apply_config_line(
            core, SOFTWARE_RENDERING_LINE, _("Hardware rendering on"), wanted=False
        )

    if cmd_lower in ["allow ads", "stop blocking ads", "disable ad blocking"]:
        return _apply_config_line(
            core, ADBLOCK_OFF_LINE, _("Ad blocking off"), wanted=True
        )

    if cmd_lower in ["block ads", "enable ad blocking"]:
        return _apply_config_line(
            core, ADBLOCK_OFF_LINE, _("Ad blocking on"), wanted=False
        )

    # --- Keystrokes ---
    request = mediakeys.parse_key_request(cmd_lower.split(), BARE_KEYS)
    if request is not None:
        if not mediakeys.press_key(*request):
            core.speak(_("Keystrokes need GNOME."))
        return True

    # --- Bookmarks ---
    # Save current page as quickmark
    if (
        "bookmark this" in cmd_lower or "save this" in cmd_lower
    ) and " as " in cmd_lower:
        name = cmd_lower.split(" as ")[-1].strip()
        if name:
            core.speak(_("Saved as {name}.").format(name=name))
            qb(f"quickmark-save {name}")
            return True

    # Load quickmark (user-saved)
    if cmd_lower.startswith(("go to ", "open ")):
        target = cmd_lower.replace("go to ", "").replace("open ", "").strip()

        # Check predefined bookmarks first
        for site, url in BOOKMARKS.items():
            if site == target:
                core.speak(_("Opening {site}.").format(site=site))
                qb_open(url)
                return True

        # Try as spoken URL (contains "dot")
        if "dot" in target or "." in target:
            url = parse_spoken_url(target)
            core.speak(_("Opening {url}.").format(url=url))
            qb_open(url)
            return True

        # Don't catch generic "open X" - let other plugins handle it
        # Only use quickmark for explicit "go to X"
        if cmd_lower.startswith("go to "):
            qb(f"quickmark-load {target}")
            return True

        return None  # Let another plugin handle "open X"

    # --- Search ---
    if cmd_lower.startswith(("search ", "search for ")):
        query = cmd_lower.replace("search for ", "").replace("search ", "").strip()
        if query:
            url = f"https://duckduckgo.com/?q={query.replace(' ', '+')}"
            core.speak(_("Searching for {query}.").format(query=query))
            qb_open(url)
            return True

    # --- Phonetic hint selection (LAST - only if nothing else matched) ---
    # Only try hint parsing if it actually looks like a hint
    if looks_like_hint(cmd_lower):
        # Direct digit input (e.g., "02", "92", "0-2")
        stripped = re.sub(r"[^0-9a-z]", "", cmd_lower)
        if stripped.replace("o", "0").isdigit():
            hint = stripped.replace("o", "0")
            logger.debug("  🔤 Direct digits: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            return True

        # Try phonetic parsing
        hint = parse_hint_numbers(cmd_lower)
        if hint and hint.isdigit():
            logger.debug("  🔤 Phonetic parsed: '%s' → '%s'", cmd_lower, hint)
            qb(f"hint-follow {hint}")
            return True

    return None

dictation

plugins.dictation

Dictation Plugin - Voice to text via AT-SPI.

load_vocabulary 

load_vocabulary(language, locale_dir=None)

Return the words spoken inside dictation in language.

They are the table locale/<language>/vocabulary.toml beside the reply catalog: how to leave dictation, undo, press keys, count, and every spoken punctuation mark with the text it inserts. What the app listens for is a table like this; what it says is the gettext catalog next to it. A language without a table gets the English one.

Source code in src/plugins/dictation/__init__.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def load_vocabulary(language, locale_dir=None):
    """Return the words spoken inside dictation in `language`.

    They are the table `locale/<language>/vocabulary.toml` beside the reply
    catalog: how to leave dictation, undo, press keys, count, and every spoken
    punctuation mark with the text it inserts. What the app listens for is a
    table like this; what it says is the gettext catalog next to it. A language
    without a table gets the English one.
    """
    locale_dir = (
        Path(__file__).with_name("locale") if locale_dir is None else locale_dir
    )
    for code in (language, "en"):
        table = load_table(locale_dir, code)
        if table is not None:
            return table
    msg = "the English dictation vocabulary is missing"
    raise FileNotFoundError(msg)

compile_replacements 

compile_replacements(vocabulary)

Return the spoken punctuation of vocabulary as (pattern, text) pairs.

Source code in src/plugins/dictation/__init__.py 
53
54
55
56
57
58
59
60
61
62
def compile_replacements(vocabulary):
    """Return the spoken punctuation of `vocabulary` as (pattern, text) pairs."""
    return [
        (
            re.compile(rf"\s*,?\s*\b{re.escape(phrase)}\b\s*", re.IGNORECASE),
            entry["insert"],
        )
        for entry in vocabulary["replace"]
        for phrase in entry["say"]
    ]

exit_phrases 

exit_phrases(vocabulary)

Return every way vocabulary lets a dictated utterance leave dictation.

Source code in src/plugins/dictation/__init__.py 
65
66
67
68
69
70
71
72
73
def exit_phrases(vocabulary):
    """Return every way `vocabulary` lets a dictated utterance leave dictation."""
    exit_words = vocabulary["exit"]
    return frozenset(
        phrase
        for verb in exit_words["verbs"]
        for noun in exit_words["nouns"]
        for phrase in (f"{verb} {noun}", f"{noun} {verb}", f"{verb}{noun}")
    )

ensure_gnome_accessibility 

ensure_gnome_accessibility()

Enable GNOME's toolkit-accessibility for the AT-SPI bridge.

Silently skipped if gsettings isn't on PATH or the schema isn't installed (i.e. user isn't on GNOME). Warns if it's present but the flip fails. Tells the user to re-login when newly enabled.

Source code in src/plugins/dictation/__init__.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def ensure_gnome_accessibility():
    """Enable GNOME's toolkit-accessibility for the AT-SPI bridge.

    Silently skipped if gsettings isn't on PATH or the schema isn't installed (i.e. user
    isn't on GNOME). Warns if it's present but the flip fails. Tells the user to
    re-login when newly enabled.
    """
    if shutil.which("gsettings") is None:
        return
    schema, key = "org.gnome.desktop.interface", "toolkit-accessibility"
    try:
        current = subprocess.run(
            ["gsettings", "get", schema, key],
            capture_output=True,
            text=True,
            check=False,
        )
        if current.returncode != 0:
            return  # schema missing — not really on GNOME
        if current.stdout.strip() == "true":
            return
        result = subprocess.run(
            ["gsettings", "set", schema, key, "true"],
            capture_output=True,
            text=True,
            check=False,
        )
        if result.returncode == 0:
            logger.info(
                "enabled GNOME toolkit-accessibility — "
                "log out and back in for dictation to work"
            )
        else:
            logger.warning(
                "could not enable GNOME toolkit-accessibility; dictation will not work"
            )
    except OSError:
        pass

setup 

setup(c)

Store the core reference and enable the GNOME accessibility bridge.

Source code in src/plugins/dictation/__init__.py 
156
157
158
159
160
161
162
163
164
165
166
167
def setup(c):
    """Store the core reference and enable the GNOME accessibility bridge."""
    global core
    core = c
    c.dictation_last_length = 0
    ensure_gnome_accessibility()
    # Offer dictation to the keyboard (silent) activation path too: core runs
    # this while the hotkey combo is held, with no wake word spoken.
    if hasattr(c, "register_push_to_talk"):
        c.register_push_to_talk(
            lambda should_continue: run_push_to_talk(c, should_continue)
        )

says 

says(text, phrases)

Whether text contains one of phrases as whole words.

Source code in src/plugins/dictation/__init__.py 
178
179
180
def says(text, phrases):
    """Whether `text` contains one of `phrases` as whole words."""
    return any(re.search(rf"\b{re.escape(phrase)}\b", text) for phrase in phrases)

is_exit_phrase 

is_exit_phrase(text, phrases=None)

Whether a dictated utterance asks to leave dictation mode.

Source code in src/plugins/dictation/__init__.py 
183
184
185
186
def is_exit_phrase(text, phrases=None):
    """Whether a dictated utterance asks to leave dictation mode."""
    phrases = EXIT_PHRASES if phrases is None else phrases
    return any(phrase in text.lower() for phrase in phrases)

focused_wm_class 

focused_wm_class()

Return the focused window's WM class, or None if it can't be read.

Asks the bundled GNOME extension, the same way the mouse grid asks it for the screen size. Only used to pick the paste chord, so any failure simply means the default one.

Source code in src/plugins/dictation/__init__.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
def focused_wm_class():
    """Return the focused window's WM class, or None if it can't be read.

    Asks the bundled GNOME extension, the same way the mouse grid asks it for the
    screen size. Only used to pick the paste chord, so any failure simply means
    the default one.
    """
    try:
        result = subprocess.run(
            [
                "gdbus",
                "call",
                "--session",
                "--dest",
                "org.gnome.Shell",
                "--object-path",
                "/org/easyspeak/Desktop",
                "--method",
                "org.easyspeak.Desktop.GetWindows",
            ],
            capture_output=True,
            text=True,
            check=False,
            timeout=WINDOW_QUERY_TIMEOUT,
        )
    except (OSError, subprocess.TimeoutExpired):
        return None
    if result.returncode != 0:
        return None
    try:
        # gdbus wraps the reply in GVariant tuple syntax: ('<json>',)
        payload = ast.literal_eval(result.stdout.strip())[0]
        windows = json.loads(payload)
    except (SyntaxError, ValueError, IndexError, TypeError):
        return None
    focused = next((w for w in windows if w.get("focused")), None)
    return focused.get("wm_class") if focused else None

paste_chord 

paste_chord(wm_class=UNKNOWN)

Return the evdev keycodes for the paste chord to send.

wm_class is the focused window if the caller already looked it up; reading it again costs a gdbus round trip on the path between speaking and seeing text.

EASYSPEAK_PASTE_KEYS overrides it outright (e.g. shift+insert); otherwise terminals get Ctrl+Shift+V and everything else Ctrl+V. An unknown key name in the override is reported and the default used, rather than silently sending a chord that does nothing.

Source code in src/plugins/dictation/__init__.py 
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
def paste_chord(wm_class=UNKNOWN):
    """Return the evdev keycodes for the paste chord to send.

    `wm_class` is the focused window if the caller already looked it up; reading it
    again costs a gdbus round trip on the path between speaking and seeing text.

    `EASYSPEAK_PASTE_KEYS` overrides it outright (e.g. `shift+insert`); otherwise
    terminals get Ctrl+Shift+V and everything else Ctrl+V. An unknown key name in
    the override is reported and the default used, rather than silently sending a
    chord that does nothing.
    """
    override = os.environ.get("EASYSPEAK_PASTE_KEYS", "").strip().lower()
    if override:
        codes = [KEYCODES.get(part.strip()) for part in override.split("+")]
        if all(codes):
            return codes
        logger.warning(
            "Ignoring EASYSPEAK_PASTE_KEYS=%r: unknown key. Known keys: %s.",
            override,
            ", ".join(sorted(KEYCODES)),
        )
    if wm_class is UNKNOWN:
        wm_class = focused_wm_class()
    chord = (
        TERMINAL_PASTE_CHORD
        if (wm_class or "").lower() in TERMINAL_WM_CLASSES
        else DEFAULT_PASTE_CHORD
    )
    return [KEYCODES[part] for part in chord.split("+")]

clipboard_tools 

clipboard_tools()

Return the (copy, paste) commands, or (None, None) if wl-clipboard is absent.

EasySpeak targets Wayland, so wl-clipboard is the only supported tool. Without it there is no way to put text on the clipboard.

Source code in src/plugins/dictation/__init__.py 
317
318
319
320
321
322
323
324
325
def clipboard_tools():
    """Return the (copy, paste) commands, or (None, None) if wl-clipboard is absent.

    EasySpeak targets Wayland, so wl-clipboard is the only supported tool. Without
    it there is no way to put text on the clipboard.
    """
    if shutil.which("wl-copy") and shutil.which("wl-paste"):
        return ["wl-copy"], ["wl-paste", "--no-newline"]
    return None, None

read_clipboard 

read_clipboard(paste_cmd)

Return the current clipboard text, or None if there isn't any.

Read as bytes and decoded here rather than by subprocess, because the clipboard often isn't text at all -- a copied image arrives as PNG bytes, and decoding those as UTF-8 raised straight out of the middle of an insertion and killed dictation mid-sentence.

Non-text content simply isn't preserved: the dictated text still gets pasted, but whatever was on the clipboard is gone afterwards. Restoring arbitrary binary would mean tracking MIME types through both tools, which is a lot of machinery for something a user rarely notices.

Source code in src/plugins/dictation/__init__.py 
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
def read_clipboard(paste_cmd):
    """Return the current clipboard text, or None if there isn't any.

    Read as bytes and decoded here rather than by `subprocess`, because the
    clipboard often isn't text at all -- a copied image arrives as PNG bytes, and
    decoding those as UTF-8 raised straight out of the middle of an insertion and
    killed dictation mid-sentence.

    Non-text content simply isn't preserved: the dictated text still gets pasted,
    but whatever was on the clipboard is gone afterwards. Restoring arbitrary
    binary would mean tracking MIME types through both tools, which is a lot of
    machinery for something a user rarely notices.
    """
    try:
        result = subprocess.run(
            paste_cmd, capture_output=True, check=False, timeout=CLIPBOARD_TIMEOUT
        )
    except (OSError, subprocess.TimeoutExpired):
        return None
    if result.returncode != 0:
        return None
    try:
        return result.stdout.decode("utf-8")
    except UnicodeDecodeError:
        logger.debug("clipboard holds non-text data; it won't be restored")
        return None

write_clipboard 

write_clipboard(copy_cmd, text)

Put text on the clipboard; False if that failed.

Output goes to /dev/null rather than a pipe. wl-copy forks a daemon to hold the selection for as long as it owns it, and that daemon inherits whatever pipes we hand the parent -- so capturing output makes subprocess.run wait on a process that never exits (CPython bpo-37424). Every copy timed out, and dictation reported that it couldn't reach the clipboard.

Source code in src/plugins/dictation/__init__.py 
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
def write_clipboard(copy_cmd, text):
    """Put text on the clipboard; False if that failed.

    Output goes to /dev/null rather than a pipe. `wl-copy` forks a daemon to hold
    the selection for as long as it owns it, and that daemon inherits whatever
    pipes we hand the parent -- so capturing output makes `subprocess.run` wait on
    a process that never exits (CPython bpo-37424). Every copy timed out, and
    dictation reported that it couldn't reach the clipboard.
    """
    try:
        result = subprocess.run(
            copy_cmd,
            input=text,
            text=True,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            check=False,
            timeout=CLIPBOARD_TIMEOUT,
        )
    except subprocess.TimeoutExpired:
        logger.warning("%s did not finish within %ss", copy_cmd[0], CLIPBOARD_TIMEOUT)
        return False
    except OSError as exc:
        logger.warning("could not run %s: %s", copy_cmd[0], exc)
        return False
    if result.returncode != 0:
        logger.warning("%s exited with code %s", copy_cmd[0], result.returncode)
        return False
    return True

leave_browser_insert_mode 

leave_browser_insert_mode()

Hand qutebrowser back in normal mode once dictation is over.

Insert mode is entered so a paste reaches the page, but leaving the browser in it means every later keystroke is treated as text: hinting and scrolling both misbehave, which looked like dictation breaking the browser on its way out.

Source code in src/plugins/dictation/__init__.py 
420
421
422
423
424
425
426
427
428
429
430
431
def leave_browser_insert_mode():
    """Hand qutebrowser back in normal mode once dictation is over.

    Insert mode is entered so a paste reaches the page, but leaving the browser in
    it means every later keystroke is treated as text: hinting and scrolling both
    misbehave, which looked like dictation breaking the browser on its way out.
    """
    global _left_browser_in_insert_mode
    if not _left_browser_in_insert_mode:
        return
    _left_browser_in_insert_mode = False
    _qb_command("mode-leave")

has_focused_text_field 

has_focused_text_field()

Whether anything is focused that could receive text.

Chromium-based apps accept an AT-SPI insertion and quietly discard it, which is why text goes in by clipboard and paste instead. They do still report which element holds focus, though, so the helper is run with an empty string purely as a question: it finds the insertion point, writes nothing, and says whether there was one.

Any failure answers yes. A probe that can't run is no reason to refuse to paste -- the old behaviour, blind but willing, is the better fallback.

Source code in src/plugins/dictation/__init__.py 
434
435
436
437
438
439
440
441
442
443
444
445
446
def has_focused_text_field():
    """Whether anything is focused that could receive text.

    Chromium-based apps accept an AT-SPI insertion and quietly discard it, which
    is why text goes in by clipboard and paste instead. They do still report which
    element holds focus, though, so the helper is run with an empty string purely
    as a question: it finds the insertion point, writes nothing, and says whether
    there was one.

    Any failure answers yes. A probe that can't run is no reason to refuse to
    paste -- the old behaviour, blind but willing, is the better fallback.
    """
    return insert_via_atspi("") != NO_FOCUS

insert_text 

insert_text(text)

Insert text into the focused application by pasting it.

The clipboard is put back afterwards, so dictating doesn't quietly cost the user whatever they had copied. Falls back to AT-SPI when no clipboard tool is installed, which is the only case where the old path was ever the better one.

Source code in src/plugins/dictation/__init__.py 
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
def insert_text(text):
    """Insert text into the focused application by pasting it.

    The clipboard is put back afterwards, so dictating doesn't quietly cost the
    user whatever they had copied. Falls back to AT-SPI when no clipboard tool is
    installed, which is the only case where the old path was ever the better one.
    """
    if not has_focused_text_field():
        return NO_FOCUS

    copy_cmd, paste_cmd = clipboard_tools()
    if copy_cmd is None:
        logger.warning(
            "wl-clipboard is not installed, so dictation is falling back to "
            "AT-SPI, which Chromium-based apps accept and ignore. Install "
            "wl-clipboard for text to reach the browser."
        )
        return insert_via_atspi(text)

    global _clipboard_generation

    target = focused_wm_class()

    saved = read_clipboard(paste_cmd)
    if not write_clipboard(copy_cmd, text):
        logger.warning("Could not put the dictated text on the clipboard.")
        return BACKEND_ERROR

    _clipboard_generation += 1
    chord = paste_chord(target)

    if target in BROWSER_WM_CLASSES:
        _enter_browser_insert_mode()
    logger.info(
        "⌨️  pasting %d chars into %s with %s",
        len(text),
        target or "an unknown window",
        "+".join(
            next(key for key, code in KEYCODES.items() if code == part)
            for part in chord
        ),
    )

    try:
        mediakeys.tap_chord(chord)
    except mediakeys.RemoteDesktopUnavailableError as exc:
        logger.warning(
            "Could not send the paste keystroke (%s). Dictation needs GNOME's "
            "RemoteDesktop interface, which this desktop doesn't provide.",
            exc,
        )
        return NO_KEYSTROKES
    except Exception:
        logger.warning("Could not send the paste keystroke.", exc_info=True)
        return BACKEND_ERROR

    logger.debug("pasted into %s", target or "an unknown window")

    _restore_clipboard_later(copy_cmd, saved, _clipboard_generation)
    return INSERTED

atspi_python 

atspi_python()

Path to an interpreter that can actually run the AT-SPI helper, or None.

The helper needs PyGObject and the AT-SPI typelib, which the app's own venv usually lacks. EASYSPEAK_ATSPI_PYTHON lets the packaging point straight at an interpreter that has them (the Nix flake sets it). Otherwise each candidate is probed with the helper's own import chain and the first that works is kept.

Falling back to a bare python3 used to be enough, but only until the daemon was run from an activated virtualenv: python3 then resolves to the venv's interpreter, which has no PyGObject, and dictation transcribed perfectly and inserted nothing.

Source code in src/plugins/dictation/__init__.py 
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
def atspi_python():
    """Path to an interpreter that can actually run the AT-SPI helper, or None.

    The helper needs PyGObject and the AT-SPI typelib, which the app's own venv
    usually lacks. `EASYSPEAK_ATSPI_PYTHON` lets the packaging point straight at an
    interpreter that has them (the Nix flake sets it). Otherwise each candidate is
    probed with the helper's own import chain and the first that works is kept.

    Falling back to a bare `python3` used to be enough, but only until the daemon
    was run from an activated virtualenv: `python3` then resolves to the venv's
    interpreter, which has no PyGObject, and dictation transcribed perfectly and
    inserted nothing.
    """
    global _atspi_python
    if _atspi_python is None:
        _atspi_python = _find_atspi_python() or False
    return _atspi_python or None

insert_via_atspi 

insert_via_atspi(text)

Insert text via AT-SPI.

Returns one of INSERTED, NO_FOCUS or BACKEND_ERROR so the caller can give feedback that matches the real cause instead of always blaming focus.

Source code in src/plugins/dictation/__init__.py 
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
def insert_via_atspi(text):
    """Insert text via AT-SPI.

    Returns one of INSERTED, NO_FOCUS or BACKEND_ERROR so the caller can give feedback
    that matches the real cause instead of always blaming focus.
    """
    python = atspi_python()
    if text:
        logger.info("⌨️  inserting %d chars via AT-SPI (%s)", len(text), python)
    if python is None:
        logger.warning(
            "Dictation needs PyGObject and the AT-SPI typelib, and no interpreter "
            "on this system has both. Install them (Fedora: sudo dnf install "
            "python3-gobject at-spi2-core; Debian/Ubuntu: sudo apt install "
            "python3-gi gir1.2-atspi-2.0), or point EASYSPEAK_ATSPI_PYTHON at an "
            "interpreter that already does."
        )
        return BACKEND_ERROR

    cmd = [python, ATSPI_HELPER, text]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
    except OSError as exc:
        logger.warning("dictation backend could not start: %s", exc)
        return BACKEND_ERROR

    if "OK" in result.stdout:
        return INSERTED
    if "NO_BACKEND" in result.stdout or result.returncode != 0:
        logger.warning(
            "dictation backend unavailable (PyGObject/AT-SPI missing): %s",
            result.stderr.strip() or "no detail",
        )
        return BACKEND_ERROR
    return NO_FOCUS

format_text 

format_text(text, vocabulary=None)

Convert spoken punctuation to actual punctuation.

The spoken words come from the dictation language's vocabulary table; pass vocabulary to format for another language.

Source code in src/plugins/dictation/__init__.py 
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
def format_text(text, vocabulary=None):
    """Convert spoken punctuation to actual punctuation.

    The spoken words come from the dictation language's vocabulary table; pass
    `vocabulary` to format for another language.
    """
    text = text.strip()

    # Strip ALL punctuation Whisper auto-adds; only explicit commands add it back
    text = re.sub(r"[.,!?;:]+", "", text)
    text = text.strip()

    replacements = (
        REPLACEMENTS if vocabulary is None else compile_replacements(vocabulary)
    )
    for pattern, insert in replacements:
        text = pattern.sub(lambda _match, insert=insert: insert, text)

    # A comma right before a period is Whisper's, not the speaker's.
    text = re.sub(r",\s*\.", ",", text)

    # Capitalize after sentence endings
    def capitalize_after(match):
        return match.group(1) + match.group(2).upper()

    text = re.sub(r"([.!?]\s+)([a-z])", capitalize_after, text)

    # Capitalize first letter
    if text:
        text = text[0].upper() + text[1:]

    # Clean up extra spaces
    text = re.sub(r" +", " ", text)
    text = re.sub(r" ([.,!?:;])", r"\1", text)

    # Fix double periods
    text = re.sub(r"\.+", ".", text)

    return text.strip()

run_push_to_talk 

run_push_to_talk(core, should_continue)

Dictate while the activation keys are held (the silent-activation path).

Mirrors the voice notes loop but is gated on should_continue — a predicate that is True while the keys remain held — instead of a spoken "stop notes": each utterance captured from core is formatted and inserted until the keys are released. The capture waits re-check should_continue so releasing stops dictation promptly.

Source code in src/plugins/dictation/__init__.py 
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
def run_push_to_talk(core, should_continue):
    """Dictate while the activation keys are held (the silent-activation path).

    Mirrors the voice `notes` loop but is gated on `should_continue` — a predicate that
    is True while the keys remain held — instead of a spoken "stop notes": each
    utterance captured from `core` is formatted and inserted until the keys are
    released. The capture waits re-check `should_continue` so releasing stops dictation
    promptly.
    """
    logger.info("🎙️ Push-to-talk dictation — release to end")
    while should_continue():
        core.flush_stream()
        first = core.wait_for_speech(
            timeout=PTT_LISTEN_TIMEOUT, should_continue=should_continue
        )
        if not first:
            continue
        audio = first + core.record_until_silence(should_continue=should_continue)
        text = core.transcribe(audio, prompt=DICTATION_PROMPT, language=LANGUAGE)
        if not text:
            continue
        if _dictate_utterance(core, text.strip()):
            return

handle 

handle(cmd, core)

Enter dictation mode on the entry word, "dictation" or the active language's.

The words come from the vocabulary tables, matched whole so "notebook" or "noted" do not trigger it, and a command that also names leaving ("stop dictation") is not an entry. While in dictation mode core drives the listening (see listen_modal), transcribing speech and inserting it into the focused field until the exit phrase is heard — or until the mode ends on its own, so an open microphone can't be left dictating unattended.

Source code in src/plugins/dictation/__init__.py 
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
def handle(cmd, core):
    """Enter dictation mode on the entry word, "dictation" or the active language's.

    The words come from the vocabulary tables, matched whole so "notebook" or
    "noted" do not trigger it, and a command that also names leaving ("stop
    dictation") is not an entry. While in dictation mode core drives the listening
    (see [`listen_modal`][core.main.EasySpeak.listen_modal]), transcribing speech
    and inserting it into the focused field until the exit phrase is heard — or
    until the mode ends on its own, so an open microphone can't be left dictating
    unattended.
    """
    leaving = set(EXIT_VERBS) | set(ENGLISH["exit"]["verbs"])
    if says(cmd, ENTER_PHRASES) and not set(cmd.split()) & leaving:
        core.speak(_("Dictation"))

        logger.info("🎙️ Dictation mode - say '%s' to end", VOCABULARY["exit"]["say"])

        try:
            return _dictation_session(core)
        finally:
            # However the session ended -- "stop notes", an idle timeout, the
            # tray -- the browser must not be left in insert mode.
            leave_browser_insert_mode()

    return None

mousegrid

plugins.mousegrid

Mouse Grid Plugin - Voice-controlled mouse via GNOME Shell D-Bus extension.

Features: - Chained numbers: "3 7 5" zooms three times in one utterance - Repetition: "nudge up 5", "scroll down 3" - Drag support: "mark" to start, "drag" to end

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/mousegrid/__init__.py 
109
110
111
112
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

host_run 

host_run(cmd)

Run a command and capture its output (the plugin's own subprocess seam).

A missing executable returns a failed result (returncode 1) rather than raising, so callers treat an absent gdbus—e.g. on a headless host such as WSL with no GNOME session—as a plain D-Bus failure. Without this the atexit cleanup would raise FileNotFoundError on exit.

Source code in src/plugins/mousegrid/__init__.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def host_run(cmd):
    """Run a command and capture its output (the plugin's own subprocess seam).

    A missing executable returns a failed result (returncode 1) rather than
    raising, so callers treat an absent `gdbus`—e.g. on a headless host such as
    WSL with no GNOME session—as a plain D-Bus failure. Without this the atexit
    cleanup would raise FileNotFoundError on exit.
    """
    try:
        return subprocess.run(cmd, capture_output=True, text=True)
    except FileNotFoundError as exc:
        logger.debug("Command not found: %s", exc)
        return subprocess.CompletedProcess(
            cmd, returncode=1, stdout="", stderr=str(exc)
        )

dbus_call 

dbus_call(method, *args)

Call a method on the grid extension over D-Bus; True on success.

Source code in src/plugins/mousegrid/__init__.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def dbus_call(method, *args):
    """Call a method on the grid extension over D-Bus; True on success."""
    cmd = [
        "gdbus",
        "call",
        "--session",
        "--dest",
        "org.gnome.Shell",
        "--object-path",
        "/org/easyspeak/Desktop",
        "--method",
        f"org.easyspeak.Desktop.{method}",
    ]
    cmd.extend(str(a) for a in args)
    result = host_run(cmd)
    return result.returncode == 0

get_screen_size 

get_screen_size()

Get screen size from GNOME Shell extension (accurate for Wayland).

Source code in src/plugins/mousegrid/__init__.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
def get_screen_size():
    """Get screen size from GNOME Shell extension (accurate for Wayland)."""
    result = host_run(
        [
            "gdbus",
            "call",
            "--session",
            "--dest",
            "org.gnome.Shell",
            "--object-path",
            "/org/easyspeak/Desktop",
            "--method",
            "org.easyspeak.Desktop.GetScreenSize",
        ]
    )

    if result.returncode == 0:
        # Parse output like "((1920, 1200),)"

        match = re.search(r"\((\d+),\s*(\d+)\)", result.stdout)
        if match:
            return int(match.group(1)), int(match.group(2))

    # Fallback to /sys/class/drm
    for card in [
        "card0-eDP-1",
        "card1-eDP-1",
        "card0-DP-1",
        "card1-DP-1",
        "card0-HDMI-A-1",
        "card1-HDMI-A-1",
    ]:
        result = host_run(["cat", f"/sys/class/drm/{card}/modes"])
        if result.returncode == 0 and result.stdout.strip():
            match = re.match(r"(\d+)x(\d+)", result.stdout.strip().split("\n")[0])
            if match:
                return int(match.group(1)), int(match.group(2))
    return 1920, 1080

cleanup 

cleanup()

Hide the grid on interpreter exit so no overlay is left on screen.

Source code in src/plugins/mousegrid/__init__.py 
210
211
212
213
def cleanup():
    """Hide the grid on interpreter exit so no overlay is left on screen."""
    _release_pending_drag()
    dbus_call("Hide")

parse_number_sequence 

parse_number_sequence(text)

Extract ALL numbers from text as a sequence.

'3 7 5' -> [3, 7, 5].

Source code in src/plugins/mousegrid/__init__.py 
219
220
221
222
223
224
225
226
227
228
229
230
def parse_number_sequence(text):
    """Extract ALL numbers from text as a sequence.

    '3 7 5' -> [3, 7, 5].
    """
    # Replace word numbers with digits
    text_lower = text.lower()
    for word, num in WORD_TO_NUM.items():
        text_lower = re.sub(rf"\b{word}\b", str(num), text_lower)

    # Extract all single digits (grid zones are 1-9)
    return [int(char) for char in text_lower if char.isdigit() and char != "0"]

parse_count 

parse_count(text)

Extract a repeat count (for nudge/scroll).

Returns single number or 1.

Source code in src/plugins/mousegrid/__init__.py 
233
234
235
236
237
238
239
240
241
242
243
244
245
def parse_count(text):
    """Extract a repeat count (for nudge/scroll).

    Returns single number or 1.
    """
    text_lower = text.lower()
    for word, num in WORD_TO_NUM.items():
        text_lower = re.sub(rf"\b{word}\b", str(num), text_lower)

    match = re.search(r"\b(\d+)\b", text_lower)
    if match:
        return min(int(match.group(1)), 20)
    return 1

parse_direction 

parse_direction(text)

Return the direction (up/down/left/right) named in text, or None.

Source code in src/plugins/mousegrid/__init__.py 
248
249
250
251
252
253
254
def parse_direction(text):
    """Return the direction (up/down/left/right) named in text, or None."""
    for direction, words in DIRECTIONS.items():
        for word in words:
            if word in text:
                return direction
    return None

get_center 

get_center()

Return the (x, y) center of the current grid cell, or None if no grid.

Source code in src/plugins/mousegrid/__init__.py 
257
258
259
260
261
262
def get_center():
    """Return the (x, y) center of the current grid cell, or None if no grid."""
    if not grid_bounds:
        return None
    x, y, w, h = grid_bounds
    return x + w // 2, y + h // 2

show_grid 

show_grid()

Show the full-screen grid overlay (no-op if it is already active).

Source code in src/plugins/mousegrid/__init__.py 
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
def show_grid():
    """Show the full-screen grid overlay (no-op if it is already active)."""
    global grid_active, grid_bounds, screen_size

    if grid_active:
        return

    screen_size = get_screen_size()
    grid_bounds = (0, 0, screen_size[0], screen_size[1])

    if dbus_call("Show", screen_size[0], screen_size[1]):
        grid_active = True
        core.speak(_("Grid"))
        logger.info("Grid shown (%sx%s)", screen_size[0], screen_size[1])
    else:
        report_grid_unavailable(core)

report_grid_unavailable 

report_grid_unavailable(core)

Tell the user the grid couldn't be shown, and the likely reason.

Source code in src/plugins/mousegrid/__init__.py 
286
287
288
289
290
291
292
293
def report_grid_unavailable(core):
    """Tell the user the grid couldn't be shown, and the likely reason."""
    if on_gnome():
        logger.warning("Failed to show the grid. Is the EasySpeak extension enabled?")
        core.speak(_("I couldn't show the grid. Is the EasySpeak extension enabled?"))
    else:
        logger.warning("The mouse grid needs GNOME and EasySpeak's Shell extension.")
        core.speak(_("The mouse grid only works on GNOME so far."))

close_grid 

close_grid()

Hide the grid overlay and clear its bounds.

Source code in src/plugins/mousegrid/__init__.py 
296
297
298
299
300
301
def close_grid():
    """Hide the grid overlay and clear its bounds."""
    global grid_active, grid_bounds
    dbus_call("Hide")
    grid_active = False
    grid_bounds = None

update_grid 

update_grid(zone)

Zoom to a single zone.

Returns True if successful.

Source code in src/plugins/mousegrid/__init__.py 
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
def update_grid(zone):
    """Zoom to a single zone.

    Returns True if successful.
    """
    global grid_bounds

    if not grid_active or not grid_bounds:
        return False

    x, y, w, h = grid_bounds
    zone_w, zone_h = w // 3, h // 3

    zone_map = {
        1: (0, 0),
        2: (1, 0),
        3: (2, 0),  # top row
        4: (0, 1),
        5: (1, 1),
        6: (2, 1),  # middle row
        7: (0, 2),
        8: (1, 2),
        9: (2, 2),  # bottom row
    }

    if zone not in zone_map:
        return False

    col, row = zone_map[zone]
    new_x = x + col * zone_w
    new_y = y + row * zone_h

    # Only clamp if actually going off-screen (shouldn't happen, but safety)
    new_x = max(new_x, 0)
    new_y = max(new_y, 0)
    if new_x + zone_w > screen_size[0]:
        new_x = screen_size[0] - zone_w
    if new_y + zone_h > screen_size[1]:
        new_y = screen_size[1] - zone_h

    grid_bounds = (new_x, new_y, zone_w, zone_h)

    cx, cy = get_center()
    logger.debug("  → Zone %s: %sx%s center=(%s, %s)", zone, zone_w, zone_h, cx, cy)
    return dbus_call("Update", new_x, new_y, zone_w, zone_h)

process_zones 

process_zones(zones)

Process a sequence of zones.

'three seven five' zooms 3 times.

Source code in src/plugins/mousegrid/__init__.py 
351
352
353
354
355
356
357
def process_zones(zones):
    """Process a sequence of zones.

    'three seven five' zooms 3 times.
    """
    for zone in zones:
        update_grid(zone)

do_click 

do_click(click_type='click')

Click at the current cell center and close the grid; False if no grid.

Source code in src/plugins/mousegrid/__init__.py 
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
def do_click(click_type="click"):
    """Click at the current cell center and close the grid; False if no grid."""
    global grid_bounds, grid_active, last_bounds

    center = get_center()
    if not center:
        return False

    cx, cy = center
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None

    method_map = {
        "click": "Click",
        "double": "DoubleClick",
        "right": "RightClick",
        "middle": "MiddleClick",
    }

    return dbus_call(method_map.get(click_type, "Click"), cx, cy)

do_scroll 

do_scroll(direction, count=1)

Scroll count steps at the current cell center; False if no grid.

Source code in src/plugins/mousegrid/__init__.py 
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
def do_scroll(direction, count=1):
    """Scroll `count` steps at the current cell center; False if no grid."""
    global grid_bounds, grid_active, last_bounds

    center = get_center()
    if not center:
        return False

    cx, cy = center
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None

    logger.debug("  → Scroll %s x%s", direction, count)
    return dbus_call("Scroll", cx, cy, direction, count)

nudge_grid 

nudge_grid(direction, count=1)

Shift the current cell count steps in a direction; False if no grid.

Source code in src/plugins/mousegrid/__init__.py 
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
def nudge_grid(direction, count=1):
    """Shift the current cell `count` steps in a direction; False if no grid."""
    global grid_bounds

    if not grid_bounds:
        return False

    x, y, w, h = grid_bounds
    amount = NUDGE_AMOUNT * count

    if direction == "up":
        y = max(0, y - amount)
    elif direction == "down":
        y = min(screen_size[1] - h, y + amount)
    elif direction == "left":
        x = max(0, x - amount)
    elif direction == "right":
        x = min(screen_size[0] - w, x + amount)

    grid_bounds = (x, y, w, h)
    cx, cy = get_center()
    logger.debug("  → Nudge %s x%s: center=(%s, %s)", direction, count, cx, cy)
    return dbus_call("Update", x, y, w, h)

start_drag 

start_drag()

Press at the current cell center and reset the grid to full screen.

Leaves a drag in progress so a later end_drag releases at the new target. False if no grid is active.

Source code in src/plugins/mousegrid/__init__.py 
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
def start_drag():
    """Press at the current cell center and reset the grid to full screen.

    Leaves a drag in progress so a later [`end_drag`][plugins.mousegrid.end_drag]
    releases at the new target. False if no grid is active.
    """
    global drag_start, grid_bounds
    center = get_center()
    if not center:
        return False
    drag_start = center
    logger.debug("  → Drag start: %s", center)
    dbus_call("StartDrag", center[0], center[1])

    # Reset grid to full screen so user can navigate to destination
    grid_bounds = (0, 0, screen_size[0], screen_size[1])
    dbus_call("Update", 0, 0, screen_size[0], screen_size[1])
    logger.debug("  → Grid reset - navigate to drop location")
    return True

end_drag 

end_drag()

Release a drag and close the grid.

Releases a drag started by start_drag. False if no drag is in progress or no grid is active.

Source code in src/plugins/mousegrid/__init__.py 
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
def end_drag():
    """Release a drag and close the grid.

    Releases a drag started by [`start_drag`][plugins.mousegrid.start_drag].
    False if no drag is in progress or no grid is active.
    """
    global drag_start, grid_bounds, grid_active, last_bounds
    if not drag_start:
        logger.debug("  → No drag in progress")
        return False
    center = get_center()
    if not center:
        return False
    logger.debug("  → Drag end: %s", center)
    dbus_call("EndDrag", center[0], center[1])
    last_bounds = grid_bounds
    grid_active = False
    grid_bounds = None
    drag_start = None
    return True

handle 

handle(cmd, core)

Show or reopen the grid on a trigger word, then enter grid mode.

Returns None for commands that don't open the grid.

Source code in src/plugins/mousegrid/__init__.py 
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
def handle(cmd, core):
    """Show or reopen the grid on a trigger word, then enter grid mode.

    Returns None for commands that don't open the grid.
    """
    global grid_active, grid_bounds, last_bounds, screen_size

    cmd_lower = cmd.lower().strip()
    words = cmd_lower.split()

    # "Again" - reopen at last position
    if any(w in words for w in ["again", "repeat", "reopen"]) and last_bounds:
        screen_size = get_screen_size()
        if not dbus_call("Show", screen_size[0], screen_size[1]):
            report_grid_unavailable(core)
            return True
        grid_bounds = last_bounds
        grid_active = True
        dbus_call("Update", *last_bounds)
        core.speak(_("Grid"))
        logger.info("Grid reopened at last position")
        listen_for_grid_commands(core)
        return True

    # Show grid
    if (
        any(w in words for w in GRID_TRIGGERS)
        and "close" not in words
        and "hide" not in words
    ):
        show_grid()
        listen_for_grid_commands(core)
        return True

    return None

listen_for_grid_commands 

listen_for_grid_commands(core)

Handle grid commands until the grid closes or the mode ends.

Core owns the listening loop (see listen_modal) so the tray stays responsive while the grid holds the microphone, and an unattended grid times out instead of trapping the session. Any drag still in progress is released on the way out, so a timeout can't leave the mouse button held down.

Source code in src/plugins/mousegrid/__init__.py 
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
def listen_for_grid_commands(core):
    """Handle grid commands until the grid closes or the mode ends.

    Core owns the listening loop (see
    [`listen_modal`][core.main.EasySpeak.listen_modal]) so the tray stays responsive
    while the grid holds the microphone, and an
    unattended grid times out instead of trapping the session. Any drag still in
    progress is released on the way out, so a timeout can't leave the mouse button
    held down.
    """
    global grid_active, drag_start

    if not grid_active:
        return  # the grid never opened (no extension); nothing to listen for

    logger.info("Grid mode: say numbers (e.g. '3 7 5'), nudge, scroll, click, close")

    try:
        _run_grid_mode(core)
    finally:
        _release_pending_drag()
        if grid_active:
            logger.debug("  → Hiding grid left open by an ended mode")
            close_grid()

headtrack

plugins.headtrack

Head Tracking Plugin - Cursor control via SixDRepNet.

Uses a 6D rotation representation for smooth head pose estimation.

OneEuroFilter 

OneEuroFilter(freq=30.0, min_cutoff=1.0, beta=0.007, d_cutoff=1.0)

One-Euro filter for smoothing noisy signals.

Adaptive: smooth when the signal is still, responsive when it moves.

Initialise the filter's cutoff/responsiveness parameters.

Source code in src/plugins/headtrack/__init__.py 
53
54
55
56
57
58
59
60
def __init__(self, freq=30.0, min_cutoff=1.0, beta=0.007, d_cutoff=1.0):
    """Initialise the filter's cutoff/responsiveness parameters."""
    self.freq = freq
    self.min_cutoff = min_cutoff
    self.beta = beta
    self.d_cutoff = d_cutoff
    self.x_prev = None
    self.dx_prev = 0.0

__call__ 

__call__(x)

Filter the next sample x and return the smoothed value.

Source code in src/plugins/headtrack/__init__.py 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def __call__(self, x):
    """Filter the next sample `x` and return the smoothed value."""
    if self.x_prev is None:
        self.x_prev = x
        return x

    # Derivative
    dx = (x - self.x_prev) * self.freq

    # Smooth derivative
    a_d = self._alpha(self.d_cutoff)
    dx_smooth = a_d * dx + (1 - a_d) * self.dx_prev
    self.dx_prev = dx_smooth

    # Adaptive cutoff based on speed
    cutoff = self.min_cutoff + self.beta * abs(dx_smooth)

    # Smooth value
    a = self._alpha(cutoff)
    x_smooth = a * x + (1 - a) * self.x_prev
    self.x_prev = x_smooth

    return x_smooth

setup 

setup(c)

Store the core reference for use by the plugin's handlers.

Source code in src/plugins/headtrack/__init__.py 
92
93
94
95
def setup(c):
    """Store the core reference for use by the plugin's handlers."""
    global core
    core = c

host_run 

host_run(cmd)

Run a command and capture its output (the plugin's own subprocess seam).

Source code in src/plugins/headtrack/__init__.py 
 98
 99
100
def host_run(cmd):
    """Run a command and capture its output (the plugin's own subprocess seam)."""
    return subprocess.run(cmd, capture_output=True, text=True)

dbus_call 

dbus_call(method, *args)

Call GNOME Shell extension for cursor movement.

Source code in src/plugins/headtrack/__init__.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def dbus_call(method, *args):
    """Call GNOME Shell extension for cursor movement."""
    cmd = [
        "gdbus",
        "call",
        "--session",
        "--dest",
        "org.gnome.Shell",
        "--object-path",
        "/org/easyspeak/Desktop",
        "--method",
        f"org.easyspeak.Desktop.{method}",
    ]
    cmd.extend(str(a) for a in args)
    result = host_run(cmd)
    return result.returncode == 0

get_screen_size 

get_screen_size()

Get screen size from GNOME Shell extension.

Source code in src/plugins/headtrack/__init__.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def get_screen_size():
    """Get screen size from GNOME Shell extension."""
    result = host_run(
        [
            "gdbus",
            "call",
            "--session",
            "--dest",
            "org.gnome.Shell",
            "--object-path",
            "/org/easyspeak/Desktop",
            "--method",
            "org.easyspeak.Desktop.GetScreenSize",
        ]
    )

    if result.returncode == 0:
        import re

        match = re.search(r"\((\d+),\s*(\d+)\)", result.stdout)
        if match:
            return int(match.group(1)), int(match.group(2))
    return 1920, 1080

give_up 

give_up(reason)

End tracking from inside the tracking thread, leaving the reason for the user.

The thread doesn't speak itself: start_tracking waits for it and hands the reason to the foreground, which speaks it in order and before listening again.

Source code in src/plugins/headtrack/__init__.py 
146
147
148
149
150
151
152
153
154
155
def give_up(reason):
    """End tracking from inside the tracking thread, leaving the reason for the user.

    The thread doesn't speak itself: `start_tracking` waits for it and hands the
    reason to the foreground, which speaks it in order and before listening again.
    """
    global tracking_active, start_failure
    tracking_active = False
    start_failure = reason
    tracking_started.set()

run_tracking 

run_tracking()

Main tracking loop.

Source code in src/plugins/headtrack/__init__.py 
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
def run_tracking():
    """Main tracking loop."""
    global tracking_active, stop_event, cursor_x, cursor_y, frozen

    try:
        import cv2
        from sixdrepnet import SixDRepNet
    except ImportError as exc:
        missing = isinstance(exc, ModuleNotFoundError)
        if not missing or exc.name not in ("cv2", "sixdrepnet"):
            logger.exception("The head tracking libraries are installed but broken")
            give_up(_("Sorry, head tracking couldn't start."))
            return
        logger.warning(
            "Head tracking isn't installed (%s). Install EasySpeak with the "
            "head-tracking extra.",
            exc,
        )
        give_up(_("Head tracking isn't installed."))
        return

    try:
        logger.info("Loading SixDRepNet model...")
        model = SixDRepNet(gpu_id=-1)  # CPU mode
    except Exception:
        logger.exception("The head tracking model couldn't be loaded")
        give_up(_("Sorry, head tracking couldn't start."))
        return

    SCREEN_W, SCREEN_H = get_screen_size()
    logger.debug("Screen: %sx%s", SCREEN_W, SCREEN_H)

    cap = cv2.VideoCapture(1)  # Integrated webcam
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
    cap.set(cv2.CAP_PROP_FPS, 30)

    if not cap.isOpened():
        logger.warning("Cannot open the webcam for head tracking.")
        give_up(_("I couldn't open the webcam."))
        return
    tracking_started.set()

    filter_yaw = OneEuroFilter(freq=30.0, min_cutoff=0.8, beta=0.005)
    filter_pitch = OneEuroFilter(freq=30.0, min_cutoff=0.8, beta=0.005)

    # Rolling average buffers for stable output
    BUFFER_SIZE = 15
    yaw_buffer = []
    pitch_buffer = []

    # Velocity tracking for movement gating
    prev_yaw = None
    prev_pitch = None
    VELOCITY_THRESHOLD = 0.3  # Degrees per frame - ignore movement below this

    # Calibration center
    center_yaw = None
    center_pitch = None

    # Settings
    SENSITIVITY_X = 40.0  # Pixels per degree
    SENSITIVITY_Y = 25.0
    DEAD_ZONE = 3.0  # Degrees - ignore small movements

    # Initialize cursor position
    cursor_x = SCREEN_W // 2
    cursor_y = SCREEN_H // 2

    logger.info("Head tracking started. Look at center of screen...")
    logger.info("Say 'recalibrate' to reset center, 'stop tracking' to end")

    calibration_frames = 0
    calibration_yaw = 0.0
    calibration_pitch = 0.0

    while tracking_active and not stop_event.is_set():
        ret, frame = cap.read()
        if not ret:
            continue

        frame = cv2.flip(frame, 1)  # Mirror

        pitch, yaw, _roll = model.predict(frame)

        if len(pitch) == 0:
            continue

        raw_pitch = -pitch[0]  # Invert: look up = cursor up
        raw_yaw = -yaw[0]  # Invert: look right = cursor right

        # Auto-calibrate on first 10 frames
        if center_yaw is None:
            calibration_yaw += raw_yaw
            calibration_pitch += raw_pitch
            calibration_frames += 1

            if calibration_frames >= 10:
                center_yaw = calibration_yaw / 10
                center_pitch = calibration_pitch / 10
                logger.info("Calibrated: center=(%.1f, %.1f)", center_yaw, center_pitch)
            continue

        # Apply one-euro filter to raw values
        smooth_yaw = filter_yaw(raw_yaw)
        smooth_pitch = filter_pitch(raw_pitch)

        # Add to rolling average buffers
        yaw_buffer.append(smooth_yaw)
        pitch_buffer.append(smooth_pitch)
        if len(yaw_buffer) > BUFFER_SIZE:
            yaw_buffer.pop(0)  # pragma: no cover
            pitch_buffer.pop(0)  # pragma: no cover

        # Use averaged values
        avg_yaw = sum(yaw_buffer) / len(yaw_buffer)
        avg_pitch = sum(pitch_buffer) / len(pitch_buffer)

        # Velocity gating - ignore tiny movements (noise)
        if prev_yaw is not None:
            vel_yaw = abs(avg_yaw - prev_yaw)
            vel_pitch = abs(avg_pitch - prev_pitch)

            # If head is basically still, skip this frame
            if vel_yaw < VELOCITY_THRESHOLD and vel_pitch < VELOCITY_THRESHOLD:
                prev_yaw = avg_yaw
                prev_pitch = avg_pitch
                time.sleep(0.033)
                continue

        prev_yaw = avg_yaw
        prev_pitch = avg_pitch

        # Offset from center
        offset_yaw = avg_yaw - center_yaw
        offset_pitch = avg_pitch - center_pitch

        # Dead zone
        if abs(offset_yaw) < DEAD_ZONE:
            offset_yaw = 0
        else:
            offset_yaw = (
                offset_yaw - DEAD_ZONE if offset_yaw > 0 else offset_yaw + DEAD_ZONE
            )

        if abs(offset_pitch) < DEAD_ZONE:
            offset_pitch = 0
        else:
            offset_pitch = (
                offset_pitch - DEAD_ZONE
                if offset_pitch > 0
                else offset_pitch + DEAD_ZONE
            )

        # Exponential curve - small moves stay small, accelerates toward edges
        # Preserves sign, applies power curve to magnitude
        EXPO = 2.0  # Higher = more acceleration toward edges
        if offset_yaw != 0:
            sign = 1 if offset_yaw > 0 else -1
            offset_yaw = sign * (abs(offset_yaw) ** EXPO)
        if offset_pitch != 0:
            sign = 1 if offset_pitch > 0 else -1
            offset_pitch = sign * (abs(offset_pitch) ** EXPO)

        # Map to cursor position (reduced sensitivity since expo amplifies)
        target_x = SCREEN_W // 2 + (offset_yaw * SENSITIVITY_X / 3)
        target_y = SCREEN_H // 2 + (offset_pitch * SENSITIVITY_Y / 3)

        # Clamp target to screen FIRST
        target_x = max(0, min(SCREEN_W - 1, target_x))
        target_y = max(0, min(SCREEN_H - 1, target_y))

        # Edge dampening - slow down as we approach edges/corners
        EDGE_MARGIN = 150  # Pixels from edge where dampening kicks in
        MIN_DAMP = 0.3  # Minimum speed at very edge (0.3 = 30% speed)

        # Calculate distance from edges
        dist_left = target_x
        dist_right = SCREEN_W - 1 - target_x
        dist_top = target_y
        dist_bottom = SCREEN_H - 1 - target_y

        # Find closest edge distance
        closest_x = min(dist_left, dist_right)
        closest_y = min(dist_top, dist_bottom)

        # Dampening: 1.0 when far from edge, MIN_DAMP at edge
        damp_x = (
            1.0
            if closest_x > EDGE_MARGIN
            else MIN_DAMP + (1.0 - MIN_DAMP) * (closest_x / EDGE_MARGIN)
        )
        damp_y = (
            1.0
            if closest_y > EDGE_MARGIN
            else MIN_DAMP + (1.0 - MIN_DAMP) * (closest_y / EDGE_MARGIN)
        )

        # Apply dampening (blend toward target more slowly near edges)
        damp = min(damp_x, damp_y)  # Use tightest dampening for corners

        # Apply cursor position update (only if not frozen)
        if not frozen:
            # Apply edge dampening to movement
            target_x_damped = cursor_x + (target_x - cursor_x) * damp
            target_y_damped = cursor_y + (target_y - cursor_y) * damp

            cursor_x = target_x_damped
            cursor_y = target_y_damped

            # Strictly clamp to screen bounds
            cursor_x = max(5, min(SCREEN_W - 5, cursor_x))
            cursor_y = max(5, min(SCREEN_H - 5, cursor_y))

            # Move cursor via GNOME Shell extension
            dbus_call("MoveTo", int(cursor_x), int(cursor_y))

        # Throttle to reduce jitter
        time.sleep(0.033)  # ~30fps

    cap.release()
    tracking_active = False
    logger.info("Tracking stopped.")

start_tracking 

start_tracking()

Start the head-tracking thread; returns (started, message).

Waits until the thread runs or gives up, at most STARTUP_TIMEOUT seconds, so the message tells the truth: "Tracking", or why tracking couldn't start. A thread from an earlier attempt that is still busy, e.g. still loading the model after a timeout, must finish first: started anew, it would report into this attempt.

Source code in src/plugins/headtrack/__init__.py 
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
def start_tracking():
    """Start the head-tracking thread; returns (started, message).

    Waits until the thread runs or gives up, at most `STARTUP_TIMEOUT` seconds, so
    the message tells the truth: "Tracking", or why tracking couldn't start. A thread
    from an earlier attempt that is still busy, e.g. still loading the model after a
    timeout, must finish first: started anew, it would report into this attempt.
    """
    global tracking_active, tracking_thread, stop_event, frozen, start_failure

    if tracking_active:
        return False, _("Already tracking")
    if tracking_thread is not None:
        tracking_thread.join(timeout=1.0)
        if tracking_thread.is_alive():
            return False, _("Head tracking is still busy. Try again in a moment.")

    frozen = False
    stop_event.clear()
    tracking_started.clear()
    start_failure = None
    tracking_active = True
    tracking_thread = threading.Thread(target=run_tracking, daemon=True)
    tracking_thread.start()
    if not tracking_started.wait(STARTUP_TIMEOUT):
        logger.warning("Head tracking didn't start within %s seconds.", STARTUP_TIMEOUT)
        stop_event.set()
        tracking_active = False
        return False, _("Sorry, head tracking couldn't start.")
    if start_failure:
        return False, start_failure
    return True, _("Tracking")

stop_tracking 

stop_tracking()

Signal the tracking thread to stop; returns (stopped, message).

Source code in src/plugins/headtrack/__init__.py 
417
418
419
420
421
422
423
def stop_tracking():
    """Signal the tracking thread to stop; returns (stopped, message)."""
    global tracking_active, stop_event
    stop_event.set()
    tracking_active = False
    time.sleep(0.2)
    return True, _("Stopped")

recalibrate 

recalibrate()

Reset calibration - will auto-calibrate on next frames.

Restarts tracking, so a restart that fails, e.g. because the webcam is gone, ends tracking and returns the reason instead of "Recalibrating".

Source code in src/plugins/headtrack/__init__.py 
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
def recalibrate():
    """Reset calibration - will auto-calibrate on next frames.

    Restarts tracking, so a restart that fails, e.g. because the webcam is gone,
    ends tracking and returns the reason instead of "Recalibrating".
    """
    # center_yaw/pitch are in the tracking thread scope, so restart tracking
    if tracking_active:
        stop_tracking()
        time.sleep(0.3)
        started, msg = start_tracking()
        if not started:
            return False, msg
        return True, _("Recalibrating")
    return False, _("Not tracking")

handle 

handle(cmd, core)

Start/stop tracking or run a tracking command; None if not head-tracking.

Source code in src/plugins/headtrack/__init__.py 
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
def handle(cmd, core):
    """Start/stop tracking or run a tracking command; None if not head-tracking."""
    global cursor_x, cursor_y
    cmd_lower = cmd.lower()

    # Start tracking
    if any(
        w in cmd_lower for w in ["start tracking", "begin tracking", "enable tracking"]
    ):
        if not tracking_active:
            core.speak(_("Starting head tracking."))
        success, msg = start_tracking()
        core.speak(msg)
        if success:
            listen_for_tracking_commands(core)
        return True

    # Stop tracking
    if any(
        w in cmd_lower
        for w in [
            "stop tracking",
            "end tracking",
            "close tracking",
            "quit tracking",
            "disable tracking",
            "tracking off",
            "stop track",
        ]
    ):
        success, msg = stop_tracking()
        core.speak(msg)
        return True

    # Recalibrate
    if "recalibrate" in cmd_lower or "calibrate" in cmd_lower:
        success, msg = recalibrate()
        core.speak(msg)
        return True

    return None

listen_for_tracking_commands 

listen_for_tracking_commands(core)

Handle tracking commands until tracking stops or the mode ends.

Core owns the listening loop (see listen_modal) so the tray keeps working while tracking holds the microphone. The mode also ends as soon as the tracking thread is gone — a webcam that failed to open used to leave the user in a mode with nothing driving the cursor.

Source code in src/plugins/headtrack/__init__.py 
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
def listen_for_tracking_commands(core):
    """Handle tracking commands until tracking stops or the mode ends.

    Core owns the listening loop (see
    [`listen_modal`][core.main.EasySpeak.listen_modal]) so the tray keeps working while
    tracking holds the microphone. The mode also ends
    as soon as the tracking thread is gone — a webcam that failed to open used to
    leave the user in a mode with nothing driving the cursor.
    """
    global tracking_active, cursor_x, cursor_y, frozen

    SCREEN_W, SCREEN_H = get_screen_size()

    logger.info("Tracking mode: freeze, nudge, click, or stop tracking")

    for spoken in core.listen_modal(
        "tracking",
        prompt=(
            "click double click right click freeze go nudge up down left "
            "right recalibrate stop tracking close cancel"
        ),
        timeout=10,
        idle_timeout=30,
    ):
        cmd_lower = spoken.lower()
        if not tracking_active:
            logger.info("Tracking stopped; leaving tracking mode")
            return

        logger.debug("  ← %s", cmd_lower)

        # Exit commands
        if any(
            w in cmd_lower
            for w in [
                "stop tracking",
                "end tracking",
                "close tracking",
                "stop",
                "cancel",
                "escape",
                "exit",
                "quit",
                "done",
            ]
        ):
            frozen = False
            stop_tracking()
            core.speak(_("Stopped"))
            return

        # Freeze/Go
        if any(w in cmd_lower for w in ["freeze", "free", "rees", "frees"]):
            frozen = True
            # "Frozen" contains none of the words that trigger this branch, so
            # hearing itself back can't freeze again.
            core.speak(_("Frozen"))
            logger.debug("  → Frozen at (%d, %d)", int(cursor_x), int(cursor_y))
            continue

        if cmd_lower in ["go", "go go", "unfreeze", "resume", "track"]:
            frozen = False
            core.speak(_("Following"))
            logger.debug("  → Resumed")
            continue

        # Nudge (only when frozen)
        if frozen and any(
            w in cmd_lower for w in ["nudge", "move", "up", "down", "left", "right"]
        ):
            if any(w in cmd_lower for w in ["up", "north"]):
                cursor_y = max(0, cursor_y - NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["down", "south"]):
                cursor_y = min(SCREEN_H - 1, cursor_y + NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["left", "west"]):
                cursor_x = max(0, cursor_x - NUDGE_AMOUNT)
            if any(w in cmd_lower for w in ["right", "east", "write"]):
                cursor_x = min(SCREEN_W - 1, cursor_x + NUDGE_AMOUNT)
            dbus_call("MoveTo", int(cursor_x), int(cursor_y))
            continue

        # Recalibrate
        if "recalibrate" in cmd_lower or "calibrate" in cmd_lower:
            frozen = False
            recalibrated, msg = recalibrate()
            core.speak(msg)
            if not recalibrated:
                return
            continue

        # Click commands
        if "double" in cmd_lower:
            dbus_call("DoubleClick", int(cursor_x), int(cursor_y))
            continue

        if any(w in cmd_lower for w in ["right click", "right-click", "write click"]):
            dbus_call("RightClick", int(cursor_x), int(cursor_y))
            continue

        if any(
            w in cmd_lower
            for w in ["click", "select", "press", "kick", "quick", "flick"]
        ):
            dbus_call("Click", int(cursor_x), int(cursor_y))
            continue