feat(transcription): add the self-hosted ratio backend

scripts/ratio-transcribe runs whisper plus speaker diarization on my own transcription host over an ssh job queue, with a local worker as the fallback when the host is unreachable. It needs no API key, so its descriptor carries nil for the auth host and the env var. The process environment passes through untouched. SPEAKERS=N in the environment pins the diarizer's speaker count.

The interactive switcher used to carry its own copy of the backend list, so a backend added to the descriptors was reachable only by setq. It now derives its choices from the descriptor alist, and a test pins the two to each other.

The script's own behavior (argument validation, the job-id hash, the remote and local branches) is tested in the repo it came from. A bats mirror of those cases is tracked separately.

3 files changed, 321 insertions, 6 deletions

diff --git a/modules/transcription-config.el b/modules/transcription-config.el
index 944063b8..6d7e39b9 100644
--- a/modules/transcription-config.el
+++ b/modules/transcription-config.el
@@ -33,6 +33,11 @@
 ;;   - 'assemblyai: Cloud transcription with speaker diarization
 ;;     API key retrieved from authinfo.gpg (machine api.assemblyai.com)
 ;;   - 'local-whisper: Local transcription (requires whisper installed)
+;;   - 'ratio: Self-hosted whisper plus speaker diarization on my transcription
+;;     host, via scripts/ratio-transcribe (ssh queue; falls back to a local
+;;     worker when the host is unreachable).  No API key.  SPEAKERS=N in the
+;;     environment pins the diarizer's speaker count; 1 is right for a
+;;     recording that holds only one side of a call.
 ;;
 ;; NOTIFICATIONS:
 ;;   - "Transcription started on <file>"
@@ -56,7 +61,8 @@
   "Transcription backend to use.
 - `openai-api': Fast cloud transcription via OpenAI API
 - `assemblyai': Cloud transcription with speaker diarization via AssemblyAI
-- `local-whisper': Local transcription using installed Whisper")
+- `local-whisper': Local transcription using installed Whisper
+- `ratio': Self-hosted whisper plus diarization on my transcription host")
 
 (defvar cj/transcription-keep-log-when-done nil
   "Whether to keep log files after successful transcription.
@@ -74,9 +80,11 @@ Status: running, complete, error")
 (defconst cj/--transcription-backends
   '((openai-api    :script "oai-transcribe"        :auth-host "api.openai.com"     :env-var "OPENAI_API_KEY")
     (assemblyai    :script "assemblyai-transcribe" :auth-host "api.assemblyai.com" :env-var "ASSEMBLYAI_API_KEY")
-    (local-whisper :script "local-whisper"         :auth-host nil                  :env-var nil))
+    (local-whisper :script "local-whisper"         :auth-host nil                  :env-var nil)
+    (ratio         :script "ratio-transcribe"      :auth-host nil                  :env-var nil))
   "Per-backend descriptors. Each entry: (SYMBOL :script S :auth-host H :env-var V).
-`:auth-host' and `:env-var' are nil for local backends that need no API key.")
+`:auth-host' and `:env-var' are nil for backends that need no API key: the
+local whisper install, and the self-hosted ratio queue reached over ssh.")
 
 (defun cj/--backend-plist (backend)
   "Return the descriptor plist for BACKEND, or signal if unknown."
@@ -470,9 +478,11 @@ duration is computed from START-TIME."
   "Switch transcription backend.
 Prompts with completing-read to select from available backends."
   (interactive)
-  (let* ((backends '(("assemblyai" . assemblyai)
-                     ("openai-api" . openai-api)
-                     ("local-whisper" . local-whisper)))
+  ;; Offer exactly the descriptor set.  This used to carry its own list, so
+  ;; a backend added to the descriptors was reachable only by setq.
+  (let* ((backends (mapcar (lambda (entry)
+                             (cons (symbol-name (car entry)) (car entry)))
+                           cj/--transcription-backends))
          (current (symbol-name cj/transcribe-backend))
          (prompt (format "Transcription backend (current: %s): " current))
          (choice (completing-read prompt backends nil t))
diff --git a/scripts/ratio-transcribe b/scripts/ratio-transcribe
new file mode 100755
index 00000000..8db59b66
--- /dev/null
+++ b/scripts/ratio-transcribe
@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+# ratio-transcribe - Transcribe audio on my own transcription host, with speaker labels
+# Usage: ratio-transcribe <audio-file> [language]
+#
+# Same contract as assemblyai-transcribe: the transcript goes to stdout, one line
+# per speaker turn ("HH:MM:SS Speaker A: text"); progress and errors go to stderr;
+# any failure exits non-zero with nothing on stdout.
+#
+# The work happens on a host that runs the meeting-transcribe queue (whisper-cpp
+# plus pyannote). This script copies the audio over ssh, drops a job into the
+# queue, waits, and prints the result. The job id is a hash of the audio and its
+# options, so if the connection drops or the laptop sleeps, running the same
+# command again just collects the finished transcript. If the host can't be
+# reached at all, the same queue and worker run on this machine instead.
+#
+# Optional environment:
+#   SPEAKERS                     exact number of speakers, when you know it
+#   MIN_SPEAKERS, MAX_SPEAKERS   a range instead
+#   TRANSCRIBE_HOST              ssh name of the host (default: ratio)
+#   TRANSCRIBE_TIMEOUT           seconds to wait for the job (default: 3600)
+#   TRANSCRIBE_POLL              seconds between checks (default: 10)
+#   TRANSCRIBE_LOCAL=1           skip the host and run here
+#   TRANSCRIBE_WORKER            path to the local worker
+
+set -euo pipefail
+
+AUDIO="${1:-}"
+LANG_CODE="${2:-en}"
+HOST="${TRANSCRIBE_HOST:-ratio}"
+TIMEOUT="${TRANSCRIBE_TIMEOUT:-3600}"
+POLL="${TRANSCRIBE_POLL:-10}"
+WORKER="${TRANSCRIBE_WORKER:-$HOME/.local/share/pyannote-diarize/src/transcribe-worker}"
+STATE=".local/state/meeting-transcribe"   # relative to the home directory, on either machine
+
+if [[ -z "$AUDIO" ]]; then
+  echo "Usage: ratio-transcribe <audio-file> [language]" >&2
+  echo "Example: SPEAKERS=3 ratio-transcribe meeting.m4a en" >&2
+  exit 1
+fi
+
+if [[ ! -f "$AUDIO" ]]; then
+  echo "Error: Audio file not found: $AUDIO" >&2
+  exit 1
+fi
+# scp reads "name:with:colons" as host:path; an absolute path removes the ambiguity.
+AUDIO="$(realpath -- "$AUDIO")"
+
+# Everything below ends up in a job file and on command lines, so check it first.
+if [[ ! "$LANG_CODE" =~ ^[A-Za-z]{2,8}(-[A-Za-z0-9]{1,8})*$ ]]; then
+  echo "Error: Invalid language code: $LANG_CODE" >&2
+  exit 1
+fi
+
+for name in SPEAKERS MIN_SPEAKERS MAX_SPEAKERS; do
+  value="${!name:-}"
+  if [[ -n "$value" && ! "$value" =~ ^[1-9][0-9]*$ ]]; then
+    echo "Error: $name must be a positive whole number of speakers, got: $value" >&2
+    exit 1
+  fi
+done
+if [[ -n "${SPEAKERS:-}" && ( -n "${MIN_SPEAKERS:-}" || -n "${MAX_SPEAKERS:-}" ) ]]; then
+  echo "Error: give an exact SPEAKERS count or a MIN/MAX speaker range, not both" >&2
+  exit 1
+fi
+if [[ -n "${MIN_SPEAKERS:-}" && -n "${MAX_SPEAKERS:-}" ]] && (( MIN_SPEAKERS > MAX_SPEAKERS )); then
+  echo "Error: MIN_SPEAKERS cannot exceed MAX_SPEAKERS (speaker range)" >&2
+  exit 1
+fi
+
+for tool in jq sha256sum; do
+  if ! command -v "$tool" &> /dev/null; then
+    echo "Error: $tool command not found" >&2
+    exit 1
+  fi
+done
+
+EXT="${AUDIO##*.}"
+[[ "$EXT" =~ ^[A-Za-z0-9]{1,5}$ ]] || EXT="bin"
+EXT="${EXT,,}"
+
+if [[ -n "${SPEAKERS:-}" ]]; then
+  COUNT_TAG="s${SPEAKERS}"
+elif [[ -n "${MIN_SPEAKERS:-}${MAX_SPEAKERS:-}" ]]; then
+  COUNT_TAG="r${MIN_SPEAKERS:-x}-${MAX_SPEAKERS:-x}"
+else
+  COUNT_TAG="auto"
+fi
+JOB_ID="$(sha256sum "$AUDIO" | cut -c1-16)-${LANG_CODE,,}-${COUNT_TAG}"
+
+JOB_JSON=$(jq -cn \
+  --arg language "$LANG_CODE" \
+  --arg name "$(basename "$AUDIO")" \
+  --arg speakers "${SPEAKERS:-}" --arg min "${MIN_SPEAKERS:-}" --arg max "${MAX_SPEAKERS:-}" \
+  '{language: $language}
+   + (if $speakers != "" then {speakers: ($speakers | tonumber)} else {} end)
+   + (if $min != "" then {min_speakers: ($min | tonumber)} else {} end)
+   + (if $max != "" then {max_speakers: ($max | tonumber)} else {} end)
+   + {original_name: $name}')
+
+# ssh reads stdin unless told not to, which would swallow the input of any loop
+# this script is called from. Only the job-file upload needs stdin.
+remote() { ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$HOST" "$@"; }
+remote_with_stdin() { ssh -o BatchMode=yes -o ConnectTimeout=8 "$HOST" "$@"; }
+
+# One word for where the job stands on the host: done, failed, queued or new.
+remote_status() {
+  remote "cd $STATE 2>/dev/null || { echo new; exit 0; }
+    if [ -e done/$JOB_ID.txt ]; then echo done
+    elif [ -e failed/$JOB_ID.log ]; then echo failed
+    elif [ -d incoming/$JOB_ID ] || [ -d work/$JOB_ID ]; then echo queued
+    else echo new; fi"
+}
+
+print_transcript() {  # $1 = the transcript text
+  if [[ -z "${1//[[:space:]]/}" ]]; then
+    echo "Error: the transcript came back empty" >&2
+    exit 1
+  fi
+  echo "Transcription complete! (${SECONDS}s total)" >&2
+  printf '%s\n' "$1"
+}
+
+run_remote() {
+  local status
+  status=$(remote_status)
+
+  if [[ "$status" == "failed" ]]; then
+    echo "An earlier attempt at this job failed; trying again..." >&2
+    remote "rm -f $STATE/failed/$JOB_ID.log"
+    status="new"
+  fi
+
+  if [[ "$status" == "new" ]]; then
+    echo "Uploading audio file to $HOST..." >&2
+    # Copy into uploading/, then rename into incoming/. The queue only ever sees
+    # a complete job.
+    remote "mkdir -p $STATE/incoming $STATE/uploading/$JOB_ID"
+    scp -q -o BatchMode=yes "$AUDIO" "$HOST:$STATE/uploading/$JOB_ID/audio.$EXT" < /dev/null
+    printf '%s' "$JOB_JSON" | remote_with_stdin "cat > $STATE/uploading/$JOB_ID/job.json"
+    remote "mv $STATE/uploading/$JOB_ID $STATE/incoming/$JOB_ID"
+    echo "Job $JOB_ID queued. Waiting for completion..." >&2
+  elif [[ "$status" == "queued" ]]; then
+    echo "Job $JOB_ID is already queued on $HOST. Waiting for completion..." >&2
+  fi
+
+  while true; do
+    # A dropped connection is not a failed job; keep asking until the timeout.
+    status=$(remote_status 2> /dev/null) || status="unreachable"
+    case "$status" in
+      done)
+        print_transcript "$(remote "cat $STATE/done/$JOB_ID.txt")"
+        return 0
+        ;;
+      failed)
+        echo "Error: transcription failed on $HOST" >&2
+        remote "cat $STATE/failed/$JOB_ID.log" >&2 || true
+        exit 1
+        ;;
+    esac
+    if (( SECONDS >= TIMEOUT )); then
+      echo "Error: no result after ${TIMEOUT}s. The job is still with $HOST;" >&2
+      echo "run the same command again to collect the transcript." >&2
+      exit 1
+    fi
+    sleep "$POLL"
+    [[ "$status" == "unreachable" ]] || echo "Processing... (${SECONDS}s elapsed)" >&2
+  done
+}
+
+run_local() {
+  if [[ ! -x "$WORKER" ]]; then
+    echo "Error: $HOST is unreachable and there is no local worker at $WORKER" >&2
+    exit 1
+  fi
+  local state="$HOME/$STATE"
+  if [[ ! -s "$state/done/$JOB_ID.txt" ]]; then
+    echo "Running the transcription locally (this machine is slower; expect a wait)..." >&2
+    rm -f "$state/failed/$JOB_ID.log"
+    rm -rf "$state/uploading/$JOB_ID"
+    mkdir -p "$state/incoming" "$state/uploading/$JOB_ID"
+    cp "$AUDIO" "$state/uploading/$JOB_ID/audio.$EXT"
+    printf '%s' "$JOB_JSON" > "$state/uploading/$JOB_ID/job.json"
+    [[ -d "$state/incoming/$JOB_ID" ]] || mv "$state/uploading/$JOB_ID" "$state/incoming/$JOB_ID"
+    HF_HUB_OFFLINE=1 "$WORKER" >&2 < /dev/null
+  fi
+  if [[ -e "$state/failed/$JOB_ID.log" ]]; then
+    echo "Error: local transcription failed" >&2
+    cat "$state/failed/$JOB_ID.log" >&2
+    exit 1
+  fi
+  if [[ ! -e "$state/done/$JOB_ID.txt" ]]; then
+    echo "Error: the local worker finished without producing a transcript" >&2
+    exit 1
+  fi
+  print_transcript "$(< "$state/done/$JOB_ID.txt")"
+}
+
+if [[ -z "${TRANSCRIBE_LOCAL:-}" ]] && remote true 2> /dev/null; then
+  run_remote
+else
+  [[ -n "${TRANSCRIBE_LOCAL:-}" ]] || echo "$HOST is unreachable." >&2
+  run_local
+fi
diff --git a/tests/test-transcription-config--ratio-backend.el b/tests/test-transcription-config--ratio-backend.el
new file mode 100644
index 00000000..f368b859
--- /dev/null
+++ b/tests/test-transcription-config--ratio-backend.el
@@ -0,0 +1,102 @@
+;;; test-transcription-config--ratio-backend.el --- the ratio self-hosted backend -*- lexical-binding: t; -*-
+
+;;; Commentary:
+;; The `ratio' backend runs whisper plus speaker diarization on my own host
+;; through the scripts/ratio-transcribe client.  It needs no API key, so its
+;; descriptor carries nil for both :auth-host and :env-var, and the process
+;; environment passes through unchanged.
+;;
+;; Two seams have to agree for a backend to be usable: the descriptor alist
+;; that resolves the script, and the completing-read list the interactive
+;; switcher offers.  The switcher used to carry its own copy of that list, so
+;; a descriptor added without a switcher entry was reachable only by setq.  It
+;; now derives its choices from the alist; the boundary test below guards
+;; against a return to a hardcoded list.
+
+;;; Code:
+
+(require 'ert)
+(require 'cl-lib)
+
+(add-to-list 'load-path (expand-file-name "modules" user-emacs-directory))
+
+(defvar cj/custom-keymap (make-sparse-keymap)
+  "Stub keymap for testing.")
+
+(unless (fboundp 'notifications-notify)
+  (defun notifications-notify (&rest _args)
+    "Stub notification function for testing."
+    nil))
+
+(require 'transcription-config)
+
+(defun test-transcription-ratio--switcher-choices ()
+  "Return the backend names `cj/transcription-switch-backend' offers.
+Captures the collection handed to `completing-read' and answers with the
+current backend so the switcher is a no-op."
+  (let (offered)
+    (cl-letf (((symbol-function 'completing-read)
+               (lambda (_prompt collection &rest _)
+                 (setq offered (mapcar #'car collection))
+                 (symbol-name cj/transcribe-backend))))
+      (let ((cj/transcribe-backend cj/transcribe-backend))
+        (cj/transcription-switch-backend)))
+    offered))
+
+;;; Normal
+
+(ert-deftest test-transcription-config-ratio-descriptor-resolves ()
+  "Normal: the ratio backend resolves to the ratio-transcribe client with no
+API-key requirement."
+  (let ((desc (cj/--backend-plist 'ratio)))
+    (should (equal (plist-get desc :script) "ratio-transcribe"))
+    (should (null (plist-get desc :auth-host)))
+    (should (null (plist-get desc :env-var)))))
+
+(ert-deftest test-transcription-config-ratio-script-path-and-executable ()
+  "Normal: the script path lands on scripts/ratio-transcribe and the file is
+there and executable, so a transcription can actually start."
+  (let ((cj/transcribe-backend 'ratio))
+    (let ((path (cj/--transcription-script-path)))
+      (should (string-suffix-p "scripts/ratio-transcribe" path))
+      (should (file-executable-p path)))))
+
+(ert-deftest test-transcription-config-ratio-environment-passes-through ()
+  "Normal: no API key means the process environment is returned unchanged,
+and nothing consults authinfo."
+  (cl-letf (((symbol-function 'cj/--auth-source-password)
+             (lambda (&rest _) (ert-fail "auth-source consulted for a keyless backend"))))
+    (should (eq (cj/--build-process-environment 'ratio) process-environment))))
+
+(ert-deftest test-transcription-config-switcher-offers-ratio ()
+  "Normal: the interactive switcher lists ratio, so the backend is reachable
+without a setq."
+  (should (member "ratio" (test-transcription-ratio--switcher-choices))))
+
+;;; Boundary
+
+(ert-deftest test-transcription-config-switcher-matches-descriptors ()
+  "Boundary: the switcher offers exactly the descriptor set.  It derives the
+list from the alist now; this is what stops a hardcoded copy coming back."
+  (should (equal (sort (test-transcription-ratio--switcher-choices) #'string<)
+                 (sort (mapcar (lambda (entry) (symbol-name (car entry)))
+                               cj/--transcription-backends)
+                       #'string<))))
+
+(ert-deftest test-transcription-config-every-descriptor-script-exists ()
+  "Boundary: every descriptor names a script that exists under scripts/.
+A descriptor for a script that never landed (a hosted alternative referenced
+from another repo, say) would fail at transcription time instead."
+  (dolist (entry cj/--transcription-backends)
+    (let ((cj/transcribe-backend (car entry)))
+      (should (file-exists-p (cj/--transcription-script-path))))))
+
+;;; Error
+
+(ert-deftest test-transcription-config-unknown-backend-still-signals ()
+  "Error: adding ratio did not loosen the descriptor lookup; an unknown
+backend still signals `user-error'."
+  (should-error (cj/--backend-plist 'no-such-backend) :type 'user-error))
+
+(provide 'test-transcription-config--ratio-backend)
+;;; test-transcription-config--ratio-backend.el ends here