feat: komplette Codebase vom Board ins Repo gespiegelt
app/ = /opt/callrec (Node-Backend, Vite-Frontend, Python-Inferenz, Audio-Skripte, rkllm_server.cpp), system/ = systemd-Units, WirePlumber/PipeWire-Drop-ins, sudoers. Modelle, Binaries, Aufnahmen und node_modules bleiben draussen (.gitignore). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,116 @@
|
||||
"""ctypes-Binding auf librknnrt.so für das wespeaker-Embedding auf der NPU (specs/007).
|
||||
|
||||
Warum kein rknn-toolkit-lite2: dessen PyPI-Wheels enden bei cp312, der R1 läuft auf
|
||||
Python 3.13 → direktes C-API-Binding nach dem bewährten loop_native.py-Muster (DEC-708).
|
||||
Struct-Layouts verifiziert gegen rknn_api.h v2.3.2 (airockchip). Läuft NUR auf dem R1
|
||||
(braucht /usr/lib/librknnrt.so 2.3.2 + Kernel-Treiber 0.9.8) — auf Windows nicht
|
||||
importier-testbar, wie alles unter device/inference/.
|
||||
|
||||
Modell: /opt/aime/models/diar/wespeaker-pyannote-fp16.rknn (Fixed-Shape,
|
||||
feats [1,998,80] float32 + weights [1,125] float32 → embs [1,256] float32;
|
||||
Konvertierung siehe specs/007 T702/T703, Toolchain aime-rknn:2.3.2 auf dem PC).
|
||||
|
||||
Nutzung:
|
||||
e = RknnEmbedding("/opt/aime/models/diar/wespeaker-pyannote-fp16.rknn")
|
||||
emb = e.run(feats, weights) # → np.ndarray [1, 256]
|
||||
e.close()
|
||||
"""
|
||||
import ctypes as C
|
||||
|
||||
import numpy as np
|
||||
|
||||
RKNN_SUCC = 0
|
||||
RKNN_TENSOR_FLOAT32 = 0
|
||||
RKNN_TENSOR_UNDEFINED_FMT = 3 # rknn_api.h: NCHW=0, NHWC=1, NC1HWC2=2, UNDEFINED=3
|
||||
RKNN_NPU_CORE_0_1_2 = 7 # alle 3 NPU-Kerne (T704b: 1,84x vs. 1 Kern, 39 ms/Chunk)
|
||||
|
||||
|
||||
class _RknnInput(C.Structure):
|
||||
_fields_ = [
|
||||
("index", C.c_uint32),
|
||||
("buf", C.c_void_p),
|
||||
("size", C.c_uint32),
|
||||
("pass_through", C.c_uint8),
|
||||
("type", C.c_int),
|
||||
("fmt", C.c_int),
|
||||
]
|
||||
|
||||
|
||||
class _RknnOutput(C.Structure):
|
||||
_fields_ = [
|
||||
("want_float", C.c_uint8),
|
||||
("is_prealloc", C.c_uint8),
|
||||
("index", C.c_uint32),
|
||||
("buf", C.c_void_p),
|
||||
("size", C.c_uint32),
|
||||
]
|
||||
|
||||
|
||||
class RknnEmbedding:
|
||||
"""Eine geladene .rknn-Instanz; run() ist NICHT thread-safe (ein Kontext)."""
|
||||
|
||||
def __init__(self, model_path: str, lib_path: str = "librknnrt.so",
|
||||
core_mask: int = RKNN_NPU_CORE_0_1_2):
|
||||
self._lib = C.CDLL(lib_path)
|
||||
self._lib.rknn_init.argtypes = [C.POINTER(C.c_uint64), C.c_void_p,
|
||||
C.c_uint32, C.c_uint32, C.c_void_p]
|
||||
self._lib.rknn_set_core_mask.argtypes = [C.c_uint64, C.c_int]
|
||||
self._lib.rknn_inputs_set.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnInput)]
|
||||
self._lib.rknn_run.argtypes = [C.c_uint64, C.c_void_p]
|
||||
self._lib.rknn_outputs_get.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnOutput), C.c_void_p]
|
||||
self._lib.rknn_outputs_release.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnOutput)]
|
||||
self._lib.rknn_destroy.argtypes = [C.c_uint64]
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
blob = f.read()
|
||||
self._ctx = C.c_uint64(0)
|
||||
buf = C.create_string_buffer(blob, len(blob))
|
||||
rc = self._lib.rknn_init(C.byref(self._ctx), buf, len(blob), 0, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_init rc={rc} (librknnrt/Treiber prüfen)")
|
||||
if core_mask: # best effort — Speed-Sache, kein Korrektheits-Thema
|
||||
self._lib.rknn_set_core_mask(self._ctx, core_mask)
|
||||
|
||||
def run(self, feats: np.ndarray, weights: np.ndarray) -> np.ndarray:
|
||||
feats = np.ascontiguousarray(feats, dtype=np.float32)
|
||||
weights = np.ascontiguousarray(weights, dtype=np.float32)
|
||||
ins = (_RknnInput * 2)()
|
||||
for i, arr in enumerate((feats, weights)):
|
||||
ins[i].index = i
|
||||
ins[i].buf = arr.ctypes.data_as(C.c_void_p)
|
||||
ins[i].size = arr.nbytes
|
||||
ins[i].pass_through = 0
|
||||
ins[i].type = RKNN_TENSOR_FLOAT32
|
||||
ins[i].fmt = RKNN_TENSOR_UNDEFINED_FMT
|
||||
rc = self._lib.rknn_inputs_set(self._ctx, 2, ins)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_inputs_set rc={rc}")
|
||||
rc = self._lib.rknn_run(self._ctx, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_run rc={rc}")
|
||||
outs = (_RknnOutput * 1)()
|
||||
outs[0].want_float = 1
|
||||
outs[0].is_prealloc = 0
|
||||
outs[0].index = 0
|
||||
rc = self._lib.rknn_outputs_get(self._ctx, 1, outs, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_outputs_get rc={rc}")
|
||||
n = outs[0].size // 4
|
||||
emb = np.ctypeslib.as_array(
|
||||
C.cast(outs[0].buf, C.POINTER(C.c_float)), shape=(n,)).copy()
|
||||
self._lib.rknn_outputs_release(self._ctx, 1, outs)
|
||||
return emb.reshape(1, -1)
|
||||
|
||||
def close(self):
|
||||
if getattr(self, "_ctx", None) and self._ctx.value:
|
||||
self._lib.rknn_destroy(self._ctx)
|
||||
self._ctx = C.c_uint64(0)
|
||||
|
||||
def __del__(self): # best effort
|
||||
try:
|
||||
self.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,131 @@
|
||||
"""ctypes-Binding auf librknnrt.so für den Whisper-turbo-Encoder auf der NPU (specs/009 Option C).
|
||||
|
||||
Muster: rknpu_embed.py (specs/007, DEC-708 — kein rknn-toolkit-lite2, cp313-Lücke).
|
||||
Läuft NUR auf dem R1 (librknnrt.so 2.3.2 + Kernel-Treiber 0.9.8); auf Windows nicht
|
||||
importier-testbar, wie alles unter device/inference/.
|
||||
|
||||
Modell: /opt/aime/models/asr/turbo-encoder-fp16.rknn (Fixed-Shape,
|
||||
mel [1, 128, 3000] float32 → enc_out [1, 1500, 1280] float32 = ein 30-s-Fenster;
|
||||
Konvertierung: aime-rknn:2.3.2-Docker auf dem PC, specs/009 T903).
|
||||
|
||||
Integrations-Naht: faster-whisper `WhisperModel.encode()` ist die EINZIGE
|
||||
Encoder-Stelle (auch für BatchedInferencePipeline); der Transcriber ersetzt sie
|
||||
bei `transcription.compute: npu` durch run() + get_ctranslate2_storage.
|
||||
Batch-Eingaben (B, 128, 3000) laufen als Schleife über das statische 1er-Modell.
|
||||
|
||||
Nutzung:
|
||||
e = RknnWhisperEncoder("/opt/aime/models/asr/turbo-encoder-fp16.rknn")
|
||||
out = e.run(mel) # (B, 128, 3000) float32 → (B, 1500, 1280) float32
|
||||
e.close()
|
||||
"""
|
||||
import ctypes as C
|
||||
|
||||
import numpy as np
|
||||
|
||||
RKNN_SUCC = 0
|
||||
RKNN_TENSOR_FLOAT32 = 0
|
||||
RKNN_TENSOR_UNDEFINED_FMT = 3 # rknn_api.h: NCHW=0, NHWC=1, NC1HWC2=2, UNDEFINED=3
|
||||
RKNN_NPU_CORE_0_1_2 = 7 # alle 3 NPU-Kerne
|
||||
|
||||
MEL_SHAPE = (128, 3000)
|
||||
OUT_SHAPE = (1500, 1280)
|
||||
|
||||
|
||||
class _RknnInput(C.Structure):
|
||||
_fields_ = [
|
||||
("index", C.c_uint32),
|
||||
("buf", C.c_void_p),
|
||||
("size", C.c_uint32),
|
||||
("pass_through", C.c_uint8),
|
||||
("type", C.c_int),
|
||||
("fmt", C.c_int),
|
||||
]
|
||||
|
||||
|
||||
class _RknnOutput(C.Structure):
|
||||
_fields_ = [
|
||||
("want_float", C.c_uint8),
|
||||
("is_prealloc", C.c_uint8),
|
||||
("index", C.c_uint32),
|
||||
("buf", C.c_void_p),
|
||||
("size", C.c_uint32),
|
||||
]
|
||||
|
||||
|
||||
class RknnWhisperEncoder:
|
||||
"""Eine geladene .rknn-Instanz; run() ist NICHT thread-safe (ein Kontext)."""
|
||||
|
||||
def __init__(self, model_path: str, lib_path: str = "librknnrt.so",
|
||||
core_mask: int = RKNN_NPU_CORE_0_1_2):
|
||||
self._lib = C.CDLL(lib_path)
|
||||
self._lib.rknn_init.argtypes = [C.POINTER(C.c_uint64), C.c_void_p,
|
||||
C.c_uint32, C.c_uint32, C.c_void_p]
|
||||
self._lib.rknn_set_core_mask.argtypes = [C.c_uint64, C.c_int]
|
||||
self._lib.rknn_inputs_set.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnInput)]
|
||||
self._lib.rknn_run.argtypes = [C.c_uint64, C.c_void_p]
|
||||
self._lib.rknn_outputs_get.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnOutput), C.c_void_p]
|
||||
self._lib.rknn_outputs_release.argtypes = [C.c_uint64, C.c_uint32,
|
||||
C.POINTER(_RknnOutput)]
|
||||
self._lib.rknn_destroy.argtypes = [C.c_uint64]
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
blob = f.read()
|
||||
self._ctx = C.c_uint64(0)
|
||||
buf = C.create_string_buffer(blob, len(blob))
|
||||
rc = self._lib.rknn_init(C.byref(self._ctx), buf, len(blob), 0, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_init rc={rc} (librknnrt/Treiber prüfen)")
|
||||
if core_mask: # best effort — Speed-Sache, kein Korrektheits-Thema
|
||||
self._lib.rknn_set_core_mask(self._ctx, core_mask)
|
||||
|
||||
def _run_one(self, mel: np.ndarray) -> np.ndarray:
|
||||
ins = (_RknnInput * 1)()
|
||||
ins[0].index = 0
|
||||
ins[0].buf = mel.ctypes.data_as(C.c_void_p)
|
||||
ins[0].size = mel.nbytes
|
||||
ins[0].pass_through = 0
|
||||
ins[0].type = RKNN_TENSOR_FLOAT32
|
||||
ins[0].fmt = RKNN_TENSOR_UNDEFINED_FMT
|
||||
rc = self._lib.rknn_inputs_set(self._ctx, 1, ins)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_inputs_set rc={rc}")
|
||||
rc = self._lib.rknn_run(self._ctx, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_run rc={rc}")
|
||||
outs = (_RknnOutput * 1)()
|
||||
outs[0].want_float = 1
|
||||
outs[0].is_prealloc = 0
|
||||
outs[0].index = 0
|
||||
rc = self._lib.rknn_outputs_get(self._ctx, 1, outs, None)
|
||||
if rc != RKNN_SUCC:
|
||||
raise RuntimeError(f"rknn_outputs_get rc={rc}")
|
||||
n = outs[0].size // 4
|
||||
flat = np.ctypeslib.as_array(
|
||||
C.cast(outs[0].buf, C.POINTER(C.c_float)), shape=(n,)).copy()
|
||||
self._lib.rknn_outputs_release(self._ctx, 1, outs)
|
||||
return flat.reshape(OUT_SHAPE)
|
||||
|
||||
def run(self, mel: np.ndarray) -> np.ndarray:
|
||||
"""(B, 128, 3000) oder (128, 3000) float32 → (B, 1500, 1280) float32."""
|
||||
mel = np.ascontiguousarray(mel, dtype=np.float32)
|
||||
if mel.ndim == 2:
|
||||
mel = mel[np.newaxis]
|
||||
if mel.shape[1:] != MEL_SHAPE:
|
||||
raise ValueError(f"mel shape {mel.shape} != (B,)+{MEL_SHAPE}")
|
||||
out = np.empty((mel.shape[0],) + OUT_SHAPE, dtype=np.float32)
|
||||
for i in range(mel.shape[0]):
|
||||
out[i] = self._run_one(np.ascontiguousarray(mel[i]))
|
||||
return out
|
||||
|
||||
def close(self):
|
||||
if getattr(self, "_ctx", None) and self._ctx.value:
|
||||
self._lib.rknn_destroy(self._ctx)
|
||||
self._ctx = C.c_uint64(0)
|
||||
|
||||
def __del__(self): # best effort
|
||||
try:
|
||||
self.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,23 @@
|
||||
import sys, time
|
||||
import numpy as np
|
||||
sys.path.insert(0, "/opt/callrec")
|
||||
from inference.rknpu_whisper_encoder import RknnWhisperEncoder
|
||||
|
||||
M = "/opt/callrec/models/turbo-encoder-fp16.rknn"
|
||||
print("== init ==", flush=True)
|
||||
t0 = time.time()
|
||||
e = RknnWhisperEncoder(M, lib_path="/usr/lib/librknnrt.so")
|
||||
print(f"init OK in {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
rng = np.random.default_rng(9)
|
||||
mel = (rng.standard_normal((1,128,3000))*0.5).astype(np.float32)
|
||||
print("== run ==", flush=True)
|
||||
t0 = time.time()
|
||||
out = e.run(mel)
|
||||
dt = time.time()-t0
|
||||
print(f"run OK in {dt:.2f}s out.shape={out.shape} dtype={out.dtype}", flush=True)
|
||||
print(f"out mean={out.mean():.4f} std={out.std():.4f} min={out.min():.3f} max={out.max():.3f}", flush=True)
|
||||
# second run (warm) timing
|
||||
t0=time.time(); e.run(mel); print(f"warm run {time.time()-t0:.2f}s", flush=True)
|
||||
e.close()
|
||||
print("== done ==", flush=True)
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/opt/aime-venv/bin/python
|
||||
"""Batched NPU transcriber A/B test: wav -> JSON words on stdout.
|
||||
large-v3-turbo decoder CPU (int8), encoder on RK3588 NPU (rknn fp16),
|
||||
faster-whisper BatchedInferencePipeline (Fabio-style batched+NPU).
|
||||
Usage: transcribe_batched.py <wav> [batch_size]
|
||||
"""
|
||||
import sys, json, time
|
||||
import numpy as np
|
||||
|
||||
WAV = sys.argv[1]
|
||||
BATCH = int(sys.argv[2]) if len(sys.argv) > 2 else 8
|
||||
MODEL = "large-v3-turbo"
|
||||
ROOT = "/opt/callrec/models/whisper"
|
||||
RKNN = "/opt/callrec/models/turbo-encoder-fp16.rknn"
|
||||
sys.path.insert(0, "/opt/callrec")
|
||||
|
||||
from faster_whisper import WhisperModel, BatchedInferencePipeline
|
||||
from faster_whisper.transcribe import get_ctranslate2_storage
|
||||
from inference.rknpu_whisper_encoder import RknnWhisperEncoder
|
||||
|
||||
t0 = time.time()
|
||||
model = WhisperModel(MODEL, device="cpu", compute_type="int8",
|
||||
download_root=ROOT, cpu_threads=4)
|
||||
npu = RknnWhisperEncoder(RKNN, lib_path="/usr/lib/librknnrt.so")
|
||||
def npu_encode(features):
|
||||
feats = np.asarray(features, dtype=np.float32)
|
||||
out = npu.run(feats)
|
||||
return get_ctranslate2_storage(np.ascontiguousarray(out))
|
||||
model.encode = npu_encode
|
||||
|
||||
batched = BatchedInferencePipeline(model=model)
|
||||
t_load = time.time() - t0
|
||||
|
||||
t0 = time.time()
|
||||
segments, info = batched.transcribe(WAV, language="de", beam_size=1,
|
||||
batch_size=BATCH, word_timestamps=True)
|
||||
words = []
|
||||
texts = []
|
||||
for seg in segments:
|
||||
texts.append(seg.text.strip())
|
||||
for w in (seg.words or []):
|
||||
words.append({
|
||||
"word": w.word.strip(),
|
||||
"t_start_ms": int(round((w.start or 0) * 1000)),
|
||||
"t_end_ms": int(round((w.end or 0) * 1000)),
|
||||
"probability": round(w.probability, 3),
|
||||
})
|
||||
t_trans = time.time() - t0
|
||||
out = {
|
||||
"text": " ".join(texts).strip(),
|
||||
"n_words": len(words),
|
||||
"duration": round(float(info.duration or 0), 1),
|
||||
"load_s": round(t_load, 1),
|
||||
"transcribe_s": round(t_trans, 1),
|
||||
"rtf": round(t_trans / max(float(info.duration or 0.1), 0.1), 3),
|
||||
"batch": BATCH,
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/opt/aime-venv/bin/python
|
||||
"""CallRec NPU transcriber: wav -> JSON words on stdout.
|
||||
large-v3-turbo decoder on CPU (int8), encoder on RK3588 NPU (rknn fp16).
|
||||
Output matches what recorder.js consumes: {"text","words":[{word,t_start_ms,t_end_ms,probability}]}.
|
||||
Usage: transcribe_cli.py <wav> [--cpu] (--cpu = no NPU, plain CPU encoder for A/B)
|
||||
"""
|
||||
import sys, json, time
|
||||
import numpy as np
|
||||
|
||||
WAV = sys.argv[1]
|
||||
USE_NPU = "--cpu" not in sys.argv[2:]
|
||||
import os
|
||||
BATCH = int(os.environ.get("WHISPER_BATCH", "8"))
|
||||
MODEL = "large-v3-turbo"
|
||||
ROOT = "/opt/callrec/models/whisper"
|
||||
RKNN = "/opt/callrec/models/turbo-encoder-fp16.rknn"
|
||||
sys.path.insert(0, "/opt/callrec")
|
||||
|
||||
from faster_whisper import WhisperModel, BatchedInferencePipeline
|
||||
|
||||
t0 = time.time()
|
||||
model = WhisperModel(MODEL, device="cpu", compute_type="int8",
|
||||
download_root=ROOT, cpu_threads=4)
|
||||
|
||||
if USE_NPU:
|
||||
from faster_whisper.transcribe import get_ctranslate2_storage
|
||||
from inference.rknpu_whisper_encoder import RknnWhisperEncoder
|
||||
npu = RknnWhisperEncoder(RKNN, lib_path="/usr/lib/librknnrt.so")
|
||||
def npu_encode(features):
|
||||
feats = np.asarray(features, dtype=np.float32)
|
||||
out = npu.run(feats)
|
||||
return get_ctranslate2_storage(np.ascontiguousarray(out))
|
||||
model.encode = npu_encode
|
||||
|
||||
engine = BatchedInferencePipeline(model=model) if BATCH > 1 else model
|
||||
t_load = time.time() - t0
|
||||
t0 = time.time()
|
||||
if BATCH > 1:
|
||||
segments, info = engine.transcribe(WAV, language="de", beam_size=1,
|
||||
batch_size=BATCH, word_timestamps=True)
|
||||
else:
|
||||
segments, info = engine.transcribe(WAV, language="de", beam_size=1,
|
||||
word_timestamps=True, vad_filter=True)
|
||||
words = []
|
||||
texts = []
|
||||
for seg in segments:
|
||||
texts.append(seg.text.strip())
|
||||
for w in (seg.words or []):
|
||||
words.append({
|
||||
"word": w.word.strip(),
|
||||
"t_start_ms": int(round((w.start or 0) * 1000)),
|
||||
"t_end_ms": int(round((w.end or 0) * 1000)),
|
||||
"probability": round(w.probability, 3),
|
||||
})
|
||||
t_trans = time.time() - t0
|
||||
out = {
|
||||
"text": " ".join(texts).strip(),
|
||||
"words": words,
|
||||
"duration": round(float(info.duration or 0), 1),
|
||||
"load_s": round(t_load, 1),
|
||||
"transcribe_s": round(t_trans, 1),
|
||||
"rtf": round(t_trans / max(float(info.duration or 0.1), 0.1), 3),
|
||||
"compute": "npu" if USE_NPU else "cpu",
|
||||
"batch": BATCH,
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/opt/aime-venv/bin/python
|
||||
"""CallRec persistent transcriber daemon.
|
||||
|
||||
Loads the faster-whisper large-v3-turbo CT2 model (decoder, CPU) ONCE and keeps
|
||||
it warm for the whole callrec lifetime — this kills the ~22s CT2 load + ~14s
|
||||
import cost that the old per-call `transcribe_cli.py` paid on every recording.
|
||||
|
||||
The NPU turbo ENCODER (rknn, 1.35 GB NPU memory) is acquired per request and
|
||||
RELEASED right after, because the RK3588 NPU cannot hold the encoder and the
|
||||
Qwen-3B LLM (rkllm_server) at the same time (proven: mali OOM). Freeing it here
|
||||
lets rkllm load for the analysis that follows. The rknn file stays in the page
|
||||
cache, so re-init after the first request is a few seconds, not the cold ~17s.
|
||||
|
||||
Protocol (line based, like spk_embed / rkllm_server):
|
||||
stdin : one absolute wav path per line
|
||||
stdout: "READY" once, then per request "OK <base64(json)>" or "ERR <base64(msg)>"
|
||||
The JSON is base64-wrapped so newlines inside it never break the line protocol.
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, "/opt/callrec")
|
||||
|
||||
MODEL = "large-v3-turbo"
|
||||
ROOT = "/opt/callrec/models/whisper"
|
||||
RKNN = "/opt/callrec/models/turbo-encoder-fp16.rknn"
|
||||
|
||||
from faster_whisper import WhisperModel
|
||||
from faster_whisper.transcribe import get_ctranslate2_storage
|
||||
from inference.rknpu_whisper_encoder import RknnWhisperEncoder
|
||||
|
||||
# CT2 decoder loaded ONCE, stays warm.
|
||||
_model = WhisperModel(MODEL, device="cpu", compute_type="int8",
|
||||
download_root=ROOT, cpu_threads=4)
|
||||
|
||||
|
||||
def transcribe(wav):
|
||||
t_load = time.time()
|
||||
npu = RknnWhisperEncoder(RKNN, lib_path="/usr/lib/librknnrt.so")
|
||||
load_s = time.time() - t_load
|
||||
|
||||
def npu_encode(features):
|
||||
feats = np.asarray(features, dtype=np.float32)
|
||||
return get_ctranslate2_storage(np.ascontiguousarray(npu.run(feats)))
|
||||
|
||||
_model.encode = npu_encode
|
||||
try:
|
||||
t0 = time.time()
|
||||
segments, info = _model.transcribe(
|
||||
wav, language="de", beam_size=1, word_timestamps=True, vad_filter=True)
|
||||
words = []
|
||||
texts = []
|
||||
for seg in segments:
|
||||
texts.append(seg.text.strip())
|
||||
for w in (seg.words or []):
|
||||
words.append({
|
||||
"word": w.word.strip(),
|
||||
"t_start_ms": int(round((w.start or 0) * 1000)),
|
||||
"t_end_ms": int(round((w.end or 0) * 1000)),
|
||||
"probability": round(w.probability, 3),
|
||||
})
|
||||
dur = float(info.duration or 0)
|
||||
trans_s = time.time() - t0
|
||||
return {
|
||||
"text": " ".join(texts).strip(),
|
||||
"words": words,
|
||||
"duration": round(dur, 1),
|
||||
"load_s": round(load_s, 1),
|
||||
"transcribe_s": round(trans_s, 1),
|
||||
"rtf": round(trans_s / max(dur, 0.1), 3),
|
||||
"compute": "npu",
|
||||
}
|
||||
finally:
|
||||
# Release the NPU encoder so rkllm can load for the analysis step.
|
||||
npu.close()
|
||||
|
||||
|
||||
def main():
|
||||
print("READY", flush=True)
|
||||
for line in sys.stdin:
|
||||
wav = line.strip()
|
||||
if not wav:
|
||||
continue
|
||||
try:
|
||||
out = transcribe(wav)
|
||||
payload = base64.b64encode(
|
||||
json.dumps(out, ensure_ascii=False).encode("utf-8")).decode("ascii")
|
||||
print("OK " + payload, flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = base64.b64encode(str(e).encode("utf-8")).decode("ascii")
|
||||
print("ERR " + msg, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/opt/aime-venv/bin/python
|
||||
"""CallRec CPU transcriber daemon (SHORT clips).
|
||||
|
||||
Loads faster-whisper large-v3-turbo (CT2, CPU int8) ONCE and keeps it warm for
|
||||
the whole callrec lifetime. Used for SHORT recordings where per-call model load
|
||||
(~28-40s) dwarfs the transcription itself. CPU-ONLY — it never touches the NPU,
|
||||
so it coexists with a resident rkllm_server (Qwen-3B): short clips need NO
|
||||
NPU-encoder swap AND no model reload → a 10s clip drops from ~120s to ~20s.
|
||||
|
||||
(The NPU turbo-encoder path stays in transcribe_cli.py for LONG clips, where the
|
||||
2.4x encoder speedup beats the swap overhead.)
|
||||
|
||||
Protocol (line based, like the NPU daemon / spk_embed / rkllm_server):
|
||||
stdin : one absolute wav path per line
|
||||
stdout: "READY" once, then per request "OK <base64(json)>" or "ERR <base64(msg)>"
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
import os
|
||||
|
||||
sys.path.insert(0, "/opt/callrec")
|
||||
|
||||
MODEL = "large-v3-turbo"
|
||||
ROOT = "/opt/callrec/models/whisper"
|
||||
BATCH = int(os.environ.get("WHISPER_BATCH", "8"))
|
||||
|
||||
from faster_whisper import WhisperModel, BatchedInferencePipeline
|
||||
|
||||
# CT2 model loaded ONCE (CPU), stays warm. No NPU encoder override -> ct2 runs
|
||||
# its own encoder on the CPU, so nothing here competes with rkllm on the NPU.
|
||||
_model = WhisperModel(MODEL, device="cpu", compute_type="int8",
|
||||
download_root=ROOT, cpu_threads=4)
|
||||
_engine = BatchedInferencePipeline(model=_model) if BATCH > 1 else _model
|
||||
|
||||
|
||||
def transcribe(wav):
|
||||
t0 = time.time()
|
||||
if BATCH > 1:
|
||||
segments, info = _engine.transcribe(
|
||||
wav, language="de", beam_size=1, batch_size=BATCH, word_timestamps=True)
|
||||
else:
|
||||
segments, info = _engine.transcribe(
|
||||
wav, language="de", beam_size=1, word_timestamps=True, vad_filter=True)
|
||||
words = []
|
||||
texts = []
|
||||
for seg in segments:
|
||||
texts.append(seg.text.strip())
|
||||
for w in (seg.words or []):
|
||||
words.append({
|
||||
"word": w.word.strip(),
|
||||
"t_start_ms": int(round((w.start or 0) * 1000)),
|
||||
"t_end_ms": int(round((w.end or 0) * 1000)),
|
||||
"probability": round(w.probability, 3),
|
||||
})
|
||||
dur = float(info.duration or 0)
|
||||
trans_s = time.time() - t0
|
||||
return {
|
||||
"text": " ".join(texts).strip(),
|
||||
"words": words,
|
||||
"duration": round(dur, 1),
|
||||
"load_s": 0.0,
|
||||
"transcribe_s": round(trans_s, 1),
|
||||
"rtf": round(trans_s / max(dur, 0.1), 3),
|
||||
"compute": "cpu-daemon",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
print("READY", flush=True)
|
||||
for line in sys.stdin:
|
||||
wav = line.strip()
|
||||
if not wav:
|
||||
continue
|
||||
try:
|
||||
out = transcribe(wav)
|
||||
payload = base64.b64encode(
|
||||
json.dumps(out, ensure_ascii=False).encode("utf-8")).decode("ascii")
|
||||
print("OK " + payload, flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = base64.b64encode(str(e).encode("utf-8")).decode("ascii")
|
||||
print("ERR " + msg, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user