From b97a973b7cd32c54cf82fee6ead00fdc3377f88c Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 16 Jul 2026 20:55:34 +0300 Subject: [PATCH 01/44] pkg/connector/wadb: store MatrixRTC call state --- pkg/connector/wadb/call.go | 241 ++++++++++++++++++ pkg/connector/wadb/database.go | 19 +- .../wadb/upgrades/00-latest-schema.sql | 35 ++- .../wadb/upgrades/10-matrixrtc-call.sql | 33 +++ 4 files changed, 321 insertions(+), 7 deletions(-) create mode 100644 pkg/connector/wadb/call.go create mode 100644 pkg/connector/wadb/upgrades/10-matrixrtc-call.sql diff --git a/pkg/connector/wadb/call.go b/pkg/connector/wadb/call.go new file mode 100644 index 0000000..804abb5 --- /dev/null +++ b/pkg/connector/wadb/call.go @@ -0,0 +1,241 @@ +package wadb + +import ( + "context" + "database/sql" + "time" + + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/id" +) + +type MatrixRTCCallQuery struct { + BridgeID networkid.BridgeID + *dbutil.QueryHelper[*MatrixRTCCall] +} + +type MatrixRTCCall struct { + BridgeID networkid.BridgeID + UserLoginID networkid.UserLoginID + WACallID string + RoomID id.RoomID + PortalKey networkid.PortalKey + PeerJID types.JID + Direction string + MediaKind string + FocusType string + LiveKitServiceURL string + LiveKitRoom string + MatrixParticipantMXID id.UserID + MatrixSessionID string + SelectedPublisherID string + AudioPolicy string + State string + CreatedTS time.Time + JoinedTS time.Time + AnsweredTS time.Time + EndedTS time.Time + EndReason string + LastError string +} + +const ( + upsertMatrixRTCCallQuery = ` + INSERT INTO whatsapp_matrixrtc_call ( + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) + ON CONFLICT (bridge_id, user_login_id, wa_call_id) DO UPDATE SET + room_id=excluded.room_id, + portal_id=excluded.portal_id, + portal_receiver=excluded.portal_receiver, + peer_jid=excluded.peer_jid, + direction=excluded.direction, + media_kind=excluded.media_kind, + focus_type=excluded.focus_type, + livekit_service_url=excluded.livekit_service_url, + livekit_room=excluded.livekit_room, + matrix_participant_mxid=excluded.matrix_participant_mxid, + matrix_session_id=excluded.matrix_session_id, + selected_publisher_id=excluded.selected_publisher_id, + audio_policy=excluded.audio_policy, + state=excluded.state, + joined_ts=excluded.joined_ts, + answered_ts=excluded.answered_ts, + ended_ts=excluded.ended_ts, + end_reason=excluded.end_reason, + last_error=excluded.last_error + ` + getMatrixRTCCallQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` + getActiveMatrixRTCCallsForLoginQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND ended_ts IS NULL + ` + getActiveMatrixRTCCallsInRoomQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND room_id=$2 AND ended_ts IS NULL + ` + markMatrixRTCCallEndedQuery = ` + UPDATE whatsapp_matrixrtc_call + SET state=$4, ended_ts=$5, end_reason=$6, last_error=$7 + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` + deleteMatrixRTCCallQuery = ` + DELETE FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` +) + +func (cq *MatrixRTCCallQuery) Put(ctx context.Context, call *MatrixRTCCall) error { + call.BridgeID = cq.BridgeID + return cq.Exec(ctx, upsertMatrixRTCCallQuery, call.sqlVariables()...) +} + +func (cq *MatrixRTCCallQuery) Get(ctx context.Context, loginID networkid.UserLoginID, waCallID string) (*MatrixRTCCall, error) { + return cq.QueryOne(ctx, getMatrixRTCCallQuery, cq.BridgeID, loginID, waCallID) +} + +func (cq *MatrixRTCCallQuery) GetActiveForLogin(ctx context.Context, loginID networkid.UserLoginID) ([]*MatrixRTCCall, error) { + return cq.QueryMany(ctx, getActiveMatrixRTCCallsForLoginQuery, cq.BridgeID, loginID) +} + +func (cq *MatrixRTCCallQuery) GetActiveInRoom(ctx context.Context, roomID id.RoomID) ([]*MatrixRTCCall, error) { + return cq.QueryMany(ctx, getActiveMatrixRTCCallsInRoomQuery, cq.BridgeID, roomID) +} + +func (cq *MatrixRTCCallQuery) MarkEnded(ctx context.Context, loginID networkid.UserLoginID, waCallID, state, reason, lastError string, ended time.Time) error { + return cq.Exec(ctx, markMatrixRTCCallEndedQuery, cq.BridgeID, loginID, waCallID, state, nullableUnix(ended), reason, lastError) +} + +func (cq *MatrixRTCCallQuery) Delete(ctx context.Context, loginID networkid.UserLoginID, waCallID string) error { + return cq.Exec(ctx, deleteMatrixRTCCallQuery, cq.BridgeID, loginID, waCallID) +} + +func (call *MatrixRTCCall) Scan(row dbutil.Scannable) (*MatrixRTCCall, error) { + var liveKitRoom, participantMXID, matrixSessionID, selectedPublisherID, endReason, lastError sql.NullString + var joinedTS, answeredTS, endedTS sql.NullInt64 + var createdTS int64 + err := row.Scan( + &call.BridgeID, + &call.UserLoginID, + &call.WACallID, + &call.RoomID, + &call.PortalKey.ID, + &call.PortalKey.Receiver, + &call.PeerJID, + &call.Direction, + &call.MediaKind, + &call.FocusType, + &call.LiveKitServiceURL, + &liveKitRoom, + &participantMXID, + &matrixSessionID, + &selectedPublisherID, + &call.AudioPolicy, + &call.State, + &createdTS, + &joinedTS, + &answeredTS, + &endedTS, + &endReason, + &lastError, + ) + if err != nil { + return nil, err + } + call.CreatedTS = unixToTime(createdTS) + call.JoinedTS = nullUnixToTime(joinedTS) + call.AnsweredTS = nullUnixToTime(answeredTS) + call.EndedTS = nullUnixToTime(endedTS) + call.LiveKitRoom = liveKitRoom.String + call.MatrixParticipantMXID = id.UserID(participantMXID.String) + call.MatrixSessionID = matrixSessionID.String + call.SelectedPublisherID = selectedPublisherID.String + call.EndReason = endReason.String + call.LastError = lastError.String + return call, nil +} + +func (call *MatrixRTCCall) sqlVariables() []any { + return []any{ + call.BridgeID, + call.UserLoginID, + call.WACallID, + call.RoomID, + call.PortalKey.ID, + call.PortalKey.Receiver, + call.PeerJID, + call.Direction, + call.MediaKind, + call.FocusType, + call.LiveKitServiceURL, + nullString(call.LiveKitRoom), + nullString(string(call.MatrixParticipantMXID)), + nullString(call.MatrixSessionID), + nullString(call.SelectedPublisherID), + call.AudioPolicy, + call.State, + nullableUnix(call.CreatedTS), + nullableUnix(call.JoinedTS), + nullableUnix(call.AnsweredTS), + nullableUnix(call.EndedTS), + nullString(call.EndReason), + nullString(call.LastError), + } +} + +func nullString(str string) *string { + if str == "" { + return nil + } + return &str +} + +func nullableUnix(ts time.Time) *int64 { + if ts.IsZero() { + return nil + } + unix := ts.Unix() + return &unix +} + +func unixToTime(ts int64) time.Time { + if ts == 0 { + return time.Time{} + } + return time.Unix(ts, 0) +} + +func nullUnixToTime(ts sql.NullInt64) time.Time { + if !ts.Valid { + return time.Time{} + } + return unixToTime(ts.Int64) +} diff --git a/pkg/connector/wadb/database.go b/pkg/connector/wadb/database.go index 9d13568..2ba92eb 100644 --- a/pkg/connector/wadb/database.go +++ b/pkg/connector/wadb/database.go @@ -10,12 +10,13 @@ import ( type Database struct { *dbutil.Database - Conversation *ConversationQuery - Message *MessageQuery - PollOption *PollOptionQuery - MediaRequest *MediaRequestQuery - HSNotif *HistorySyncNotificationQuery - AvatarCache *AvatarCacheQuery + Conversation *ConversationQuery + Message *MessageQuery + PollOption *PollOptionQuery + MediaRequest *MediaRequestQuery + HSNotif *HistorySyncNotificationQuery + AvatarCache *AvatarCacheQuery + MatrixRTCCall *MatrixRTCCallQuery } func New(bridgeID networkid.BridgeID, db *dbutil.Database, log zerolog.Logger) *Database { @@ -51,5 +52,11 @@ func New(bridgeID networkid.BridgeID, db *dbutil.Database, log zerolog.Logger) * return &AvatarCacheEntry{} }), }, + MatrixRTCCall: &MatrixRTCCallQuery{ + BridgeID: bridgeID, + QueryHelper: dbutil.MakeQueryHelper(db, func(_ *dbutil.QueryHelper[*MatrixRTCCall]) *MatrixRTCCall { + return &MatrixRTCCall{} + }), + }, } } diff --git a/pkg/connector/wadb/upgrades/00-latest-schema.sql b/pkg/connector/wadb/upgrades/00-latest-schema.sql index 850f5b4..ba6ada4 100644 --- a/pkg/connector/wadb/upgrades/00-latest-schema.sql +++ b/pkg/connector/wadb/upgrades/00-latest-schema.sql @@ -1,4 +1,4 @@ --- v0 -> v9 (compatible with v3+): Latest revision +-- v0 -> v10 (compatible with v3+): Latest revision CREATE TABLE whatsapp_poll_option_id ( bridge_id TEXT NOT NULL, @@ -98,3 +98,36 @@ CREATE TABLE whatsapp_avatar_cache ( PRIMARY KEY (entity_jid, avatar_id) ); + +CREATE TABLE whatsapp_matrixrtc_call ( + bridge_id TEXT NOT NULL, + user_login_id TEXT NOT NULL, + wa_call_id TEXT NOT NULL, + room_id TEXT NOT NULL, + portal_id TEXT NOT NULL, + portal_receiver TEXT NOT NULL, + peer_jid TEXT NOT NULL, + direction TEXT NOT NULL, + media_kind TEXT NOT NULL, + focus_type TEXT NOT NULL, + livekit_service_url TEXT NOT NULL, + livekit_room TEXT, + matrix_participant_mxid TEXT, + matrix_session_id TEXT, + selected_publisher_id TEXT, + audio_policy TEXT NOT NULL, + state TEXT NOT NULL, + created_ts BIGINT NOT NULL, + joined_ts BIGINT, + answered_ts BIGINT, + ended_ts BIGINT, + end_reason TEXT, + last_error TEXT, + + PRIMARY KEY (bridge_id, user_login_id, wa_call_id), + CONSTRAINT whatsapp_matrixrtc_call_user_login_fkey FOREIGN KEY (bridge_id, user_login_id) + REFERENCES user_login (bridge_id, id) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT whatsapp_matrixrtc_call_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver) + REFERENCES portal (bridge_id, id, receiver) ON UPDATE CASCADE ON DELETE CASCADE +); +CREATE INDEX whatsapp_matrixrtc_call_room_idx ON whatsapp_matrixrtc_call (bridge_id, room_id, state); diff --git a/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql b/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql new file mode 100644 index 0000000..0aba691 --- /dev/null +++ b/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql @@ -0,0 +1,33 @@ +-- v10 (compatible with v3+): Add MatrixRTC/LiveKit call metadata +CREATE TABLE whatsapp_matrixrtc_call ( + bridge_id TEXT NOT NULL, + user_login_id TEXT NOT NULL, + wa_call_id TEXT NOT NULL, + room_id TEXT NOT NULL, + portal_id TEXT NOT NULL, + portal_receiver TEXT NOT NULL, + peer_jid TEXT NOT NULL, + direction TEXT NOT NULL, + media_kind TEXT NOT NULL, + focus_type TEXT NOT NULL, + livekit_service_url TEXT NOT NULL, + livekit_room TEXT, + matrix_participant_mxid TEXT, + matrix_session_id TEXT, + selected_publisher_id TEXT, + audio_policy TEXT NOT NULL, + state TEXT NOT NULL, + created_ts BIGINT NOT NULL, + joined_ts BIGINT, + answered_ts BIGINT, + ended_ts BIGINT, + end_reason TEXT, + last_error TEXT, + + PRIMARY KEY (bridge_id, user_login_id, wa_call_id), + CONSTRAINT whatsapp_matrixrtc_call_user_login_fkey FOREIGN KEY (bridge_id, user_login_id) + REFERENCES user_login (bridge_id, id) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT whatsapp_matrixrtc_call_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver) + REFERENCES portal (bridge_id, id, receiver) ON UPDATE CASCADE ON DELETE CASCADE +); +CREATE INDEX whatsapp_matrixrtc_call_room_idx ON whatsapp_matrixrtc_call (bridge_id, room_id, state); From 4833b867c211d7ac5873963e5d03b7831d6b04d9 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 16 Jul 2026 21:04:11 +0300 Subject: [PATCH 02/44] go.mod: add MatrixRTC call dependencies --- go.mod | 79 +++++++++++++++++++- go.sum | 231 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 294 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 35a6655..795e050 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.mau.fi/mautrix-whatsapp -go 1.25.0 +go 1.26 toolchain go1.26.4 @@ -8,6 +8,11 @@ tool go.mau.fi/util/cmd/maubuild require ( github.com/lib/pq v1.12.3 + github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 + github.com/livekit/protocol v1.49.0 + github.com/livekit/server-sdk-go/v2 v2.17.0 + github.com/pion/webrtc/v4 v4.2.14 + github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 @@ -22,33 +27,99 @@ require ( ) require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + buf.build/go/protoyaml v0.7.0 // indirect + cel.dev/expr v0.25.2 // indirect filippo.io/edwards25519 v1.2.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/at-wat/ebml-go v0.18.0 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bep/debounce v1.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coder/websocket v1.8.15 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/dennwc/iters v1.2.2 // indirect github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect + github.com/frostbyte73/core v0.1.1 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/gammazero/deque v1.2.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/cel-go v0.28.1 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/hajimehoshi/go-mp3 v0.3.4 // indirect + github.com/jxskiss/base62 v1.1.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lithammer/shortuuid/v4 v4.2.0 // indirect + github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 // indirect + github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e // indirect + github.com/livekit/psrpc v0.7.2 // indirect + github.com/mackerelio/go-osstat v0.2.7 // indirect + github.com/magefile/mage v1.17.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.45 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/dtls/v3 v3.1.4 // indirect + github.com/pion/ice/v4 v4.2.7 // indirect + github.com/pion/interceptor v0.1.45 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/opus v0.1.0 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.16 // indirect + github.com/pion/rtp v1.10.2 // indirect + github.com/pion/sctp v1.10.0 // indirect + github.com/pion/sdp/v3 v3.0.18 // indirect + github.com/pion/srtp/v3 v3.0.11 // indirect + github.com/pion/stun/v3 v3.1.4 // indirect + github.com/pion/transport/v4 v4.0.2 // indirect + github.com/pion/turn/v5 v5.0.8 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.68.1 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/redis/go-redis/v9 v9.20.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + github.com/twitchtv/twirp v8.1.3+incompatible // indirect github.com/vektah/gqlparser/v2 v2.5.27 // indirect + github.com/wlynxg/anet v0.0.5 // indirect github.com/yuin/goldmark v1.8.2 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mau.fi/libsignal v0.2.2 // indirect go.mau.fi/zeroconfig v0.2.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect + gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 3d0391c..7ddfb5f 100644 --- a/go.sum +++ b/go.sum @@ -1,55 +1,219 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/at-wat/ebml-go v0.18.0 h1:SNkpBFR4jCQV1rI4Bm1tSuIYnusxe2qQ4GHJia9eQg4= +github.com/at-wat/ebml-go v0.18.0/go.mod h1:w1cJs7zmGsb5nnSvhWGKLCxvfu4FVx5ERvYDIalj1ww= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= +github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elliotchance/orderedmap/v3 v3.1.0 h1:j4DJ5ObEmMBt/lcwIecKcoRxIQUEnw0L804lXYDt/pg= github.com/elliotchance/orderedmap/v3 v3.1.0/go.mod h1:G+Hc2RwaZvJMcS4JpGCOyViCnGeKf0bTYCGTO4uhjSo= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= +github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= +github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= +github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= +github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hajimehoshi/oto/v2 v2.3.1/go.mod h1:seWLbgHH7AyUMYKfKYT9pg7PhUu9/SisyJvNTT+ASQo= +github.com/jfreymuth/oggvorbis v1.0.5 h1:u+Ck+R0eLSRhgq8WTmffYnrVtSztJcYrl588DM4e3kQ= +github.com/jfreymuth/oggvorbis v1.0.5/go.mod h1:1U4pqWmghcoVsCJJ4fRBKv9peUJMBHixthRlBeD6uII= +github.com/jfreymuth/vorbis v1.0.2 h1:m1xH6+ZI4thH927pgKD8JOH4eaGRm18rEE9/0WKjvNE= +github.com/jfreymuth/vorbis v1.0.2/go.mod h1:DoftRo4AznKnShRl1GxiTFCseHr4zR9BN3TWXyuzrqQ= +github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= +github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= +github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 h1:AyjUVuJuVd+5Kt+KEnIUoyZVAv8pejNDqs691I5e8jM= +github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97/go.mod h1:uWrLXY4JeLYynX39htMG49Dl4BhFYY+RCeoXaLdU+Lw= +github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e h1:SkgQRcG2VYEhh80Qb/zYZo8rWKJzNfJcfUQnXe6su2M= +github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= +github.com/livekit/protocol v1.49.0 h1:Q5nthDO1v7c0JHiWjMhgUQTlsKmCsBL/KCKxdHVaz00= +github.com/livekit/protocol v1.49.0/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= +github.com/livekit/psrpc v0.7.2 h1:6oZ+NODJ2pLyaT6VqDq1F4Qc/3TpDUSpyphj/P9MhQc= +github.com/livekit/psrpc v0.7.2/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= +github.com/livekit/server-sdk-go/v2 v2.17.0 h1:FzVQMoxHv0WIg164yGqSxLeV+h3aJomjAv1lFeR9MMw= +github.com/livekit/server-sdk-go/v2 v2.17.0/go.mod h1:5nzTfVBH2Jz+TW1SrfpqC7wrbcD1lT94KZCJ9hOMyvk= +github.com/mackerelio/go-osstat v0.2.7 h1:TCavZi10wF49bT6iQZ9eT2keGZQpC69MTDfdJej5e94= +github.com/mackerelio/go-osstat v0.2.7/go.mod h1:dwpYh5pIPmvk+IEwBKNIWRFMB92mrC08CmXOhDC7nQk= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.45 h1:6KA/spDguL3KV8rnybG7ezSaE4SeMR3KC9VbUoAQaIk= github.com/mattn/go-sqlite3 v1.14.45/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= +github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoPME= +github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 h1:WDsQxOJDy0N1VRAjXLpi8sCEZRSGarLWQevDxpTBRrM= github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0= +github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk= +github.com/pion/dtls/v3 v3.1.4 h1:QhvtMflMfu9Kf0RcDC5BJBle4caPskByrKQR6uuYqpY= +github.com/pion/dtls/v3 v3.1.4/go.mod h1:cr/qotLISUw/9C1m83ZPNZtj9WnXkYLpfCptPqbkInc= +github.com/pion/ice/v4 v4.2.7 h1:zDEbC6MiEdhQpF8TxBOTws+NU6ZgGpveHrQq4Lc1kao= +github.com/pion/ice/v4 v4.2.7/go.mod h1:9SNPaq0c7El/ki8leJzyCkK10zsskprR3zTNbO3monY= +github.com/pion/interceptor v0.1.45 h1:6PUo/5829bIfRFIPPJQzuDn8EjxRTSB/CSD7QVCOaqo= +github.com/pion/interceptor v0.1.45/go.mod h1:gNDYM/uFKcLe/B3gS2/7+aw6z+RDiMy2qKTnF1LO31w= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= +github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= +github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= +github.com/pion/opus v0.1.0 h1:GgK/a3DNDrffKjUFsK39rZKqfv7bQ2S2eqRKt0BnqAE= +github.com/pion/opus v0.1.0/go.mod h1:t5Xog2n682JnawoykACE6nKVmupFvmJvkpM7x6bTv6g= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= +github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= +github.com/pion/rtp v1.10.2 h1:l+f6tTDcAH6xwepaAoW791ddhuYsJlqRATOzirO04Mo= +github.com/pion/rtp v1.10.2/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= +github.com/pion/sctp v1.10.0 h1:qeoD6swF/2M5bYRcAGayqSbTKX3m4AW29CiQxG1+Pfg= +github.com/pion/sctp v1.10.0/go.mod h1:N20Dq6LY+JvJDAh9VVh1JELngb2rQ8dPgds5yBWiPgw= +github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= +github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= +github.com/pion/srtp/v3 v3.0.11 h1:GiESUr54/K4UuPigfq/CvWUed80JenQAHXn0C2MQQIQ= +github.com/pion/srtp/v3 v3.0.11/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/stun/v3 v3.1.4 h1:/7ZL0j0dmLroKOq4GfkyKQ6asByYqntwyHSp5sYLcGY= +github.com/pion/stun/v3 v3.1.4/go.mod h1:ET7PFiXo1nrD2ZNVpbEHDuT0kCPVXhKmyWdiePNMw/U= +github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= +github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= +github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= +github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= +github.com/pion/turn/v5 v5.0.8 h1:pZUCtmwWCMkrRKqh/8pL3WoGADXBe0/lOPkN7oqFjK8= +github.com/pion/turn/v5 v5.0.8/go.mod h1:1VwvxElZaOdJU0liJ/WUSm/Tsh+n2OxS5ISSDxgOWxU= +github.com/pion/webrtc/v4 v4.2.14 h1:Q6zMs+fSDsYuhZcNlvFGBxCOMHVV9oYcDa6O9/HIGTc= +github.com/pion/webrtc/v4 v4.2.14/go.mod h1:87NVKP86+g4OMrRxWhjWfUjeXP4JrV6RTlUrIW+/Jak= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pStaY= +github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 h1:D1asWnhVkAepHAEKSXQYJDG2nK5NHxdo1sRD72u6gSU= +github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52/go.mod h1:/YNSNaB2/qq6ZlFkOtPJV/z5nEWjRC+uBXC+3/LHCGU= +github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= +github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -65,10 +229,18 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= +github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTdwFp0s= github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mau.fi/libsignal v0.2.2 h1:QV+XdzQkm3x3aSG7FcqfGSZuFXz83pRZPBFaPygHbOU= go.mau.fi/libsignal v0.2.2/go.mod h1:CRlIQg2J8uYTfDFvNoO8/KcZjs5cey0vbc6oj/bssY0= go.mau.fi/util v0.9.10 h1:wzvz5iDHyqDXB8vgisD4d3SzucLXNM3iNY+1O1RoHtg= @@ -79,6 +251,30 @@ go.mau.fi/whatsmeow v0.0.0-20260709092057-73fe7355f59f h1:VZkwFBEQ9TbB9IWsldw8BJ go.mau.fi/whatsmeow v0.0.0-20260709092057-73fe7355f59f/go.mod h1:9dmNTYZ/1pHjPw/bz+azBsGjAkcrZbqzMrKcvG5bJ8U= go.mau.fi/zeroconfig v0.2.0 h1:e/OGEERqVRRKlgaro7E6bh8xXiKFSXB3eNNIud7FUjU= go.mau.fi/zeroconfig v0.2.0/go.mod h1:J0Vn0prHNOm493oZoQ84kq83ZaNCYZnq+noI1b1eN8w= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= @@ -91,16 +287,27 @@ golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302 h1:xeVptzkP8BuJhoIjNizd2bRHfq9KB9HfOLZu90T04XM= +gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302/go.mod h1:/L5E7a21VWl8DeuCPKxQBdVG5cy+L0MRZ08B1wnqt7g= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From f469f2c7213f0ab45cb5ff66047adaf45974204d Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 16 Jul 2026 21:04:15 +0300 Subject: [PATCH 03/44] pkg/connector: bridge WhatsApp calls over MatrixRTC --- pkg/connector/client.go | 8 + pkg/connector/config.go | 149 ++++++ pkg/connector/config_test.go | 33 ++ pkg/connector/connector.go | 6 + pkg/connector/example-config.yaml | 59 +++ pkg/connector/matrixrtc.go | 297 ++++++++++++ pkg/connector/matrixrtc_outgoing.go | 687 ++++++++++++++++++++++++++ pkg/connector/matrixrtc_test.go | 313 ++++++++++++ pkg/connector/voip/audio.go | 151 ++++++ pkg/connector/voip/audio_test.go | 49 ++ pkg/connector/voip/config.go | 54 +++ pkg/connector/voip/errors.go | 6 + pkg/connector/voip/focus.go | 241 ++++++++++ pkg/connector/voip/focus_test.go | 131 +++++ pkg/connector/voip/livekit.go | 480 ++++++++++++++++++ pkg/connector/voip/manager.go | 694 +++++++++++++++++++++++++++ pkg/connector/voip/manager_test.go | 121 +++++ pkg/connector/voip/matrixrtc.go | 520 ++++++++++++++++++++ pkg/connector/voip/matrixrtc_test.go | 189 ++++++++ pkg/connector/voip/video.go | 57 +++ pkg/connector/voip/video_buffer.go | 56 +++ pkg/connector/voip_config.go | 46 ++ 22 files changed, 4347 insertions(+) create mode 100644 pkg/connector/config_test.go create mode 100644 pkg/connector/matrixrtc.go create mode 100644 pkg/connector/matrixrtc_outgoing.go create mode 100644 pkg/connector/matrixrtc_test.go create mode 100644 pkg/connector/voip/audio.go create mode 100644 pkg/connector/voip/audio_test.go create mode 100644 pkg/connector/voip/config.go create mode 100644 pkg/connector/voip/errors.go create mode 100644 pkg/connector/voip/focus.go create mode 100644 pkg/connector/voip/focus_test.go create mode 100644 pkg/connector/voip/livekit.go create mode 100644 pkg/connector/voip/manager.go create mode 100644 pkg/connector/voip/manager_test.go create mode 100644 pkg/connector/voip/matrixrtc.go create mode 100644 pkg/connector/voip/matrixrtc_test.go create mode 100644 pkg/connector/voip/video.go create mode 100644 pkg/connector/voip/video_buffer.go create mode 100644 pkg/connector/voip_config.go diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 3f19bf7..f35aa60 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -40,6 +40,7 @@ import ( "maunium.net/go/mautrix/bridgev2/status" "maunium.net/go/mautrix/event" + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" "go.mau.fi/mautrix-whatsapp/pkg/waid" ) @@ -83,6 +84,9 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. w.Client.GetMessageForRetry = w.trackNotFoundRetry w.Client.PreRetryCallback = w.trackFoundRetry w.Client.BackgroundEventCtx = w.UserLogin.Log.WithContext(wa.Bridge.BackgroundCtx) + w.VOIP = voip.NewManager(w.Client, makeVOIPConfig(wa.Config.VOIP), w.UserLogin.Log.With().Str("component", "voip").Logger()) + w.VOIP.SetIncomingCallHandler(w.handleIncomingVOIPCall) + w.VOIP.SetCallEndHandler(w.handleVOIPCallEnded) w.Client.SetForceActiveDeliveryReceipts(wa.Config.ForceActiveDeliveryReceipts) w.Client.InitialAutoReconnect = wa.Config.InitialAutoReconnect w.Client.UseRetryMessageStore = wa.Config.UseWhatsAppRetryStore @@ -102,6 +106,7 @@ type WhatsAppClient struct { Main *WhatsAppConnector UserLogin *bridgev2.UserLogin Client *whatsmeow.Client + VOIP *voip.Manager Device *store.Device JID types.JID MC mClient @@ -363,6 +368,9 @@ func (wa *WhatsAppClient) callStopLoops() { func (wa *WhatsAppClient) Disconnect() { wa.callStopLoops() + if wa.VOIP != nil { + wa.VOIP.AbortAll() + } if cli := wa.Client; cli != nil { cli.Disconnect() } diff --git a/pkg/connector/config.go b/pkg/connector/config.go index e51c80a..7d15948 100644 --- a/pkg/connector/config.go +++ b/pkg/connector/config.go @@ -54,6 +54,7 @@ type Config struct { UseWhatsAppRetryStore bool `yaml:"use_whatsapp_retry_store"` AnimatedSticker msgconv.AnimatedStickerConfig `yaml:"animated_sticker"` + VOIP VOIPConfig `yaml:"voip"` HistorySync struct { MaxInitialConversations int `yaml:"max_initial_conversations"` @@ -78,6 +79,58 @@ type Config struct { displaynameTemplate *template.Template `yaml:"-"` } +type VOIPConfig struct { + Enabled bool `yaml:"enabled"` + MatrixSurface string `yaml:"matrix_surface"` + IncomingPolicy string `yaml:"incoming_policy"` + MaxActiveCallsPerLogin int `yaml:"max_active_calls_per_login"` + MatrixRTC MatrixRTCConfig `yaml:"matrixrtc"` + LiveKit LiveKitConfig `yaml:"livekit"` + Audio VOIPAudioConfig `yaml:"audio"` + Video VOIPVideoConfig `yaml:"video"` + Diagnostics VOIPDiagnostics `yaml:"diagnostics"` +} + +type MatrixRTCConfig struct { + LiveKitServiceURL string `yaml:"livekit_service_url"` + RequireLiveKitFocus bool `yaml:"require_livekit_focus"` + MembershipEventCompat string `yaml:"membership_event_compat"` + NotificationEventCompat string `yaml:"notification_event_compat"` + UseDelayedEvents bool `yaml:"use_delayed_events"` + ParticipantMode string `yaml:"participant_mode"` + FallbackParticipantMXID string `yaml:"fallback_participant_mxid"` +} + +type LiveKitConfig struct { + ConnectTimeout time.Duration `yaml:"connect_timeout"` + PublishSilenceBeforeWhatsAppAnswer bool `yaml:"publish_silence_before_whatsapp_answer"` + AutoSubscribe bool `yaml:"auto_subscribe"` + AudioUplinkPolicy string `yaml:"audio_uplink_policy"` + SelectedParticipantTimeout time.Duration `yaml:"selected_participant_timeout"` +} + +type VOIPAudioConfig struct { + Enabled bool `yaml:"enabled"` + JitterBuffer time.Duration `yaml:"jitter_buffer_ms"` + OpusBackend string `yaml:"opus_backend"` + SilenceOnUnderrun bool `yaml:"silence_on_underrun"` + MaxMixParticipants int `yaml:"max_mix_participants"` +} + +type VOIPVideoConfig struct { + Enabled bool `yaml:"enabled"` + SelectedSourcePolicy string `yaml:"selected_source_policy"` + MaxWidth int `yaml:"max_width"` + MaxHeight int `yaml:"max_height"` + MaxFPS int `yaml:"max_fps"` +} + +type VOIPDiagnostics struct { + HealthcheckFocusOnStartup bool `yaml:"healthcheck_focus_on_startup"` + EnableMeowcallerDiagnostics bool `yaml:"enable_meowcaller_diagnostics"` + MediaTraceDir string `yaml:"media_trace_dir"` +} + type umConfig Config func (c *Config) UnmarshalYAML(node *yaml.Node) error { @@ -99,9 +152,74 @@ func (c *Config) PostProcess() error { if err != nil { return fmt.Errorf("failed to execute displayname template: %w", err) } + if err = c.validateVOIP(); err != nil { + return err + } return nil } +func (c *Config) validateVOIP() error { + if !c.VOIP.Enabled { + return nil + } + if c.VOIP.MatrixSurface != "matrixrtc_livekit" { + return fmt.Errorf("voip.matrix_surface must be matrixrtc_livekit") + } + if !oneOf(c.VOIP.IncomingPolicy, "notice", "ring", "auto_answer") { + return fmt.Errorf("voip.incoming_policy must be one of notice, ring, auto_answer") + } + if c.VOIP.MaxActiveCallsPerLogin <= 0 { + return fmt.Errorf("voip.max_active_calls_per_login must be greater than 0") + } + if !oneOf(c.VOIP.MatrixRTC.MembershipEventCompat, "auto", "msc4143", "msc3401") { + return fmt.Errorf("voip.matrixrtc.membership_event_compat must be one of auto, msc4143, msc3401") + } + if !oneOf(c.VOIP.MatrixRTC.NotificationEventCompat, "auto", "disabled") { + return fmt.Errorf("voip.matrixrtc.notification_event_compat must be one of auto, disabled") + } + if !oneOf(c.VOIP.MatrixRTC.ParticipantMode, "whatsapp_ghost", "bridge_user") { + return fmt.Errorf("voip.matrixrtc.participant_mode must be one of whatsapp_ghost, bridge_user") + } + if c.VOIP.LiveKit.ConnectTimeout <= 0 { + return fmt.Errorf("voip.livekit.connect_timeout must be greater than 0") + } + if !oneOf(c.VOIP.LiveKit.AudioUplinkPolicy, "dominant_speaker", "mix_all", "selected_participant") { + return fmt.Errorf("voip.livekit.audio_uplink_policy must be one of dominant_speaker, mix_all, selected_participant") + } + if c.VOIP.Audio.Enabled { + if c.VOIP.Audio.JitterBuffer <= 0 { + return fmt.Errorf("voip.audio.jitter_buffer_ms must be greater than 0") + } + if c.VOIP.Audio.OpusBackend == "" { + return fmt.Errorf("voip.audio.opus_backend must be set") + } + if c.VOIP.Audio.MaxMixParticipants <= 0 { + return fmt.Errorf("voip.audio.max_mix_participants must be greater than 0") + } + } + if c.VOIP.Video.Enabled { + if !oneOf(c.VOIP.Video.SelectedSourcePolicy, "active_speaker", "selected_participant") { + return fmt.Errorf("voip.video.selected_source_policy must be one of active_speaker, selected_participant") + } + if c.VOIP.Video.MaxWidth <= 0 || c.VOIP.Video.MaxHeight <= 0 || c.VOIP.Video.MaxFPS <= 0 { + return fmt.Errorf("voip.video max_width, max_height and max_fps must be greater than 0") + } + } + if c.VOIP.Diagnostics.EnableMeowcallerDiagnostics && c.VOIP.Diagnostics.MediaTraceDir == "" { + return fmt.Errorf("voip.diagnostics.media_trace_dir must be set when meowcaller diagnostics are enabled") + } + return nil +} + +func oneOf(value string, allowed ...string) bool { + for _, item := range allowed { + if value == item { + return true + } + } + return false +} + func upgradeConfig(helper up.Helper) { helper.Copy(up.Str, "os_name") helper.Copy(up.Str, "browser_name") @@ -135,6 +253,36 @@ func upgradeConfig(helper up.Helper) { helper.Copy(up.Int, "animated_sticker", "args", "height") helper.Copy(up.Int, "animated_sticker", "args", "fps") + helper.Copy(up.Bool, "voip", "enabled") + helper.Copy(up.Str, "voip", "matrix_surface") + helper.Copy(up.Str, "voip", "incoming_policy") + helper.Copy(up.Int, "voip", "max_active_calls_per_login") + helper.Copy(up.Str|up.Null, "voip", "matrixrtc", "livekit_service_url") + helper.Copy(up.Bool, "voip", "matrixrtc", "require_livekit_focus") + helper.Copy(up.Str, "voip", "matrixrtc", "membership_event_compat") + helper.Copy(up.Str, "voip", "matrixrtc", "notification_event_compat") + helper.Copy(up.Bool, "voip", "matrixrtc", "use_delayed_events") + helper.Copy(up.Str, "voip", "matrixrtc", "participant_mode") + helper.Copy(up.Str|up.Null, "voip", "matrixrtc", "fallback_participant_mxid") + helper.Copy(up.Str|up.Int, "voip", "livekit", "connect_timeout") + helper.Copy(up.Bool, "voip", "livekit", "publish_silence_before_whatsapp_answer") + helper.Copy(up.Bool, "voip", "livekit", "auto_subscribe") + helper.Copy(up.Str, "voip", "livekit", "audio_uplink_policy") + helper.Copy(up.Str|up.Int, "voip", "livekit", "selected_participant_timeout") + helper.Copy(up.Bool, "voip", "audio", "enabled") + helper.Copy(up.Str|up.Int, "voip", "audio", "jitter_buffer_ms") + helper.Copy(up.Str, "voip", "audio", "opus_backend") + helper.Copy(up.Bool, "voip", "audio", "silence_on_underrun") + helper.Copy(up.Int, "voip", "audio", "max_mix_participants") + helper.Copy(up.Bool, "voip", "video", "enabled") + helper.Copy(up.Str, "voip", "video", "selected_source_policy") + helper.Copy(up.Int, "voip", "video", "max_width") + helper.Copy(up.Int, "voip", "video", "max_height") + helper.Copy(up.Int, "voip", "video", "max_fps") + helper.Copy(up.Bool, "voip", "diagnostics", "healthcheck_focus_on_startup") + helper.Copy(up.Bool, "voip", "diagnostics", "enable_meowcaller_diagnostics") + helper.Copy(up.Str|up.Null, "voip", "diagnostics", "media_trace_dir") + helper.Copy(up.Int, "history_sync", "max_initial_conversations") helper.Copy(up.Bool, "history_sync", "request_full_sync") helper.Copy(up.Str|up.Int, "history_sync", "dispatch_wait") @@ -205,6 +353,7 @@ func (wa *WhatsAppConnector) GetConfig() (string, any, up.Upgrader) { {"proxy"}, {"displayname_template"}, {"call_start_notices"}, + {"voip"}, {"history_sync"}, }, Base: ExampleConfig, diff --git a/pkg/connector/config_test.go b/pkg/connector/config_test.go new file mode 100644 index 0000000..5f2b93f --- /dev/null +++ b/pkg/connector/config_test.go @@ -0,0 +1,33 @@ +package connector + +import ( + "os" + "testing" + + "go.yaml.in/yaml/v3" +) + +func TestExampleConfigDoesNotAdvertiseUnsupportedVideoModes(t *testing.T) { + data, err := os.ReadFile("example-config.yaml") + if err != nil { + t.Fatal(err) + } + + var config map[string]any + if err = yaml.Unmarshal(data, &config); err != nil { + t.Fatal(err) + } + voip, ok := config["voip"].(map[string]any) + if !ok { + t.Fatal("example config has no voip section") + } + video, ok := voip["video"].(map[string]any) + if !ok { + t.Fatal("example config has no voip.video section") + } + for _, unsupported := range []string{"require_h264", "allow_transcode"} { + if _, exists := video[unsupported]; exists { + t.Errorf("example config advertises unsupported voip.video.%s option", unsupported) + } + } +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 91b1e11..1ca1340 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -64,6 +64,9 @@ type WhatsAppConnector struct { mediaEditCache MediaEditCache mediaEditCacheLock sync.RWMutex stopMediaEditCacheLoop atomic.Pointer[context.CancelFunc] + + matrixRTCOutboundStartLock sync.Mutex + matrixRTCOutboundStartExpires map[string]time.Time } func init() { @@ -110,10 +113,13 @@ func (wa *WhatsAppConnector) Init(bridge *bridgev2.Bridge) { } wa.DB = wadb.New(bridge.ID, bridge.DB.Database, bridge.Log.With().Str("db_section", "whatsapp").Logger()) wa.MsgConv.DB = wa.DB + wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) wa.Bridge.Commands.(*commands.Processor).AddHandlers( cmdAccept, cmdSync, cmdInviteLink, cmdResolveLink, cmdJoin, ) wa.mediaEditCache = make(MediaEditCache) + wa.initMatrixRTCEventHooks() + wa.startMatrixRTCHealthcheck() whatsmeowDBLog := bridge.Log.With().Str("db_section", "whatsmeow").Logger() wa.DeviceStore = sqlstore.NewWithWrappedDB( diff --git a/pkg/connector/example-config.yaml b/pkg/connector/example-config.yaml index 564f25e..db5ef0d 100644 --- a/pkg/connector/example-config.yaml +++ b/pkg/connector/example-config.yaml @@ -70,6 +70,65 @@ initial_auto_reconnect: true # retry receipts if the bridge is restarted after the message is sent. use_whatsapp_retry_store: false +# MatrixRTC/LiveKit call bridging for WhatsApp calls. +voip: + # Enables real call bridging through Element Call. When false, call_start_notices behavior remains. + enabled: false + # Only MatrixRTC with a LiveKit focus is supported by this bridge path. + matrix_surface: matrixrtc_livekit + # notice - keep call_start_notices fallback only + # ring - ring MatrixRTC/Element Call and answer WhatsApp only after a Matrix user joins + # auto_answer - test-only behavior that answers WhatsApp immediately + incoming_policy: ring + # WhatsApp/meowcaller currently supports one live 1:1 call leg per login. + max_active_calls_per_login: 1 + + matrixrtc: + # If null, discover from .well-known org.matrix.msc4143.rtc_foci. + livekit_service_url: null + # Refuse real call bridging when no LiveKit focus can be discovered. + require_livekit_focus: true + # auto, msc4143, or msc3401. + membership_event_compat: auto + # auto or disabled. Used for MatrixRTC call ringing/decline events when supported. + notification_event_compat: auto + # Use homeserver delayed events to expire MatrixRTC membership when supported. + use_delayed_events: true + # whatsapp_ghost or bridge_user. + participant_mode: whatsapp_ghost + # Optional explicit Matrix user to use when participant_mode is bridge_user. + fallback_participant_mxid: null + + livekit: + connect_timeout: 10s + # Publish silence before WhatsApp answers so Element Call shows a stable participant. + publish_silence_before_whatsapp_answer: true + auto_subscribe: true + # dominant_speaker, mix_all, or selected_participant. + audio_uplink_policy: dominant_speaker + selected_participant_timeout: 30s + + audio: + enabled: true + jitter_buffer_ms: 180ms + opus_backend: libopus + silence_on_underrun: true + max_mix_participants: 4 + + video: + # Video is passed through as H.264. Other codecs are ignored. + enabled: false + selected_source_policy: active_speaker + max_width: 1280 + max_height: 720 + max_fps: 30 + + diagnostics: + healthcheck_focus_on_startup: true + # Unsafe: meowcaller diagnostics can contain call secrets and media. + enable_meowcaller_diagnostics: false + media_trace_dir: null + # Settings for converting animated stickers. animated_sticker: # Format to which animated stickers should be converted. diff --git a/pkg/connector/matrixrtc.go b/pkg/connector/matrixrtc.go new file mode 100644 index 0000000..eb2081b --- /dev/null +++ b/pkg/connector/matrixrtc.go @@ -0,0 +1,297 @@ +package connector + +import ( + "context" + "strings" + "time" + + "github.com/rs/zerolog" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" +) + +const ( + matrixRTCHealthcheckTimeout = 15 * time.Second + matrixRTCOutboundStartDedupWindow = 30 * time.Second +) + +func withoutCancelOrBackground(ctx context.Context) context.Context { + if ctx == nil { + return context.Background() + } + return context.WithoutCancel(ctx) +} + +func (wa *WhatsAppConnector) initMatrixRTCEventHooks() { + matrixConnector, ok := wa.Bridge.Matrix.(*matrix.Connector) + if !ok || matrixConnector.EventProcessor == nil { + wa.Bridge.Log.Debug().Msg("Matrix connector does not expose an event processor for MatrixRTC hooks") + return + } + for _, evtType := range voip.SupportedMatrixRTCEventTypes() { + matrixConnector.EventProcessor.On(evtType, wa.handleMatrixRTCEvent) + } + wa.Bridge.Log.Debug().Int("event_type_count", len(voip.SupportedMatrixRTCEventTypes())).Msg("Registered MatrixRTC event hooks") +} + +func (wa *WhatsAppConnector) startMatrixRTCHealthcheck() { + if !wa.Config.VOIP.Enabled || !wa.Config.VOIP.Diagnostics.HealthcheckFocusOnStartup { + return + } + go func() { + ctx, cancel := context.WithTimeout(withoutCancelOrBackground(wa.Bridge.BackgroundCtx), matrixRTCHealthcheckTimeout) + defer cancel() + focus, err := voip.DiscoverLiveKitFocus(ctx, nil, wa.Bridge.Matrix.ServerName(), wa.Config.VOIP.MatrixRTC.LiveKitServiceURL) + log := wa.Bridge.Log.With().Str("component", "voip_healthcheck").Logger() + if err != nil { + event := log.Warn() + if wa.Config.VOIP.MatrixRTC.RequireLiveKitFocus { + event = log.Error() + } + event.Err(err).Msg("MatrixRTC LiveKit focus healthcheck failed") + return + } + log.Info(). + Str("focus_type", focus.Type). + Str("livekit_service_url", focus.LiveKitServiceURL). + Msg("MatrixRTC LiveKit focus healthcheck passed") + }() +} + +func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *event.Event) { + if !wa.Config.VOIP.Enabled { + return + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + return + } + + log := zerolog.Ctx(ctx).With(). + Str("matrixrtc_kind", string(parsed.Kind)). + Str("matrix_event_type", parsed.Type.Type). + Stringer("matrix_room_id", parsed.RoomID). + Stringer("matrix_sender", parsed.Sender). + Str("matrix_call_id", parsed.CallID). + Logger() + + if parsed.Sender == wa.Bridge.Bot.GetMXID() || wa.Bridge.IsGhostMXID(parsed.Sender) { + log.Debug().Msg("Ignoring MatrixRTC event sent by the bridge") + return + } + if !wa.Bridge.Config.Permissions.Get(parsed.Sender).SendEvents { + log.Debug().Msg("Dropping MatrixRTC event from user with no permission to send events") + wa.Bridge.Matrix.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt)) + return + } + + portal, err := wa.Bridge.GetPortalByMXID(ctx, parsed.RoomID) + if err != nil { + log.Err(err).Msg("Failed to look up portal for MatrixRTC event") + return + } else if portal == nil { + log.Debug().Msg("Ignoring MatrixRTC event outside a bridged portal") + return + } + + activeCalls, err := wa.DB.MatrixRTCCall.GetActiveInRoom(ctx, parsed.RoomID) + if err != nil { + log.Err(err).Msg("Failed to look up active MatrixRTC calls for room") + return + } else if len(activeCalls) == 0 { + if !shouldStartOutboundMatrixRTCCall(evt, parsed, wa.Config.VOIP.MatrixRTC.MembershipEventCompat) { + log.Debug().Msg("Ignoring MatrixRTC event without active bridged calls in the room") + return + } + if !wa.reserveMatrixRTCOutboundStart(parsed.RoomID.String()) { + log.Debug().Msg("Ignoring duplicate outbound MatrixRTC start in dedupe window") + return + } + if err = wa.startOutboundMatrixRTCCall(ctx, portal, parsed); err != nil { + log.Err(err).Msg("Failed to start outbound WhatsApp call from MatrixRTC event") + if cleanupErr := wa.cleanupFailedOutboundMatrixRTCStart(ctx, parsed); cleanupErr != nil { + log.Err(cleanupErr).Msg("Failed to clean up failed outbound MatrixRTC event") + } + } + return + } + + var matched, handled, activated, ended int + for _, activeCall := range activeCalls { + matched++ + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, activeCall.UserLoginID) + callLog := log.With(). + Str("wa_call_id", activeCall.WACallID). + Str("user_login_id", string(activeCall.UserLoginID)). + Stringer("matrix_participant_mxid", activeCall.MatrixParticipantMXID). + Str("matrix_session_id", activeCall.MatrixSessionID). + Logger() + if err != nil { + callLog.Err(err).Msg("Failed to look up WhatsApp login for MatrixRTC call") + continue + } else if login == nil { + callLog.Debug().Msg("MatrixRTC call references a missing WhatsApp login") + continue + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + callLog.Debug().Msg("WhatsApp login has no VOIP manager for MatrixRTC event") + continue + } + handled++ + endedCalls := client.VOIP.HandleMatrixRTCCallEvent(ctx, parsed, activeCall.WACallID) + if shouldEndMatrixRTCCallFromMembership(parsed, activeCall.SelectedPublisherID) { + endedCalls += client.VOIP.HandleMatrixRTCCallEvent(ctx, voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindRTCDecline, + RoomID: parsed.RoomID, + Sender: parsed.Sender, + CallID: parsed.CallID, + }, activeCall.WACallID) + } + ended += endedCalls + if shouldActivateMatrixRTCCall(parsed, activeCall.State) { + if err = client.activateMatrixRTCCall(ctx, activeCall, parsed); err != nil { + callLog.Err(err).Msg("Failed to activate MatrixRTC LiveKit bridge for WhatsApp call") + } else { + activated++ + } + } + if endedCalls > 0 { + err = wa.DB.MatrixRTCCall.MarkEnded(ctx, activeCall.UserLoginID, activeCall.WACallID, "ended", string(parsed.Kind), "", time.Now()) + if err != nil { + callLog.Err(err).Msg("Failed to mark MatrixRTC call ended after MatrixRTC event") + } + } + } + log.Debug(). + Int("active_call_count", len(activeCalls)). + Int("matched_call_count", matched). + Int("handled_call_count", handled). + Int("activated_call_count", activated). + Int("ended_call_count", ended). + Msg("Handled MatrixRTC event for active bridged calls") +} + +func shouldActivateMatrixRTCCall(evt voip.MatrixRTCEvent, callState string) bool { + if callState != "ringing" { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + return voip.MatrixRTCEventHasJoinContent(evt) + default: + return false + } +} + +func shouldStartOutboundMatrixRTCCall(evt *event.Event, parsed voip.MatrixRTCEvent, membershipCompat string) bool { + if parsed.Type.Class != event.StateEventType || !voip.MatrixRTCEventHasJoinContent(parsed) { + return false + } + switch parsed.Kind { + case voip.MatrixRTCEventKindRTCMembership: + if !matrixRTCCompatAllowsModern(membershipCompat) { + return false + } + case voip.MatrixRTCEventKindGroupCallMember: + if !matrixRTCCompatAllowsLegacy(membershipCompat) { + return false + } + default: + return false + } + return !matrixRTCPrevContentHasJoinContent(evt) +} + +func shouldEndMatrixRTCCallFromMembership(evt voip.MatrixRTCEvent, selectedParticipantID string) bool { + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + default: + return false + } + if selectedParticipantID == "" || voip.MatrixRTCEventHasJoinContent(evt) { + return false + } + return matrixRTCEventMatchesParticipant(evt, selectedParticipantID) +} + +func matrixRTCEventMatchesParticipant(evt voip.MatrixRTCEvent, participantID string) bool { + if participantID == "" { + return false + } + if matrixRTCTriggerParticipantID(evt) == participantID { + return true + } + if evt.StateKey == "" || evt.Sender == "" { + return false + } + if legacyID := legacyMatrixRTCParticipantIDFromStateKey(evt.Sender, evt.StateKey); legacyID == participantID { + return true + } + if modernID := modernMatrixRTCParticipantIDFromStateKey(evt.Sender, evt.StateKey); modernID == participantID { + return true + } + return false +} + +func legacyMatrixRTCParticipantIDFromStateKey(sender id.UserID, stateKey string) string { + prefix := "_" + string(sender) + "_" + const suffix = "_m.call" + if !strings.HasPrefix(stateKey, prefix) || !strings.HasSuffix(stateKey, suffix) { + return "" + } + deviceID := strings.TrimSuffix(strings.TrimPrefix(stateKey, prefix), suffix) + if deviceID == "" { + return "" + } + return voip.MatrixRTCMemberID(sender, deviceID) +} + +func modernMatrixRTCParticipantIDFromStateKey(sender id.UserID, stateKey string) string { + prefix := string(sender) + "_" + if !strings.HasPrefix(stateKey, prefix) { + return "" + } + deviceID := strings.TrimPrefix(stateKey, prefix) + if deviceID == "" { + return "" + } + return voip.MatrixRTCMemberID(sender, deviceID) +} + +func (wa *WhatsAppConnector) reserveMatrixRTCOutboundStart(roomID string) bool { + if roomID == "" { + return false + } + now := time.Now() + wa.matrixRTCOutboundStartLock.Lock() + defer wa.matrixRTCOutboundStartLock.Unlock() + if wa.matrixRTCOutboundStartExpires == nil { + wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) + } + for trackedRoomID, expires := range wa.matrixRTCOutboundStartExpires { + if !expires.After(now) { + delete(wa.matrixRTCOutboundStartExpires, trackedRoomID) + } + } + if expires := wa.matrixRTCOutboundStartExpires[roomID]; expires.After(now) { + return false + } + wa.matrixRTCOutboundStartExpires[roomID] = now.Add(matrixRTCOutboundStartDedupWindow) + return true +} + +func matrixRTCPrevContentHasJoinContent(evt *event.Event) bool { + if evt == nil || evt.Unsigned.PrevContent == nil { + return false + } + prevEvt := *evt + prevEvt.Content = *evt.Unsigned.PrevContent + prevEvt.Unsigned.PrevContent = nil + parsedPrev, ok := voip.ParseMatrixRTCEvent(&prevEvt) + return ok && voip.MatrixRTCEventHasJoinContent(parsedPrev) +} diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go new file mode 100644 index 0000000..73fbdd4 --- /dev/null +++ b/pkg/connector/matrixrtc_outgoing.go @@ -0,0 +1,687 @@ +package connector + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/purpshell/meowcaller" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + mxbridge "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" + "go.mau.fi/mautrix-whatsapp/pkg/waid" +) + +const ( + matrixRTCRingLifetime = 90 * time.Second + matrixRTCMembershipLifetime = 4 * time.Hour + matrixRTCStickyDuration = time.Hour +) + +func (wa *WhatsAppClient) handleIncomingVOIPCall(call *meowcaller.Call) { + if call == nil { + return + } + ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) + err := wa.announceIncomingMatrixRTCCall(ctx, call) + if err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Msg("Failed to announce incoming WhatsApp call over MatrixRTC") + } +} + +func (wa *WhatsAppClient) handleVOIPCallEnded(callID, reason string) { + ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) + log := wa.UserLogin.Log.With().Str("call_id", callID).Str("reason", reason).Logger() + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil { + log.Err(err).Msg("Failed to look up MatrixRTC call record after WhatsApp call ended") + return + } else if call == nil { + return + } + if err = wa.clearMatrixRTCMembership(ctx, call); err != nil { + log.Err(err).Msg("Failed to clear MatrixRTC membership after WhatsApp call ended") + } + endReason, lastError := matrixRTCFinalEndReason(call, reason) + if err = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, wa.UserLogin.ID, callID, "ended", endReason, lastError, time.Now()); err != nil { + log.Err(err).Msg("Failed to mark MatrixRTC call ended") + } +} + +func matrixRTCFinalEndReason(call *wadb.MatrixRTCCall, reason string) (string, string) { + if call != nil && !call.EndedTS.IsZero() && (call.EndReason != "" || call.LastError != "") { + if call.EndReason != "" { + reason = call.EndReason + } + return reason, call.LastError + } + return reason, "" +} + +func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, call *meowcaller.Call) error { + log := zerolog.Ctx(ctx).With(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Logger() + peer := wa.matrixRTCAnnouncementPeer(ctx, call.Peer()) + portal, err := wa.Main.Bridge.GetPortalByKey(ctx, wa.makeWAPortalKey(peer)) + if err != nil { + return err + } + if portal == nil || portal.MXID == "" { + log.Debug().Msg("No existing Matrix portal room for incoming MatrixRTC call announcement") + return nil + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + log.Warn(). + Int("active_call_count", len(activeCalls)). + Int("max_active_calls", wa.Main.Config.VOIP.MaxActiveCallsPerLogin). + Msg("Rejecting incoming WhatsApp call because the MatrixRTC active call limit was reached") + return call.Reject() + } + } + focus, err := voip.DiscoverLiveKitFocus(ctx, nil, wa.Main.Bridge.Matrix.ServerName(), wa.Main.Config.VOIP.MatrixRTC.LiveKitServiceURL) + if err != nil { + return err + } + intent, err := wa.matrixRTCParticipantIntent(ctx, peer) + if err != nil { + return err + } + + now := time.Now() + deviceID := voip.MatrixRTCDeviceID(string(wa.UserLogin.ID), call.ID()) + session := voip.MatrixRTCSession{ + UserID: intent.GetMXID(), + DeviceID: deviceID, + MemberID: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + Intent: matrixRTCCallIntent(call), + Focus: *focus, + Created: now, + Expires: matrixRTCMembershipLifetime, + StickyKey: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + } + record := &wadb.MatrixRTCCall{ + UserLoginID: wa.UserLogin.ID, + WACallID: call.ID(), + RoomID: portal.MXID, + PortalKey: portal.PortalKey, + PeerJID: peer, + Direction: "incoming", + MediaKind: session.Intent, + FocusType: focus.Type, + LiveKitServiceURL: focus.LiveKitServiceURL, + LiveKitRoom: portal.MXID.String(), + MatrixParticipantMXID: intent.GetMXID(), + MatrixSessionID: deviceID, + AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, + State: "ringing", + CreatedTS: now, + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + return err + } + if err = wa.sendMatrixRTCRing(ctx, intent, portal.MXID, call.ID(), &session); err != nil { + _ = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, wa.UserLogin.ID, call.ID(), "ended", "matrixrtc_announce_failed", err.Error(), time.Now()) + return err + } + log.Info(). + Stringer("room_id", portal.MXID). + Stringer("participant_mxid", intent.GetMXID()). + Str("device_id", deviceID). + Msg("Announced incoming WhatsApp call over MatrixRTC") + return nil +} + +func (wa *WhatsAppClient) matrixRTCAnnouncementPeer(ctx context.Context, peer types.JID) types.JID { + peer = peer.ToNonAD() + if peer.Server != types.HiddenUserServer { + return peer + } + pn, err := wa.GetStore().LIDs.GetPNForLID(ctx, peer) + if err != nil { + zerolog.Ctx(ctx).Err(err). + Stringer("lid", peer). + Msg("Failed to get phone number for LID incoming MatrixRTC call") + return peer + } else if pn.IsEmpty() { + return peer + } + pn = pn.ToNonAD() + zerolog.Ctx(ctx).Debug(). + Stringer("lid", peer). + Stringer("pn", pn). + Msg("Using phone number portal for incoming MatrixRTC call from LID") + return pn +} + +func (wa *WhatsAppConnector) startOutboundMatrixRTCCall(ctx context.Context, portal *bridgev2.Portal, trigger voip.MatrixRTCEvent) error { + if portal == nil { + return nil + } + if portal.Receiver == "" { + return fmt.Errorf("portal has no receiver login for outbound MatrixRTC call") + } + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, portal.Receiver) + if err != nil { + return err + } else if login == nil || login.Client == nil { + return fmt.Errorf("receiver login %s not found for outbound MatrixRTC call", portal.Receiver) + } else if !login.Client.IsLoggedIn() { + return bridgev2.ErrNotLoggedIn + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + return fmt.Errorf("receiver login %s has no WhatsApp VOIP manager", portal.Receiver) + } + return client.startOutboundMatrixRTCCall(ctx, portal, trigger) +} + +func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal *bridgev2.Portal, trigger voip.MatrixRTCEvent) error { + if wa.VOIP == nil || !wa.VOIP.Enabled() { + return voip.ErrNotEnabled + } + peer, err := waid.ParsePortalID(portal.ID) + if err != nil { + return err + } + if !matrixRTCPortalSupportsWhatsAppCalls(peer) { + return fmt.Errorf("MatrixRTC WhatsApp calls are only supported in 1:1 portals, not %s", peer.Server) + } + mediaKind, downgradedMedia := matrixRTCOutboundMediaKind(trigger) + if mediaKind == "" { + return fmt.Errorf("outbound WhatsApp MatrixRTC calls only support audio/video, not %q", trigger.Intent) + } + if downgradedMedia { + wa.UserLogin.Log.Warn(). + Stringer("room_id", trigger.RoomID). + Str("requested_media_kind", trigger.Intent). + Str("media_kind", mediaKind). + Msg("Downgrading outbound MatrixRTC call media kind") + } + if mediaKind == "video" && !wa.Main.Config.VOIP.Video.Enabled { + return fmt.Errorf("outbound WhatsApp MatrixRTC video calls require voip.video.enabled") + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + return fmt.Errorf("active MatrixRTC call limit reached for login %s", wa.UserLogin.ID) + } + } + focus, err := wa.matrixRTCLiveKitFocusForTrigger(ctx, trigger) + if err != nil { + return err + } + intent, err := wa.matrixRTCParticipantIntent(ctx, peer) + if err != nil { + return err + } + + call, err := wa.VOIP.Dial(ctx, peer.ToNonAD().String(), mediaKind == "video") + if err != nil { + return err + } + now := time.Now() + deviceID := voip.MatrixRTCDeviceID(string(wa.UserLogin.ID), call.ID()) + record := &wadb.MatrixRTCCall{ + UserLoginID: wa.UserLogin.ID, + WACallID: call.ID(), + RoomID: trigger.RoomID, + PortalKey: portal.PortalKey, + PeerJID: peer, + Direction: "outgoing", + MediaKind: mediaKind, + FocusType: focus.Type, + LiveKitServiceURL: focus.LiveKitServiceURL, + LiveKitRoom: trigger.RoomID.String(), + MatrixParticipantMXID: intent.GetMXID(), + MatrixSessionID: deviceID, + SelectedPublisherID: matrixRTCTriggerParticipantID(trigger), + AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, + State: "joining_livekit", + CreatedTS: now, + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + _ = call.Hangup() + return err + } + session := &voip.MatrixRTCSession{ + UserID: intent.GetMXID(), + DeviceID: deviceID, + MemberID: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + Intent: mediaKind, + Focus: *focus, + Created: now, + Expires: matrixRTCMembershipLifetime, + StickyKey: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + } + if err = wa.sendMatrixRTCMembership(ctx, intent, trigger.RoomID, session); err != nil { + return wa.failMatrixRTCActivation(ctx, record, "matrixrtc_membership_failed", err) + } + if err = wa.connectOutboundMatrixRTCCall(ctx, record, trigger); err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("room_id", trigger.RoomID). + Stringer("peer_jid", peer). + Msg("Failed to connect outbound MatrixRTC call to LiveKit") + return err + } + wa.UserLogin.Log.Info(). + Str("call_id", call.ID()). + Stringer("room_id", trigger.RoomID). + Stringer("peer_jid", peer). + Stringer("matrix_participant_mxid", trigger.Sender). + Msg("Started outbound WhatsApp call from MatrixRTC") + return nil +} + +func (wa *WhatsAppClient) sendMatrixRTCRing(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, waCallID string, session *voip.MatrixRTCSession) error { + now := time.Now() + notificationMode := wa.Main.Config.VOIP.MatrixRTC.NotificationEventCompat + if matrixRTCCompatAllowsModern(notificationMode) { + resp, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCNotificationEventType(), voip.BuildRTCNotificationContent(now, matrixRTCRingLifetime, session.Intent), 0) + if err != nil { + return err + } + if resp != nil { + session.NotificationEventID = resp.EventID + } + } + if matrixRTCCompatAllowsLegacy(notificationMode) { + _, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.LegacyCallNotifyEventType(), voip.BuildLegacyCallNotifyContent(waCallID, session.Intent), 0) + if err != nil { + return err + } + } + return wa.sendMatrixRTCMembership(ctx, intent, roomID, session) +} + +func (wa *WhatsAppClient) sendMatrixRTCMembership(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, session *voip.MatrixRTCSession) error { + now := time.Now() + membershipMode := wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat + modernMessageSent := false + if matrixRTCCompatAllowsModern(membershipMode) { + content := voip.BuildRTCMembershipContent(*session) + if _, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCMembershipEventType(event.MessageEventType), content, matrixRTCStickyDuration); err != nil { + return err + } + modernMessageSent = true + stateKey := voip.MatrixRTCStateKey(session.UserID, session.DeviceID) + if _, err := intent.SendState(ctx, roomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: content}, now); err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", roomID). + Str("state_key", stateKey). + Msg("Failed to send MatrixRTC membership state event after sticky message membership") + } + } + if matrixRTCCompatAllowsLegacy(membershipMode) { + _, err := intent.SendState(ctx, roomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: voip.BuildLegacyCallMemberContent(*session)}, now) + if err != nil { + if modernMessageSent { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", roomID). + Msg("Failed to send legacy MatrixRTC membership state event after modern membership") + return nil + } + return err + } + } + return nil +} + +func (wa *WhatsAppClient) clearMatrixRTCMembership(ctx context.Context, call *wadb.MatrixRTCCall) error { + if call.RoomID == "" || call.MatrixParticipantMXID == "" { + return nil + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + if intent == nil { + intent = wa.Main.Bridge.Bot + } + stickyKey := voip.MatrixRTCMemberID(call.MatrixParticipantMXID, call.MatrixSessionID) + emptyContent := voip.EmptyMatrixRTCContent(stickyKey) + now := time.Now() + membershipMode := wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat + modernMessageSent := false + if matrixRTCCompatAllowsModern(membershipMode) { + if _, err := sendMatrixRTCMessage(ctx, intent, call.RoomID, voip.RTCMembershipEventType(event.MessageEventType), emptyContent, matrixRTCStickyDuration); err != nil { + return err + } + modernMessageSent = true + stateKey := voip.MatrixRTCStateKey(call.MatrixParticipantMXID, call.MatrixSessionID) + if _, err := intent.SendState(ctx, call.RoomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", call.RoomID). + Str("state_key", stateKey). + Msg("Failed to clear MatrixRTC membership state event after sticky message cleanup") + } + } + if matrixRTCCompatAllowsLegacy(membershipMode) { + if _, err := intent.SendState(ctx, call.RoomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: map[string]any{}}, now); err != nil { + if modernMessageSent { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", call.RoomID). + Msg("Failed to clear legacy MatrixRTC membership state event after modern cleanup") + return nil + } + return err + } + } + return nil +} + +func (wa *WhatsAppConnector) cleanupFailedOutboundMatrixRTCStart(ctx context.Context, trigger voip.MatrixRTCEvent) error { + if trigger.RoomID == "" || wa.Bridge == nil || wa.Bridge.Bot == nil { + return nil + } + intent := wa.Bridge.Bot + now := time.Now() + membershipMode := wa.Config.VOIP.MatrixRTC.MembershipEventCompat + + if trigger.Kind == voip.MatrixRTCEventKindRTCMembership && matrixRTCCompatAllowsModern(membershipMode) { + emptyContent := voip.EmptyMatrixRTCContent(matrixRTCTriggerStickyKey(trigger)) + if _, err := sendMatrixRTCMessage(ctx, intent, trigger.RoomID, voip.RTCMembershipEventType(event.MessageEventType), emptyContent, matrixRTCStickyDuration); err != nil { + return err + } + stateKey := matrixRTCTriggerStateKey(trigger) + if stateKey != "" { + if _, err := intent.SendState(ctx, trigger.RoomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + return err + } + } + } + + if trigger.Kind == voip.MatrixRTCEventKindGroupCallMember && matrixRTCCompatAllowsLegacy(membershipMode) { + if _, err := intent.SendState(ctx, trigger.RoomID, voip.GroupCallMemberEventType(), trigger.StateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + return err + } + } + return nil +} + +func (wa *WhatsAppClient) activateMatrixRTCCall(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) error { + if call == nil { + return nil + } + call.State = "joining_livekit" + call.LastError = "" + call.SelectedPublisherID = matrixRTCTriggerParticipantID(trigger) + if err := wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + return err + } + authResp, err := wa.requestMatrixRTCLiveKitAuth(ctx, call, trigger) + if err != nil { + return err + } + if err = wa.VOIP.BridgeCallToLiveKit(ctx, call.WACallID, authResp, call.SelectedPublisherID); err != nil { + return wa.failMatrixRTCActivation(ctx, call, "livekit_bridge_failed", err) + } + now := time.Now() + call.State = "active" + call.JoinedTS = now + call.AnsweredTS = now + if authResp.RoomName != "" { + call.LiveKitRoom = authResp.RoomName + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + return err + } + wa.UserLogin.Log.Info(). + Str("call_id", call.WACallID). + Stringer("room_id", call.RoomID). + Stringer("trigger_sender", trigger.Sender). + Msg("Activated MatrixRTC LiveKit bridge for WhatsApp call") + return nil +} + +func (wa *WhatsAppClient) connectOutboundMatrixRTCCall(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) error { + authResp, err := wa.requestMatrixRTCLiveKitAuth(ctx, call, trigger) + if err != nil { + return err + } + if err = wa.VOIP.BridgeCallToLiveKit(ctx, call.WACallID, authResp, call.SelectedPublisherID); err != nil { + return wa.failMatrixRTCActivation(ctx, call, "livekit_bridge_failed", err) + } + now := time.Now() + call.State = "active" + call.JoinedTS = now + if authResp.RoomName != "" { + call.LiveKitRoom = authResp.RoomName + } + return wa.Main.DB.MatrixRTCCall.Put(ctx, call) +} + +func (wa *WhatsAppClient) requestMatrixRTCLiveKitAuth(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) (*voip.LiveKitAuthResponse, error) { + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + openIDToken, err := requestMatrixOpenIDToken(ctx, intent) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "matrix_openid_failed", err) + } + if matrixRTCCompatAllowsLegacy(wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat) { + authResp, err := voip.RequestLegacyLiveKitAuth(ctx, nil, call.LiveKitServiceURL, matrixRTCLegacyLiveKitAuthRequest(call, openIDToken)) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "livekit_auth_failed", err) + } + return authResp, nil + } + authResp, err := voip.RequestLiveKitAuth(ctx, nil, call.LiveKitServiceURL, matrixRTCLiveKitAuthRequest(call, openIDToken)) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "livekit_auth_failed", err) + } + return authResp, nil +} + +func (wa *WhatsAppClient) failMatrixRTCActivation(ctx context.Context, call *wadb.MatrixRTCCall, reason string, err error) error { + _ = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, call.UserLoginID, call.WACallID, "ended", reason, err.Error(), time.Now()) + if wa.VOIP != nil { + wa.VOIP.HandleMatrixRTCCallEvent(ctx, voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindRTCDecline, + RoomID: call.RoomID, + }, call.WACallID) + } + return err +} + +func matrixRTCLiveKitAuthRequest(call *wadb.MatrixRTCCall, openIDToken voip.MatrixOpenIDToken) voip.LiveKitAuthRequest { + if call == nil { + return voip.LiveKitAuthRequest{OpenIDToken: openIDToken} + } + return voip.LiveKitAuthRequest{ + RoomID: call.RoomID.String(), + SlotID: voip.MatrixRTCDefaultSlotID, + OpenIDToken: openIDToken, + Member: matrixRTCLiveKitAuthMember(call), + } +} + +func matrixRTCLegacyLiveKitAuthRequest(call *wadb.MatrixRTCCall, openIDToken voip.MatrixOpenIDToken) voip.LegacyLiveKitAuthRequest { + if call == nil { + return voip.LegacyLiveKitAuthRequest{OpenIDToken: openIDToken} + } + return voip.LegacyLiveKitAuthRequest{ + Room: call.RoomID.String(), + OpenIDToken: openIDToken, + DeviceID: call.MatrixSessionID, + } +} + +func matrixRTCLiveKitAuthMember(call *wadb.MatrixRTCCall) *voip.LiveKitAuthMember { + if call == nil { + return nil + } + return &voip.LiveKitAuthMember{ + ID: voip.MatrixRTCMemberID(call.MatrixParticipantMXID, call.MatrixSessionID), + ClaimedDeviceID: call.MatrixSessionID, + ClaimedUserID: call.MatrixParticipantMXID.String(), + } +} + +func (wa *WhatsAppClient) matrixRTCLiveKitFocusForTrigger(ctx context.Context, trigger voip.MatrixRTCEvent) (*voip.Focus, error) { + for _, focus := range trigger.FociPreferred { + if focus.Type == "livekit" && focus.LiveKitServiceURL != "" { + focusCopy := focus + return &focusCopy, nil + } + } + return voip.DiscoverLiveKitFocus(ctx, nil, wa.Main.Bridge.Matrix.ServerName(), wa.Main.Config.VOIP.MatrixRTC.LiveKitServiceURL) +} + +func (wa *WhatsAppClient) matrixRTCParticipantIntent(ctx context.Context, peer types.JID) (bridgev2.MatrixAPI, error) { + mode := strings.ToLower(wa.Main.Config.VOIP.MatrixRTC.ParticipantMode) + if mode == "" || mode == "whatsapp_ghost" { + if ghostID := waid.MakeUserID(peer); ghostID != "" { + ghost, err := wa.Main.Bridge.GetGhostByID(ctx, ghostID) + if err != nil { + return nil, err + } + if ghost != nil && ghost.Intent != nil { + return ghost.Intent, nil + } + } + } + return wa.Main.Bridge.Bot, nil +} + +func (wa *WhatsAppClient) matrixRTCIntentForMXID(ctx context.Context, mxid id.UserID) bridgev2.MatrixAPI { + if mxid == "" || mxid == wa.Main.Bridge.Bot.GetMXID() { + return wa.Main.Bridge.Bot + } + if ghost, err := wa.Main.Bridge.GetGhostByMXID(ctx, mxid); err == nil && ghost != nil && ghost.Intent != nil { + return ghost.Intent + } + return wa.Main.Bridge.Bot +} + +func matrixRTCPortalSupportsWhatsAppCalls(peer types.JID) bool { + switch peer.Server { + case types.DefaultUserServer, types.HiddenUserServer: + return true + default: + return false + } +} + +func matrixRTCCallIntent(call *meowcaller.Call) string { + if call != nil && call.IsVideo() { + return "video" + } + return "audio" +} + +func matrixRTCOutboundMediaKind(trigger voip.MatrixRTCEvent) (mediaKind string, downgraded bool) { + switch trigger.Intent { + case "", "audio": + return "audio", false + case "video": + return "video", false + default: + return "", false + } +} + +func matrixRTCTriggerParticipantID(trigger voip.MatrixRTCEvent) string { + if trigger.Sender == "" { + return "" + } + deviceID := trigger.SessionID + if deviceID == "" { + deviceID = trigger.DeviceID + } + return voip.MatrixRTCMemberID(trigger.Sender, deviceID) +} + +func matrixRTCTriggerStateKey(trigger voip.MatrixRTCEvent) string { + if trigger.StateKey != "" { + return trigger.StateKey + } + deviceID := trigger.SessionID + if deviceID == "" { + deviceID = trigger.DeviceID + } + if trigger.Sender == "" { + return deviceID + } + return voip.MatrixRTCStateKey(trigger.Sender, deviceID) +} + +func matrixRTCTriggerStickyKey(trigger voip.MatrixRTCEvent) string { + if stickyKey, ok := trigger.Raw["sticky_key"].(string); ok && stickyKey != "" { + return stickyKey + } + if stickyKey, ok := trigger.Raw["msc4354_sticky_key"].(string); ok && stickyKey != "" { + return stickyKey + } + return matrixRTCTriggerParticipantID(trigger) +} + +func sendMatrixRTCMessage(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, eventType event.Type, raw map[string]any, sticky time.Duration) (*mautrix.RespSendEvent, error) { + if asIntent, ok := intent.(*mxbridge.ASIntent); ok { + return asIntent.Matrix.SendMessageEvent(ctx, roomID, eventType, &event.Content{Raw: raw}, mautrix.ReqSendEvent{ + UnstableStickyDuration: sticky, + DontEncrypt: true, + }) + } + return intent.SendMessage(ctx, roomID, eventType, &event.Content{Raw: raw}, nil) +} + +func requestMatrixOpenIDToken(ctx context.Context, intent bridgev2.MatrixAPI) (voip.MatrixOpenIDToken, error) { + asIntent, ok := intent.(*mxbridge.ASIntent) + if !ok { + return voip.MatrixOpenIDToken{}, fmt.Errorf("matrix intent %T does not support OpenID token requests", intent) + } + if asIntent.Matrix == nil || asIntent.Matrix.Client == nil { + return voip.MatrixOpenIDToken{}, fmt.Errorf("matrix intent %T has no Matrix client", intent) + } + resp, err := asIntent.Matrix.Client.RequestOpenIDToken(ctx) + if err != nil { + return voip.MatrixOpenIDToken{}, err + } + return voip.MatrixOpenIDToken{ + AccessToken: resp.AccessToken, + TokenType: resp.TokenType, + MatrixServerName: resp.MatrixServerName, + ExpiresIn: resp.ExpiresIn, + }, nil +} + +func matrixRTCCompatAllowsModern(mode string) bool { + switch strings.ToLower(mode) { + case "legacy", "legacy_only", "msc3401", "org.matrix.msc3401.call.member": + return false + default: + return true + } +} + +func matrixRTCCompatAllowsLegacy(mode string) bool { + switch strings.ToLower(mode) { + case "modern", "modern_only", "msc4143", "org.matrix.msc4143.rtc.member", "none", "off", "false", "disabled": + return false + default: + return true + } +} diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go new file mode 100644 index 0000000..aa1d8c6 --- /dev/null +++ b/pkg/connector/matrixrtc_test.go @@ -0,0 +1,313 @@ +package connector + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func TestShouldStartOutboundMatrixRTCCall(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if !shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned false for active state membership") + } +} + +func TestShouldStartOutboundMatrixRTCCallRejectsMessageMembership(t *testing.T) { + evt := matrixRTCMemberEvent(event.MessageEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned true for message membership") + } +} + +func TestShouldStartOutboundMatrixRTCCallRejectsActivePreviousState(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + evt.Unsigned.PrevContent = &event.Content{Raw: matrixRTCMemberContent()} + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned true for an active-to-active state update") + } +} + +func TestMatrixRTCTriggerStateKeyUsesEventStateKey(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if stateKey := matrixRTCTriggerStateKey(parsed); stateKey != "@alice:example.com_DEVICE" { + t.Fatalf("state key = %q, want event state key", stateKey) + } +} + +func TestMatrixRTCTriggerStateKeyFallsBackToSenderAndSession(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + SessionID: "SESSION", + } + if stateKey := matrixRTCTriggerStateKey(parsed); stateKey != "@alice:example.com_SESSION" { + t.Fatalf("state key = %q, want sender/session-derived key", stateKey) + } +} + +func TestMatrixRTCTriggerStickyKeyPrefersContentStickyKey(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + SessionID: "SESSION", + Raw: map[string]any{ + "sticky_key": "sticky", + }, + } + if stickyKey := matrixRTCTriggerStickyKey(parsed); stickyKey != "sticky" { + t.Fatalf("sticky key = %q, want content sticky key", stickyKey) + } +} + +func TestMatrixRTCTriggerStickyKeyFallsBackToParticipantID(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + DeviceID: "DEVICE", + Raw: map[string]any{}, + } + if stickyKey := matrixRTCTriggerStickyKey(parsed); stickyKey != "@alice:example.com:DEVICE" { + t.Fatalf("sticky key = %q, want participant id", stickyKey) + } +} + +func TestMatrixRTCOutboundMediaKindDefaultsToAudio(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{}) + if mediaKind != "audio" { + t.Fatalf("mediaKind = %q, want audio", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindKeepsAudio(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "audio"}) + if mediaKind != "audio" { + t.Fatalf("mediaKind = %q, want audio", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindKeepsVideo(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "video"}) + if mediaKind != "video" { + t.Fatalf("mediaKind = %q, want video", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindRejectsUnknown(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "screen"}) + if mediaKind != "" { + t.Fatalf("mediaKind = %q, want empty", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestShouldEndMatrixRTCCallFromLegacyMembershipLeave(t *testing.T) { + stateKey := "_@alice:example.com_DEVICE_m.call" + evt := &event.Event{ + Type: voip.GroupCallMemberEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{}}, + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if !shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned false for selected participant leave") + } +} + +func TestShouldEndMatrixRTCCallFromMembershipKeepsActiveJoin(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned true for active join") + } +} + +func TestShouldEndMatrixRTCCallFromMembershipRejectsOtherParticipant(t *testing.T) { + stateKey := "_@alice:example.com_OTHER_m.call" + evt := &event.Event{ + Type: voip.GroupCallMemberEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{}}, + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned true for another participant") + } +} + +func TestReserveMatrixRTCOutboundStartSuppressesDuplicates(t *testing.T) { + wa := &WhatsAppConnector{} + if !wa.reserveMatrixRTCOutboundStart("!room:example.com") { + t.Fatalf("first reserve returned false") + } + if wa.reserveMatrixRTCOutboundStart("!room:example.com") { + t.Fatalf("second reserve returned true, want false") + } + if !wa.reserveMatrixRTCOutboundStart("!other:example.com") { + t.Fatalf("reserve for another room returned false") + } +} + +func TestMatrixRTCFinalEndReasonPreservesActivationFailure(t *testing.T) { + reason, lastErr := matrixRTCFinalEndReason(&wadb.MatrixRTCCall{ + EndedTS: time.Unix(123, 0), + EndReason: "livekit_bridge_failed", + LastError: "could not connect after timeout", + }, "rejected") + if reason != "livekit_bridge_failed" { + t.Fatalf("reason = %q, want livekit_bridge_failed", reason) + } + if lastErr != "could not connect after timeout" { + t.Fatalf("lastErr = %q, want timeout error", lastErr) + } +} + +func TestMatrixRTCFinalEndReasonUsesWhatsAppReasonForFreshEnd(t *testing.T) { + reason, lastErr := matrixRTCFinalEndReason(&wadb.MatrixRTCCall{}, "rejected") + if reason != "rejected" { + t.Fatalf("reason = %q, want rejected", reason) + } + if lastErr != "" { + t.Fatalf("lastErr = %q, want empty", lastErr) + } +} + +func TestMatrixRTCLiveKitAuthRequestUsesStrictJWTFields(t *testing.T) { + req := matrixRTCLiveKitAuthRequest(&wadb.MatrixRTCCall{ + RoomID: "!room:example.com", + MatrixParticipantMXID: "@whatsapp_123:example.com", + MatrixSessionID: "WA123", + }, voip.MatrixOpenIDToken{AccessToken: "openid"}) + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + for _, field := range []string{ + `"device_id"`, + `"session_id"`, + `"participant_id"`, + `"focus_type"`, + `"extra"`, + } { + if bytes.Contains(body, []byte(field)) { + t.Fatalf("request body contains strict-JWT-incompatible field %s: %s", field, string(body)) + } + } + if req.RoomID != "!room:example.com" || req.SlotID != voip.MatrixRTCDefaultSlotID { + t.Fatalf("unexpected room or slot in request: %+v", req) + } + if req.Member == nil || req.Member.ID != "@whatsapp_123:example.com:WA123" { + t.Fatalf("unexpected member in request: %+v", req.Member) + } +} + +func TestMatrixRTCLegacyLiveKitAuthRequestUsesRoomAndDevice(t *testing.T) { + req := matrixRTCLegacyLiveKitAuthRequest(&wadb.MatrixRTCCall{ + RoomID: "!room:example.com", + MatrixSessionID: "WA123", + }, voip.MatrixOpenIDToken{AccessToken: "openid"}) + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + if !bytes.Contains(body, []byte(`"room":"!room:example.com"`)) { + t.Fatalf("request body missing legacy room: %s", string(body)) + } + if !bytes.Contains(body, []byte(`"device_id":"WA123"`)) { + t.Fatalf("request body missing legacy device_id: %s", string(body)) + } + if bytes.Contains(body, []byte(`"member"`)) || bytes.Contains(body, []byte(`"slot_id"`)) { + t.Fatalf("legacy request body contains modern fields: %s", string(body)) + } +} + +func TestMatrixRTCCompatModeConfigValues(t *testing.T) { + tests := []struct { + mode string + wantModern bool + wantLegacy bool + }{ + {mode: "auto", wantModern: true, wantLegacy: true}, + {mode: "msc4143", wantModern: true, wantLegacy: false}, + {mode: "msc3401", wantModern: false, wantLegacy: true}, + } + for _, tt := range tests { + t.Run(tt.mode, func(t *testing.T) { + if got := matrixRTCCompatAllowsModern(tt.mode); got != tt.wantModern { + t.Fatalf("matrixRTCCompatAllowsModern(%q) = %v, want %v", tt.mode, got, tt.wantModern) + } + if got := matrixRTCCompatAllowsLegacy(tt.mode); got != tt.wantLegacy { + t.Fatalf("matrixRTCCompatAllowsLegacy(%q) = %v, want %v", tt.mode, got, tt.wantLegacy) + } + }) + } +} + +func matrixRTCMemberEvent(class event.TypeClass) *event.Event { + stateKey := "@alice:example.com_DEVICE" + return &event.Event{ + Type: voip.RTCMembershipEventType(class), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: matrixRTCMemberContent()}, + } +} + +func matrixRTCMemberContent() map[string]any { + return voip.BuildRTCMembershipContent(voip.MatrixRTCSession{ + UserID: "@alice:example.com", + DeviceID: "DEVICE", + Intent: "audio", + Focus: voip.Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + }) +} diff --git a/pkg/connector/voip/audio.go b/pkg/connector/voip/audio.go new file mode 100644 index 0000000..07e9877 --- /dev/null +++ b/pkg/connector/voip/audio.go @@ -0,0 +1,151 @@ +package voip + +import ( + "errors" + "fmt" + "io" + "math" + "sync" + + lkmedia "github.com/livekit/media-sdk" + "github.com/purpshell/meowcaller" +) + +var ErrAudioSourceClosed = errors.New("voip: audio source closed") + +func Float32FrameToPCM16(frame []float32) lkmedia.PCM16Sample { + sample := make(lkmedia.PCM16Sample, len(frame)) + for i, value := range frame { + switch { + case value > 1: + value = 1 + case value < -1: + value = -1 + } + if value == 1 { + sample[i] = math.MaxInt16 + } else { + sample[i] = int16(value * 32768) + } + } + return sample +} + +func PCM16ToFloat32Frame(sample lkmedia.PCM16Sample) []float32 { + frame := make([]float32, len(sample)) + for i, value := range sample { + frame[i] = float32(value) / 32768 + } + return frame +} + +type LiveKitPCMWriter struct { + mu sync.RWMutex + track interface { + WriteSample(lkmedia.PCM16Sample) error + } + closed bool +} + +func NewLiveKitPCMWriter(track interface { + WriteSample(lkmedia.PCM16Sample) error +}) *LiveKitPCMWriter { + return &LiveKitPCMWriter{track: track} +} + +func (w *LiveKitPCMWriter) WriteFrame(frame []float32) error { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return ErrAudioSourceClosed + } + if w.track == nil { + return nil + } + return w.track.WriteSample(Float32FrameToPCM16(frame)) +} + +func (w *LiveKitPCMWriter) Close() error { + w.mu.Lock() + w.closed = true + w.track = nil + w.mu.Unlock() + return nil +} + +type MeowcallerAudioSource struct { + mu sync.Mutex + cond *sync.Cond + queue []float32 + closed bool + maxSize int +} + +func NewMeowcallerAudioSource(maxFrames int) *MeowcallerAudioSource { + if maxFrames <= 0 { + maxFrames = 8 + } + src := &MeowcallerAudioSource{ + maxSize: maxFrames * meowcaller.FrameSamples, + } + src.cond = sync.NewCond(&src.mu) + return src +} + +func (src *MeowcallerAudioSource) WriteSample(sample lkmedia.PCM16Sample) error { + src.mu.Lock() + defer src.mu.Unlock() + if src.closed { + return ErrAudioSourceClosed + } + frame := PCM16ToFloat32Frame(sample) + src.queue = append(src.queue, frame...) + if len(src.queue) > src.maxSize { + copy(src.queue, src.queue[len(src.queue)-src.maxSize:]) + src.queue = src.queue[:src.maxSize] + } + src.cond.Signal() + return nil +} + +func (src *MeowcallerAudioSource) SampleRate() int { + return meowcaller.SampleRate +} + +func (src *MeowcallerAudioSource) String() string { + return fmt.Sprintf("MeowcallerAudioSource(%d)", meowcaller.SampleRate) +} + +func (src *MeowcallerAudioSource) ReadFrame() ([]float32, error) { + src.mu.Lock() + defer src.mu.Unlock() + for len(src.queue) < meowcaller.FrameSamples && !src.closed { + src.cond.Wait() + } + if len(src.queue) == 0 && src.closed { + return nil, io.EOF + } + frame := make([]float32, meowcaller.FrameSamples) + n := copy(frame, src.queue) + if n == len(src.queue) { + src.queue = src.queue[:0] + } else { + copy(src.queue, src.queue[n:]) + src.queue = src.queue[:len(src.queue)-n] + } + return frame, nil +} + +func (src *MeowcallerAudioSource) Close() error { + src.mu.Lock() + src.closed = true + src.queue = nil + src.cond.Broadcast() + src.mu.Unlock() + return nil +} + +var ( + _ meowcaller.AudioSink = (*LiveKitPCMWriter)(nil) + _ meowcaller.AudioSource = (*MeowcallerAudioSource)(nil) +) diff --git a/pkg/connector/voip/audio_test.go b/pkg/connector/voip/audio_test.go new file mode 100644 index 0000000..6cd44b0 --- /dev/null +++ b/pkg/connector/voip/audio_test.go @@ -0,0 +1,49 @@ +package voip + +import ( + "io" + "math" + "testing" + + lkmedia "github.com/livekit/media-sdk" + "github.com/purpshell/meowcaller" +) + +func TestFloat32FrameToPCM16ClipsAndScales(t *testing.T) { + frame := []float32{-2, -1, -0.5, 0, 0.5, 1, 2} + sample := Float32FrameToPCM16(frame) + expected := lkmedia.PCM16Sample{math.MinInt16, math.MinInt16, -16384, 0, 16384, math.MaxInt16, math.MaxInt16} + for i, value := range expected { + if sample[i] != value { + t.Fatalf("sample[%d] = %d, want %d", i, sample[i], value) + } + } +} + +func TestMeowcallerAudioSourceFramesAndEOF(t *testing.T) { + src := NewMeowcallerAudioSource(1) + sample := make(lkmedia.PCM16Sample, meowcaller.FrameSamples) + for i := range sample { + sample[i] = int16(i) + } + if err := src.WriteSample(sample); err != nil { + t.Fatalf("WriteSample returned error: %v", err) + } + frame, err := src.ReadFrame() + if err != nil { + t.Fatalf("ReadFrame returned error: %v", err) + } + if len(frame) != meowcaller.FrameSamples { + t.Fatalf("frame length = %d, want %d", len(frame), meowcaller.FrameSamples) + } + if frame[1] != float32(1)/32768 { + t.Fatalf("frame[1] = %f, want %f", frame[1], float32(1)/32768) + } + if err = src.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + _, err = src.ReadFrame() + if err != io.EOF { + t.Fatalf("ReadFrame after close returned %v, want io.EOF", err) + } +} diff --git a/pkg/connector/voip/config.go b/pkg/connector/voip/config.go new file mode 100644 index 0000000..0325bd1 --- /dev/null +++ b/pkg/connector/voip/config.go @@ -0,0 +1,54 @@ +package voip + +import "time" + +type Config struct { + Enabled bool + IncomingPolicy string + MaxActiveCallsPerLogin int + MatrixRTC MatrixRTCConfig + LiveKit LiveKitConfig + Audio AudioConfig + Video VideoConfig + Diagnostics DiagnosticsConfig +} + +type MatrixRTCConfig struct { + LiveKitServiceURL string + RequireLiveKitFocus bool + MembershipEventCompat string + NotificationEventCompat string + UseDelayedEvents bool + ParticipantMode string + FallbackParticipantMXID string +} + +type LiveKitConfig struct { + ConnectTimeout time.Duration + PublishSilenceBeforeWhatsAppAnswer bool + AutoSubscribe bool + AudioUplinkPolicy string + SelectedParticipantTimeout time.Duration +} + +type AudioConfig struct { + Enabled bool + JitterBuffer time.Duration + OpusBackend string + SilenceOnUnderrun bool + MaxMixParticipants int +} + +type VideoConfig struct { + Enabled bool + SelectedSourcePolicy string + MaxWidth int + MaxHeight int + MaxFPS int +} + +type DiagnosticsConfig struct { + HealthcheckFocusOnStartup bool + EnableMeowcallerDiagnostics bool + MediaTraceDir string +} diff --git a/pkg/connector/voip/errors.go b/pkg/connector/voip/errors.go new file mode 100644 index 0000000..a6a97e5 --- /dev/null +++ b/pkg/connector/voip/errors.go @@ -0,0 +1,6 @@ +package voip + +import "errors" + +var ErrNotEnabled = errors.New("voip: MatrixRTC LiveKit bridge is not enabled") +var ErrCallNotFound = errors.New("voip: call not found") diff --git a/pkg/connector/voip/focus.go b/pkg/connector/voip/focus.go new file mode 100644 index 0000000..e1490c1 --- /dev/null +++ b/pkg/connector/voip/focus.go @@ -0,0 +1,241 @@ +package voip + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +const FocusWellKnownKey = "org.matrix.msc4143.rtc_foci" + +var ErrNoLiveKitFocus = errors.New("voip: no LiveKit MatrixRTC focus found") + +type Focus struct { + Type string `json:"type"` + LiveKitServiceURL string `json:"livekit_service_url"` +} + +type WellKnownClient struct { + RTCFoci []Focus `json:"org.matrix.msc4143.rtc_foci"` +} + +func DiscoverLiveKitFocus(ctx context.Context, httpClient *http.Client, serverName, overrideURL string) (*Focus, error) { + if overrideURL != "" { + if err := validateHTTPSURL(overrideURL); err != nil { + return nil, fmt.Errorf("invalid configured livekit service URL: %w", err) + } + return &Focus{Type: "livekit", LiveKitServiceURL: overrideURL}, nil + } + if serverName == "" { + return nil, fmt.Errorf("matrix server name is required") + } + if strings.Contains(serverName, "://") { + return nil, fmt.Errorf("matrix server name must not include a scheme") + } + if httpClient == nil { + httpClient = http.DefaultClient + } + wellKnownURL := "https://" + serverName + "/.well-known/matrix/client" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, wellKnownURL, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch Matrix client well-known: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("Matrix client well-known returned HTTP %d", resp.StatusCode) + } + var wellKnown WellKnownClient + if err = json.NewDecoder(resp.Body).Decode(&wellKnown); err != nil { + return nil, fmt.Errorf("failed to decode Matrix client well-known: %w", err) + } + for _, focus := range wellKnown.RTCFoci { + if focus.Type != "livekit" || focus.LiveKitServiceURL == "" { + continue + } + if err = validateHTTPSURL(focus.LiveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit focus URL in well-known: %w", err) + } + return &focus, nil + } + return nil, ErrNoLiveKitFocus +} + +type MatrixOpenIDToken struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + MatrixServerName string `json:"matrix_server_name"` + ExpiresIn int `json:"expires_in"` +} + +type LiveKitAuthRequest struct { + RoomID string `json:"room_id,omitempty"` + SlotID string `json:"slot_id,omitempty"` + OpenIDToken MatrixOpenIDToken `json:"openid_token"` + Member *LiveKitAuthMember `json:"member,omitempty"` + DeviceID string `json:"device_id,omitempty"` + SessionID string `json:"session_id,omitempty"` + ParticipantID string `json:"participant_id,omitempty"` + FocusType string `json:"focus_type,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +type LegacyLiveKitAuthRequest struct { + Room string `json:"room"` + OpenIDToken MatrixOpenIDToken `json:"openid_token"` + DeviceID string `json:"device_id"` +} + +type LiveKitAuthMember struct { + ID string `json:"id,omitempty"` + ClaimedDeviceID string `json:"claimed_device_id,omitempty"` + ClaimedUserID string `json:"claimed_user_id,omitempty"` +} + +type LiveKitAuthResponse struct { + URL string `json:"url,omitempty"` + Token string `json:"token,omitempty"` + JWTToken string `json:"jwt,omitempty"` + RoomName string `json:"room,omitempty"` + + ServerURL string `json:"server_url,omitempty"` + LiveKitURL string `json:"livekit_url,omitempty"` + AccessToken string `json:"access_token,omitempty"` +} + +func (resp LiveKitAuthResponse) ConnectionURL() string { + for _, candidate := range []string{resp.URL, resp.ServerURL, resp.LiveKitURL} { + if candidate != "" { + return candidate + } + } + return "" +} + +func (resp LiveKitAuthResponse) JWT() string { + for _, candidate := range []string{resp.Token, resp.JWTToken, resp.AccessToken} { + if candidate != "" { + return candidate + } + } + return "" +} + +func RequestLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, authReq LiveKitAuthRequest) (*LiveKitAuthResponse, error) { + if err := validateHTTPSURL(liveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit service URL: %w", err) + } + if httpClient == nil { + httpClient = http.DefaultClient + } + body, err := json.Marshal(authReq) + if err != nil { + return nil, err + } + var lastErr error + for _, endpoint := range liveKitAuthEndpoints(liveKitServiceURL) { + resp, err := postLiveKitAuth(ctx, httpClient, endpoint, body) + if err != nil { + lastErr = err + continue + } + if resp.ConnectionURL() == "" || resp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + return resp, nil + } + if lastErr != nil { + return nil, lastErr + } + return nil, fmt.Errorf("livekit auth did not try any endpoints") +} + +func RequestLegacyLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, authReq LegacyLiveKitAuthRequest) (*LiveKitAuthResponse, error) { + if err := validateHTTPSURL(liveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit service URL: %w", err) + } + if httpClient == nil { + httpClient = http.DefaultClient + } + body, err := json.Marshal(authReq) + if err != nil { + return nil, err + } + resp, err := postLiveKitAuth(ctx, httpClient, legacyLiveKitAuthEndpoint(liveKitServiceURL), body) + if err != nil { + return nil, err + } + if resp.ConnectionURL() == "" || resp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + return resp, nil +} + +func liveKitAuthEndpoints(rawURL string) []string { + trimmed := strings.TrimRight(rawURL, "/") + if strings.HasSuffix(trimmed, "/get_token") || strings.HasSuffix(trimmed, "/sfu/get") { + return []string{trimmed} + } + return []string{ + trimmed + "/get_token", + trimmed + "/sfu/get", + trimmed, + } +} + +func legacyLiveKitAuthEndpoint(rawURL string) string { + trimmed := strings.TrimRight(rawURL, "/") + if strings.HasSuffix(trimmed, "/sfu/get") { + return trimmed + } + if strings.HasSuffix(trimmed, "/get_token") { + return strings.TrimSuffix(trimmed, "/get_token") + "/sfu/get" + } + return trimmed + "/sfu/get" +} + +func postLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, body []byte) (*LiveKitAuthResponse, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, liveKitServiceURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to request livekit token: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("livekit auth returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + var authResp LiveKitAuthResponse + if err = json.NewDecoder(resp.Body).Decode(&authResp); err != nil { + return nil, fmt.Errorf("failed to decode livekit auth response: %w", err) + } + return &authResp, nil +} + +func validateHTTPSURL(rawURL string) error { + parsed, err := url.Parse(rawURL) + if err != nil { + return err + } + if parsed.Scheme != "https" && parsed.Scheme != "wss" { + return fmt.Errorf("URL must use https or wss") + } + if parsed.Host == "" { + return fmt.Errorf("URL must include a host") + } + return nil +} diff --git a/pkg/connector/voip/focus_test.go b/pkg/connector/voip/focus_test.go new file mode 100644 index 0000000..e41b1d8 --- /dev/null +++ b/pkg/connector/voip/focus_test.go @@ -0,0 +1,131 @@ +package voip + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestDiscoverLiveKitFocusOverride(t *testing.T) { + focus, err := DiscoverLiveKitFocus(context.Background(), nil, "", "https://rtc.example.com/livekit/jwt") + if err != nil { + t.Fatalf("DiscoverLiveKitFocus returned error: %v", err) + } + if focus.Type != "livekit" || focus.LiveKitServiceURL != "https://rtc.example.com/livekit/jwt" { + t.Fatalf("unexpected focus: %+v", focus) + } +} + +func TestRequestLiveKitAuthAcceptsResponseAliases(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/get_token" { + t.Fatalf("path = %q, want /get_token", r.URL.Path) + } + var req LiveKitAuthRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + if req.OpenIDToken.AccessToken != "openid" { + t.Fatalf("OpenID token = %q, want openid", req.OpenIDToken.AccessToken) + } + _ = json.NewEncoder(w).Encode(LiveKitAuthResponse{ + ServerURL: "wss://livekit.example.com", + JWTToken: "jwt", + }) + })) + defer server.Close() + + resp, err := RequestLiveKitAuth(context.Background(), server.Client(), server.URL, LiveKitAuthRequest{ + OpenIDToken: MatrixOpenIDToken{AccessToken: "openid"}, + }) + if err != nil { + t.Fatalf("RequestLiveKitAuth returned error: %v", err) + } + if resp.ConnectionURL() != "wss://livekit.example.com" || resp.JWT() != "jwt" { + t.Fatalf("unexpected response aliases: %+v", resp) + } +} + +func TestRequestLegacyLiveKitAuthUsesSFUEndpoint(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/sfu/get" { + t.Fatalf("path = %q, want /sfu/get", r.URL.Path) + } + var req LegacyLiveKitAuthRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + if req.Room != "!room:example.com" || req.DeviceID != "DEVICE" { + t.Fatalf("unexpected legacy request: %+v", req) + } + _ = json.NewEncoder(w).Encode(LiveKitAuthResponse{ + URL: "wss://livekit.example.com", + Token: "jwt", + }) + })) + defer server.Close() + + resp, err := RequestLegacyLiveKitAuth(context.Background(), server.Client(), server.URL, LegacyLiveKitAuthRequest{ + Room: "!room:example.com", + DeviceID: "DEVICE", + OpenIDToken: MatrixOpenIDToken{AccessToken: "openid", MatrixServerName: "example.com"}, + }) + if err != nil { + t.Fatalf("RequestLegacyLiveKitAuth returned error: %v", err) + } + if resp.ConnectionURL() != "wss://livekit.example.com" || resp.JWT() != "jwt" { + t.Fatalf("unexpected response: %+v", resp) + } +} + +func TestLiveKitAuthEndpoints(t *testing.T) { + tests := []struct { + name string + url string + want []string + }{ + { + name: "base", + url: "https://rtc.example.com/livekit/jwt", + want: []string{ + "https://rtc.example.com/livekit/jwt/get_token", + "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt", + }, + }, + { + name: "explicit", + url: "https://rtc.example.com/livekit/jwt/sfu/get", + want: []string{"https://rtc.example.com/livekit/jwt/sfu/get"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := liveKitAuthEndpoints(test.url) + if len(got) != len(test.want) { + t.Fatalf("got %d endpoints, want %d: %+v", len(got), len(test.want), got) + } + for i := range got { + if got[i] != test.want[i] { + t.Fatalf("endpoint %d = %q, want %q", i, got[i], test.want[i]) + } + } + }) + } +} + +func TestLegacyLiveKitAuthEndpoint(t *testing.T) { + tests := map[string]string{ + "https://rtc.example.com/livekit/jwt": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/get_token": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/sfu/get": "https://rtc.example.com/livekit/jwt/sfu/get", + } + for input, want := range tests { + if got := legacyLiveKitAuthEndpoint(input); got != want { + t.Fatalf("legacy endpoint for %q = %q, want %q", input, got, want) + } + } +} diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go new file mode 100644 index 0000000..198bbac --- /dev/null +++ b/pkg/connector/voip/livekit.go @@ -0,0 +1,480 @@ +package voip + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + lkpcm "github.com/livekit/media-sdk" + livekitproto "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + lksdk "github.com/livekit/server-sdk-go/v2" + lkmedia "github.com/livekit/server-sdk-go/v2/pkg/media" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media/samplebuilder" + "github.com/rs/zerolog" +) + +type LiveKitParticipant struct { + cfg LiveKitConfig + videoCfg VideoConfig + log zerolog.Logger + room *lksdk.Room + audio *lkmedia.PCMLocalTrack + audioPub *lksdk.LocalTrackPublication + audioSrc *MeowcallerAudioSource + video *lksdk.LocalTrack + videoPub *lksdk.LocalTrackPublication + + mu sync.Mutex + remoteAudio []*lkmedia.PCMRemoteTrack + remoteMediaCancel context.CancelFunc + disconnected bool + selectedRemoteParticipant string + remoteAudioMuteStateChange func(muted bool) + remoteVideoFrame func(frame LiveKitVideoFrame) error + remoteVideoMuteStateChange func(muted bool) +} + +func ConnectLiveKitParticipant(ctx context.Context, authResp *LiveKitAuthResponse, cfg LiveKitConfig, videoCfg VideoConfig, log zerolog.Logger) (*LiveKitParticipant, error) { + if authResp == nil { + return nil, fmt.Errorf("livekit auth response is nil") + } + if authResp.ConnectionURL() == "" || authResp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + remoteMediaCtx, remoteMediaCancel := context.WithCancel(context.Background()) + participant := &LiveKitParticipant{ + cfg: cfg, + videoCfg: videoCfg, + log: log, + audioSrc: NewMeowcallerAudioSource(12), + remoteMediaCancel: remoteMediaCancel, + } + callback := &lksdk.RoomCallback{ + ParticipantCallback: lksdk.ParticipantCallback{ + OnTrackSubscribed: func(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + participant.onTrackSubscribed(remoteMediaCtx, track, publication, rp) + }, + OnTrackUnsubscribed: participant.onTrackUnsubscribed, + OnTrackMuted: participant.onTrackMuted, + OnTrackUnmuted: participant.onTrackUnmuted, + }, + OnDisconnected: func() { + participant.closeRemoteTracks() + }, + OnDisconnectedWithReason: func(reason lksdk.DisconnectionReason) { + log.Info().Str("reason", string(reason)).Msg("Disconnected from LiveKit") + participant.closeRemoteTracks() + }, + } + opts := []lksdk.ConnectOption{ + lksdk.WithAutoSubscribe(cfg.AutoSubscribe), + } + if cfg.ConnectTimeout > 0 { + opts = append(opts, lksdk.WithConnectTimeout(cfg.ConnectTimeout)) + } + room, err := connectLiveKit(ctx, authResp.ConnectionURL(), authResp.JWT(), callback, opts...) + if err != nil { + remoteMediaCancel() + return nil, err + } + participant.room = room + return participant, nil +} + +func connectLiveKit(ctx context.Context, url, token string, callback *lksdk.RoomCallback, opts ...lksdk.ConnectOption) (*lksdk.Room, error) { + room := lksdk.NewRoom(callback) + if err := room.JoinWithContextAndToken(ctx, url, token, opts...); err != nil { + return nil, err + } + return room, nil +} + +func (p *LiveKitParticipant) SetRemoteAudioMuteHandler(selectedParticipant string, handler func(muted bool)) { + p.mu.Lock() + p.selectedRemoteParticipant = selectedParticipant + p.remoteAudioMuteStateChange = handler + p.mu.Unlock() +} + +func (p *LiveKitParticipant) SetRemoteVideoHandlers(selectedParticipant string, frameHandler func(frame LiveKitVideoFrame) error, muteHandler func(muted bool)) { + p.mu.Lock() + p.selectedRemoteParticipant = selectedParticipant + p.remoteVideoFrame = frameHandler + p.remoteVideoMuteStateChange = muteHandler + p.mu.Unlock() +} + +func (p *LiveKitParticipant) PublishAudioTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.audio != nil { + return nil + } + track, err := lkmedia.NewPCMLocalTrack(meowcallerSampleRate, 1, logger.GetLogger()) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-audio" + } + pub, err := p.room.LocalParticipant.PublishTrack(track, &lksdk.TrackPublicationOptions{Name: name}) + if err != nil { + track.Close() + return err + } + p.audio = track + p.audioPub = pub + return nil +} + +func (p *LiveKitParticipant) PublishVideoTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.video != nil { + return nil + } + track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-video" + } + pub, err := p.room.LocalParticipant.PublishTrack(track, &lksdk.TrackPublicationOptions{ + Name: name, + Source: livekitproto.TrackSource_CAMERA, + VideoWidth: p.videoCfg.MaxWidth, + VideoHeight: p.videoCfg.MaxHeight, + }) + if err != nil { + _ = track.Close() + return err + } + p.video = track + p.videoPub = pub + return nil +} + +func (p *LiveKitParticipant) SetWhatsAppAudioMuted(muted bool) { + p.mu.Lock() + pub := p.audioPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp audio mute state") +} + +func (p *LiveKitParticipant) SetWhatsAppVideoMuted(muted bool) { + p.mu.Lock() + pub := p.videoPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp video mute state") +} + +func (p *LiveKitParticipant) WhatsAppSink() *LiveKitPCMWriter { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitPCMWriter(p.audio) +} + +func (p *LiveKitParticipant) WhatsAppVideoSink() *LiveKitH264Writer { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitH264Writer(p.video, videoFrameDuration(p.videoCfg)) +} + +func (p *LiveKitParticipant) MatrixAudioSource() *MeowcallerAudioSource { + return p.audioSrc +} + +func (p *LiveKitParticipant) WriteWhatsAppFrame(frame []float32) error { + p.mu.Lock() + audio := p.audio + p.mu.Unlock() + if audio == nil { + return nil + } + return audio.WriteSample(Float32FrameToPCM16(frame)) +} + +func (p *LiveKitParticipant) Close() { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return + } + p.disconnected = true + room := p.room + audio := p.audio + audioPub := p.audioPub + video := p.video + videoPub := p.videoPub + p.room = nil + p.audio = nil + p.audioPub = nil + p.video = nil + p.videoPub = nil + p.mu.Unlock() + p.closeRemoteTracks() + if audioPub != nil { + audioPub.SetMuted(true) + } + if videoPub != nil { + videoPub.SetMuted(true) + } + if audio != nil { + audio.ClearQueue() + _ = audio.Close() + } + if video != nil { + _ = video.Close() + } + if room != nil { + room.Disconnect() + } + _ = p.audioSrc.Close() +} + +func (p *LiveKitParticipant) onTrackSubscribed(ctx context.Context, track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + switch track.Kind() { + case webrtc.RTPCodecTypeAudio: + p.onAudioTrackSubscribed(track, publication, rp) + case webrtc.RTPCodecTypeVideo: + p.onVideoTrackSubscribed(ctx, track, publication, rp) + } +} + +func (p *LiveKitParticipant) onAudioTrackSubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if track.Codec().MimeType != webrtc.MimeTypeOpus { + p.log.Warn(). + Str("codec", track.Codec().MimeType). + Str("participant", string(rp.Identity())). + Msg("Ignoring non-Opus LiveKit audio track") + return + } + remote, err := lkmedia.NewPCMRemoteTrack( + track, + p.audioSrc, + lkmedia.WithTargetSampleRate(meowcallerSampleRate), + lkmedia.WithTargetChannels(1), + lkmedia.WithLogger(logger.GetLogger()), + ) + if err != nil { + p.log.Warn(). + Err(err). + Str("participant", string(rp.Identity())). + Msg("Failed to subscribe LiveKit audio track") + return + } + p.mu.Lock() + p.remoteAudio = append(p.remoteAudio, remote) + p.mu.Unlock() + p.handleRemoteAudioMuteState(publication, rp, publication.IsMuted()) + _ = publication +} + +func (p *LiveKitParticipant) onVideoTrackSubscribed(ctx context.Context, track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if !p.videoCfg.Enabled { + return + } + if !remoteParticipantSelected(p.selectedParticipant(), string(rp.Identity())) { + p.log.Debug(). + Str("participant", string(rp.Identity())). + Str("selected_participant", p.selectedParticipant()). + Msg("Ignoring LiveKit video track from non-selected participant") + return + } + if !strings.EqualFold(track.Codec().MimeType, webrtc.MimeTypeH264) { + p.log.Warn(). + Str("codec", track.Codec().MimeType). + Str("participant", string(rp.Identity())). + Msg("Ignoring unsupported LiveKit video track; only H.264 passthrough is implemented") + p.handleRemoteVideoMuteState(publication, rp, true) + return + } + p.handleRemoteVideoMuteState(publication, rp, publication.IsMuted()) + go p.forwardRemoteH264Track(ctx, track, rp) +} + +func (p *LiveKitParticipant) onTrackUnsubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if track.Kind() == webrtc.RTPCodecTypeVideo { + p.handleRemoteVideoMuteState(publication, rp, true) + } +} + +func (p *LiveKitParticipant) onTrackMuted(pub lksdk.TrackPublication, participant lksdk.Participant) { + p.handleRemoteAudioMuteState(pub, participant, true) + p.handleRemoteVideoMuteState(pub, participant, true) +} + +func (p *LiveKitParticipant) onTrackUnmuted(pub lksdk.TrackPublication, participant lksdk.Participant) { + p.handleRemoteAudioMuteState(pub, participant, false) + p.handleRemoteVideoMuteState(pub, participant, false) +} + +func (p *LiveKitParticipant) handleRemoteAudioMuteState(pub lksdk.TrackPublication, participant lksdk.Participant, muted bool) { + if pub == nil || participant == nil || pub.Kind() != lksdk.TrackKindAudio { + return + } + if _, ok := participant.(*lksdk.RemoteParticipant); !ok { + return + } + identity := participant.Identity() + p.mu.Lock() + selected := p.selectedRemoteParticipant + handler := p.remoteAudioMuteStateChange + p.mu.Unlock() + if selected != "" && identity != selected { + p.log.Debug(). + Str("participant", identity). + Str("selected_participant", selected). + Bool("muted", muted). + Msg("Ignoring LiveKit mute state from non-selected participant") + return + } + p.log.Debug(). + Str("participant", identity). + Str("track_id", pub.SID()). + Bool("muted", muted). + Msg("Observed LiveKit remote audio mute state") + if handler != nil { + handler(muted) + } +} + +func (p *LiveKitParticipant) handleRemoteVideoMuteState(pub lksdk.TrackPublication, participant lksdk.Participant, muted bool) { + if pub == nil || participant == nil || pub.Kind() != lksdk.TrackKindVideo { + return + } + if _, ok := participant.(*lksdk.RemoteParticipant); !ok { + return + } + identity := participant.Identity() + p.mu.Lock() + selected := p.selectedRemoteParticipant + handler := p.remoteVideoMuteStateChange + p.mu.Unlock() + if !remoteParticipantSelected(selected, identity) { + p.log.Debug(). + Str("participant", identity). + Str("selected_participant", selected). + Bool("muted", muted). + Msg("Ignoring LiveKit video mute state from non-selected participant") + return + } + p.log.Debug(). + Str("participant", identity). + Str("track_id", pub.SID()). + Bool("muted", muted). + Msg("Observed LiveKit remote video mute state") + if handler != nil { + handler(muted) + } +} + +func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track *webrtc.TrackRemote, rp *lksdk.RemoteParticipant) { + builder := samplebuilder.New( + liveKitH264MaxLatePackets, + &codecs.H264Packet{}, + track.Codec().ClockRate, + ) + p.log.Info(). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Msg("Started forwarding LiveKit H.264 video to WhatsApp") + for { + if ctx.Err() != nil { + return + } + packet, _, err := track.ReadRTP() + if err != nil { + if !errors.Is(err, io.EOF) && ctx.Err() == nil { + p.log.Debug(). + Err(err). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Msg("Stopped reading LiveKit H.264 video track") + } + return + } + builder.Push(packet) + for sample := builder.Pop(); sample != nil; sample = builder.Pop() { + if len(sample.Data) == 0 { + continue + } + p.mu.Lock() + handler := p.remoteVideoFrame + p.mu.Unlock() + if handler == nil { + continue + } + if err = handler(LiveKitVideoFrame{ + AccessUnit: sample.Data, + Duration: sample.Duration, + }); err != nil { + p.log.Warn(). + Err(err). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Msg("Failed to forward LiveKit H.264 frame to WhatsApp") + } + } + } +} + +func (p *LiveKitParticipant) selectedParticipant() string { + p.mu.Lock() + defer p.mu.Unlock() + return p.selectedRemoteParticipant +} + +func remoteParticipantSelected(selected, identity string) bool { + return selected == "" || identity == selected +} + +func (p *LiveKitParticipant) closeRemoteTracks() { + p.mu.Lock() + tracks := p.remoteAudio + p.remoteAudio = nil + cancel := p.remoteMediaCancel + p.remoteMediaCancel = nil + p.mu.Unlock() + if cancel != nil { + cancel() + } + for _, track := range tracks { + track.Close() + } +} + +const meowcallerSampleRate = 16000 +const liveKitH264ClockRate = 90000 +const liveKitH264MaxLatePackets = 1000 + +func videoFrameDuration(cfg VideoConfig) time.Duration { + if cfg.MaxFPS <= 0 { + return time.Second / 30 + } + return time.Second / time.Duration(cfg.MaxFPS) +} + +var _ lkpcm.PCM16Writer = (*MeowcallerAudioSource)(nil) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go new file mode 100644 index 0000000..6fe7bc9 --- /dev/null +++ b/pkg/connector/voip/manager.go @@ -0,0 +1,694 @@ +package voip + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/purpshell/meowcaller" + "github.com/purpshell/meowcaller/diag" + "github.com/purpshell/meowcaller/signaling" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow" + waBinary "go.mau.fi/whatsmeow/binary" + "go.mau.fi/whatsmeow/types" +) + +const ( + localUnmuteState = "0" + localMuteState = "1" + localVideoInactiveState = 0 +) + +var localMuteRetryIntervals = []time.Duration{ + 0, + 2 * time.Second, + 2 * time.Second, + 2 * time.Second, +} + +type Manager struct { + cfg Config + waClient *whatsmeow.Client + client *meowcaller.Client + log zerolog.Logger + + mu sync.Mutex + calls map[string]*meowcaller.Call + callCreators map[string]types.JID + livekit map[string]*LiveKitParticipant + livekitConnecting map[string]struct{} + matrixAudioMuted map[string]bool + matrixVideoMuted map[string]bool + whatsAppMuted map[string]bool + whatsAppVideoMuted map[string]bool + incomingCallNotify func(*meowcaller.Call) + callEndNotify func(callID, reason string) +} + +func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Manager { + manager := &Manager{ + cfg: cfg, + waClient: waClient, + log: log, + calls: make(map[string]*meowcaller.Call), + callCreators: make(map[string]types.JID), + livekit: make(map[string]*LiveKitParticipant), + livekitConnecting: make(map[string]struct{}), + matrixAudioMuted: make(map[string]bool), + matrixVideoMuted: make(map[string]bool), + whatsAppMuted: make(map[string]bool), + whatsAppVideoMuted: make(map[string]bool), + } + if !cfg.Enabled || waClient == nil { + return manager + } + opts := []meowcaller.Option{meowcaller.WithLogger(log)} + if cfg.Diagnostics.EnableMeowcallerDiagnostics { + rec, err := diag.NewRecorder(cfg.Diagnostics.MediaTraceDir) + if err != nil { + log.Warn(). + Err(err). + Str("media_trace_dir", cfg.Diagnostics.MediaTraceDir). + Msg("Failed to enable meowcaller media diagnostics") + } else { + opts = append(opts, meowcaller.WithDiagnostics(rec)) + log.Warn(). + Str("media_trace_dir", cfg.Diagnostics.MediaTraceDir). + Msg("Enabled unsafe meowcaller media diagnostics") + } + } + manager.client = meowcaller.NewClient(waClient, opts...) + manager.client.OnIncomingCall(manager.handleIncomingCall) + return manager +} + +func (m *Manager) Enabled() bool { + return m != nil && m.cfg.Enabled && m.client != nil +} + +func (m *Manager) Client() *meowcaller.Client { + if m == nil { + return nil + } + return m.client +} + +func (m *Manager) SetIncomingCallHandler(handler func(*meowcaller.Call)) { + if m == nil { + return + } + m.mu.Lock() + m.incomingCallNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetCallEndHandler(handler func(callID, reason string)) { + if m == nil { + return + } + m.mu.Lock() + m.callEndNotify = handler + m.mu.Unlock() +} + +func (m *Manager) Dial(ctx context.Context, target string, video ...bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + opts := meowcaller.CallOptions{} + if len(video) > 0 { + opts.Video = video[0] + } + call, err := m.client.CallWithOptions(ctx, target, opts) + if err != nil { + return nil, err + } + m.trackCall(call, m.ownCallCreator()) + return call, nil +} + +func (m *Manager) AbortAll() { + if m == nil { + return + } + m.mu.Lock() + calls := make([]*meowcaller.Call, 0, len(m.calls)) + for _, call := range m.calls { + calls = append(calls, call) + } + m.calls = make(map[string]*meowcaller.Call) + m.callCreators = make(map[string]types.JID) + m.matrixAudioMuted = make(map[string]bool) + m.matrixVideoMuted = make(map[string]bool) + m.whatsAppMuted = make(map[string]bool) + m.whatsAppVideoMuted = make(map[string]bool) + participants := make([]*LiveKitParticipant, 0, len(m.livekit)) + for _, participant := range m.livekit { + participants = append(participants, participant) + } + m.livekit = make(map[string]*LiveKitParticipant) + m.livekitConnecting = make(map[string]struct{}) + m.mu.Unlock() + for _, participant := range participants { + participant.Close() + } + for _, call := range calls { + if err := call.Hangup(); err != nil { + m.log.Debug().Err(err).Str("call_id", call.ID()).Msg("Failed to hang up VOIP call during abort") + } + } +} + +func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, authResp *LiveKitAuthResponse, selectedRemoteParticipantID string) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[waCallID] + existing := m.livekit[waCallID] + _, connecting := m.livekitConnecting[waCallID] + if call != nil && existing == nil && !connecting { + m.livekitConnecting[waCallID] = struct{}{} + } + m.mu.Unlock() + if call == nil { + return ErrCallNotFound + } + if existing != nil || connecting { + return nil + } + participant, err := ConnectLiveKitParticipant(ctx, authResp, m.cfg.LiveKit, m.cfg.Video, m.log.With().Str("call_id", waCallID).Str("component", "livekit").Logger()) + if err != nil { + m.clearLiveKitConnecting(waCallID) + return err + } + participant.SetRemoteAudioMuteHandler(selectedRemoteParticipantID, func(muted bool) { + m.handleMatrixAudioMuteState(call, muted) + }) + videoEnabled := m.cfg.Video.Enabled && call.IsVideo() + if videoEnabled { + var videoBuffer whatsAppVideoStartupBuffer + var videoBufferLock sync.Mutex + participant.SetRemoteVideoHandlers(selectedRemoteParticipantID, func(frame LiveKitVideoFrame) error { + if call.State() == meowcaller.CallPhaseEnded { + return nil + } + videoBufferLock.Lock() + bufferedBefore := videoBuffer.Len() + flushed, err := videoBuffer.Send(frame, func(frame LiveKitVideoFrame) error { + return call.SendVideoWithDuration(frame.AccessUnit, frame.Duration) + }) + bufferedAfter := videoBuffer.Len() + videoBufferLock.Unlock() + if err != nil { + m.log.Debug(). + Err(err). + Str("call_id", call.ID()). + Int("buffered_frames", bufferedAfter). + Msg("Buffered LiveKit H.264 frame until WhatsApp video media is ready") + return nil + } + if bufferedBefore > 0 && bufferedAfter == 0 { + m.log.Info(). + Str("call_id", call.ID()). + Int("flushed_frames", flushed). + Msg("Flushed buffered LiveKit H.264 video to WhatsApp") + } + return nil + }, func(muted bool) { + m.handleMatrixVideoMuteState(call, muted) + }) + } + if err = participant.PublishAudioTrack("whatsapp-audio"); err != nil { + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + call.Receive(participant.WhatsAppSink()) + call.Play(participant.MatrixAudioSource()) + if videoEnabled { + if err = participant.PublishVideoTrack("whatsapp-video"); err != nil { + call.Receive(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + call.ReceiveVideo(participant.WhatsAppVideoSink()) + } + answeredIncoming := call.State() == meowcaller.CallPhaseRinging + if call.State() == meowcaller.CallPhaseRinging { + if err = call.Answer(); err != nil { + call.Receive(nil) + call.ReceiveVideo(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + } + m.mu.Lock() + whatsAppMuted, knownWhatsAppMute := m.whatsAppMuted[waCallID] + whatsAppVideoMuted, knownWhatsAppVideoMute := m.whatsAppVideoMuted[waCallID] + delete(m.livekitConnecting, waCallID) + m.livekit[waCallID] = participant + m.mu.Unlock() + if knownWhatsAppMute { + participant.SetWhatsAppAudioMuted(whatsAppMuted) + } + if knownWhatsAppVideoMute { + participant.SetWhatsAppVideoMuted(whatsAppVideoMuted) + } + if answeredIncoming { + m.log.Debug().Str("call_id", waCallID).Msg("Answered incoming WhatsApp call before sending local unmute") + } + go m.sendLocalMuteStateRetries(call) + if videoEnabled { + go m.sendLocalVideoStateRetries(call) + } + return nil +} + +func (m *Manager) sendLocalMuteStateRetries(call *meowcaller.Call) { + if m == nil || m.waClient == nil || call == nil { + return + } + for attempt, interval := range localMuteRetryIntervals { + time.Sleep(interval) + if call.State() == meowcaller.CallPhaseEnded { + return + } + muted := m.currentMatrixAudioMuted(call.ID()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalMuteState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localMuteStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Failed to send WhatsApp local mute state") + continue + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Sent WhatsApp local mute state") + } +} + +func (m *Manager) sendLocalVideoStateRetries(call *meowcaller.Call) { + if m == nil || m.waClient == nil || call == nil || !call.IsVideo() { + return + } + for attempt, interval := range localMuteRetryIntervals { + time.Sleep(interval) + if call.State() == meowcaller.CallPhaseEnded { + return + } + muted := m.currentMatrixVideoMuted(call.ID()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalVideoState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localVideoStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Failed to send WhatsApp local video state") + continue + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Sent WhatsApp local video state") + } +} + +func (m *Manager) handleMatrixAudioMuteState(call *meowcaller.Call, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + m.mu.Lock() + previous, known := m.matrixAudioMuted[call.ID()] + m.matrixAudioMuted[call.ID()] = muted + m.mu.Unlock() + if known && previous == muted { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalMuteState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localMuteStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Failed to send WhatsApp local mute state from LiveKit") + return + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Sent WhatsApp local mute state from LiveKit") +} + +func (m *Manager) handleMatrixVideoMuteState(call *meowcaller.Call, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded || !call.IsVideo() { + return + } + m.mu.Lock() + previous, known := m.matrixVideoMuted[call.ID()] + m.matrixVideoMuted[call.ID()] = muted + m.mu.Unlock() + if known && previous == muted { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalVideoState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localVideoStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Failed to send WhatsApp local video state from LiveKit") + return + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Sent WhatsApp local video state from LiveKit") +} + +func (m *Manager) handleWhatsAppAudioMuteState(callID string, muted bool) { + if m == nil || callID == "" { + return + } + m.mu.Lock() + m.whatsAppMuted[callID] = muted + participant := m.livekit[callID] + m.mu.Unlock() + if participant != nil { + participant.SetWhatsAppAudioMuted(muted) + } + m.log.Debug(). + Str("call_id", callID). + Bool("muted", muted). + Msg("Observed WhatsApp remote audio mute state") +} + +func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.VideoState) { + if m == nil || callID == "" { + return + } + muted := !state.Active && !state.Upgrade + m.mu.Lock() + m.whatsAppVideoMuted[callID] = muted + participant := m.livekit[callID] + m.mu.Unlock() + if participant != nil { + participant.SetWhatsAppVideoMuted(muted) + } + m.log.Debug(). + Str("call_id", callID). + Bool("muted", muted). + Bool("active", state.Active). + Bool("upgrade", state.Upgrade). + Int("orientation", state.Orientation). + Int("raw_state", state.Raw). + Msg("Observed WhatsApp remote video state") +} + +func (m *Manager) currentMatrixAudioMuted(callID string) bool { + if m == nil { + return false + } + m.mu.Lock() + muted := m.matrixAudioMuted[callID] + m.mu.Unlock() + return muted +} + +func (m *Manager) currentMatrixVideoMuted(callID string) bool { + if m == nil { + return false + } + m.mu.Lock() + muted := m.matrixVideoMuted[callID] + m.mu.Unlock() + return muted +} + +func localMuteStateFor(muted bool) string { + if muted { + return localMuteState + } + return localUnmuteState +} + +func localVideoStateFor(muted bool) int { + if muted { + return localVideoInactiveState + } + return signaling.VideoStateActive +} + +func (m *Manager) sendLocalMuteState(ctx context.Context, callID string, peer, callCreator types.JID, muteState string) error { + if m == nil || m.waClient == nil { + return fmt.Errorf("whatsapp client is not available") + } + if callID == "" { + return fmt.Errorf("call ID is empty") + } + if peer.IsEmpty() { + return fmt.Errorf("peer JID is empty") + } + if callCreator.IsEmpty() { + return fmt.Errorf("call creator JID is empty") + } + node := buildLocalMuteV2Node(callID, peer, callCreator, string(m.waClient.GenerateMessageID()), muteState) + //lint:ignore SA1019 low-level call signaling is not exposed by whatsmeow's public API + if err := m.waClient.DangerousInternals().SendNode(ctx, node); err != nil { + return fmt.Errorf("send mute_v2: %w", err) + } + return nil +} + +func (m *Manager) sendLocalVideoState(ctx context.Context, callID string, peer, callCreator types.JID, videoState int) error { + if m == nil || m.waClient == nil { + return fmt.Errorf("whatsapp client is not available") + } + if callID == "" { + return fmt.Errorf("call ID is empty") + } + if peer.IsEmpty() { + return fmt.Errorf("peer JID is empty") + } + if callCreator.IsEmpty() { + return fmt.Errorf("call creator JID is empty") + } + codec := "" + if videoState == signaling.VideoStateActive { + codec = signaling.VideoStateDecH264 + } + node := signaling.BuildVideoState(callID, peer, callCreator, string(m.waClient.GenerateMessageID()), videoState, 0, codec) + //lint:ignore SA1019 low-level call signaling is not exposed by whatsmeow's public API + if err := m.waClient.DangerousInternals().SendNode(ctx, node); err != nil { + return fmt.Errorf("send video state: %w", err) + } + return nil +} + +func buildLocalMuteV2Node(callID string, peer, callCreator types.JID, wrapperID, muteState string) waBinary.Node { + node := signaling.BuildMuteV2(callID, peer, callCreator, muteState) + if wrapperID != "" { + node.Attrs["id"] = wrapperID + } + return node +} + +func (m *Manager) HandleMatrixRTCEvent(ctx context.Context, evt MatrixRTCEvent) int { + return m.HandleMatrixRTCCallEvent(ctx, evt, "") +} + +func (m *Manager) HandleMatrixRTCCallEvent(_ context.Context, evt MatrixRTCEvent, waCallID string) int { + if !m.Enabled() { + return 0 + } + switch evt.Kind { + case MatrixRTCEventKindRTCDecline: + ended := m.endCallsFromMatrixRTC(waCallID) + if ended == 0 { + m.log.Debug(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Msg("Received MatrixRTC decline with no active WhatsApp VOIP calls") + } else { + m.log.Info(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Int("ended_call_count", ended). + Msg("Ended WhatsApp VOIP calls after MatrixRTC decline") + } + return ended + case MatrixRTCEventKindRTCMembership, MatrixRTCEventKindGroupCallMember, MatrixRTCEventKindRTCNotification, MatrixRTCEventKindLegacyCallNotify, MatrixRTCEventKindGroupCall: + m.log.Debug(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Str("matrixrtc_kind", string(evt.Kind)). + Msg("Observed MatrixRTC event") + } + return 0 +} + +func (m *Manager) handleIncomingCall(call *meowcaller.Call) { + m.trackCall(call, call.Peer()) + m.log.Info(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("video", call.IsVideo()). + Msg("Received incoming WhatsApp call for MatrixRTC bridge") + if m.cfg.IncomingPolicy == "notice" { + if err := call.Reject(); err != nil { + m.log.Warn().Err(err).Str("call_id", call.ID()).Msg("Failed to reject VOIP call handled as notice") + } + return + } + m.mu.Lock() + handler := m.incomingCallNotify + m.mu.Unlock() + if handler != nil { + go handler(call) + } +} + +func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { + if call == nil { + return + } + if callCreator.IsEmpty() { + callCreator = call.Peer() + } + m.mu.Lock() + m.calls[call.ID()] = call + m.callCreators[call.ID()] = callCreator + m.mu.Unlock() + call.OnEnd(func(reason string) { + m.mu.Lock() + delete(m.calls, call.ID()) + delete(m.callCreators, call.ID()) + delete(m.matrixAudioMuted, call.ID()) + delete(m.matrixVideoMuted, call.ID()) + delete(m.whatsAppMuted, call.ID()) + delete(m.whatsAppVideoMuted, call.ID()) + participant := m.livekit[call.ID()] + delete(m.livekit, call.ID()) + delete(m.livekitConnecting, call.ID()) + handler := m.callEndNotify + m.mu.Unlock() + if participant != nil { + participant.Close() + } + m.log.Info().Str("call_id", call.ID()).Str("reason", reason).Msg("WhatsApp VOIP call ended") + if handler != nil { + go handler(call.ID(), reason) + } + }) + call.OnStateChange(func(phase meowcaller.CallPhase) { + m.log.Debug().Str("call_id", call.ID()).Int("phase", int(phase)).Msg("WhatsApp VOIP call state changed") + }) + call.OnMuteState(func(muted bool) { + m.handleWhatsAppAudioMuteState(call.ID(), muted) + }) + call.OnVideoState(func(state meowcaller.VideoState) { + m.handleWhatsAppVideoState(call.ID(), state) + }) +} + +func (m *Manager) callCreatorFor(call *meowcaller.Call) types.JID { + if m == nil || call == nil { + return types.EmptyJID + } + m.mu.Lock() + callCreator := m.callCreators[call.ID()] + m.mu.Unlock() + if !callCreator.IsEmpty() { + return callCreator + } + if call.State() == meowcaller.CallPhaseCalling { + return m.ownCallCreator() + } + return call.Peer() +} + +func (m *Manager) ownCallCreator() types.JID { + if m == nil || m.waClient == nil || m.waClient.Store == nil { + return types.EmptyJID + } + return m.waClient.Store.GetLID() +} + +func (m *Manager) endCallsFromMatrixRTC(waCallID string) int { + m.mu.Lock() + calls := make([]*meowcaller.Call, 0, len(m.calls)) + if waCallID != "" { + if call := m.calls[waCallID]; call != nil { + calls = append(calls, call) + } + } else { + for _, call := range m.calls { + calls = append(calls, call) + } + } + m.mu.Unlock() + + var ended int + for _, call := range calls { + if call.State() == meowcaller.CallPhaseEnded { + continue + } + var err error + if call.State() == meowcaller.CallPhaseRinging { + err = call.Reject() + } else { + err = call.Hangup() + } + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Int("phase", int(call.State())). + Msg("Failed to end WhatsApp VOIP call after MatrixRTC event") + continue + } + ended++ + } + return ended +} + +func (m *Manager) clearLiveKitConnecting(waCallID string) { + m.mu.Lock() + delete(m.livekitConnecting, waCallID) + m.mu.Unlock() +} diff --git a/pkg/connector/voip/manager_test.go b/pkg/connector/voip/manager_test.go new file mode 100644 index 0000000..0f39de0 --- /dev/null +++ b/pkg/connector/voip/manager_test.go @@ -0,0 +1,121 @@ +package voip + +import ( + "errors" + "testing" + "time" + + "github.com/purpshell/meowcaller/signaling" + "go.mau.fi/whatsmeow/types" +) + +func TestBuildLocalMuteV2Node(t *testing.T) { + peer := types.NewJID("12345", types.HiddenUserServer) + callCreator := types.NewJID("67890", types.HiddenUserServer) + node := buildLocalMuteV2Node("call-id", peer, callCreator, "wrapper-id", localUnmuteState) + if node.Tag != "call" { + t.Fatalf("node tag = %q, want call", node.Tag) + } + if got := node.AttrGetter().JID("to"); got != peer { + t.Fatalf("to = %s, want %s", got, peer) + } + if got := node.AttrGetter().String("id"); got != "wrapper-id" { + t.Fatalf("wrapper id = %q, want wrapper-id", got) + } + children := node.GetChildren() + if len(children) != 1 { + t.Fatalf("children = %d, want 1", len(children)) + } + mute := children[0] + if mute.Tag != "mute_v2" { + t.Fatalf("child tag = %q, want mute_v2", mute.Tag) + } + attrs := mute.AttrGetter() + if got := attrs.String("call-id"); got != "call-id" { + t.Fatalf("call-id = %q, want call-id", got) + } + if got := attrs.JID("call-creator"); got != callCreator { + t.Fatalf("call-creator = %s, want %s", got, callCreator) + } + if got := attrs.String("mute-state"); got != localUnmuteState { + t.Fatalf("mute-state = %q, want %q", got, localUnmuteState) + } +} + +func TestLocalMuteStateFor(t *testing.T) { + if got := localMuteStateFor(false); got != localUnmuteState { + t.Fatalf("unmuted state = %q, want %q", got, localUnmuteState) + } + if got := localMuteStateFor(true); got != localMuteState { + t.Fatalf("muted state = %q, want %q", got, localMuteState) + } +} + +func TestLocalVideoStateFor(t *testing.T) { + if got := localVideoStateFor(false); got != signaling.VideoStateActive { + t.Fatalf("unmuted video state = %d, want %d", got, signaling.VideoStateActive) + } + if got := localVideoStateFor(true); got != localVideoInactiveState { + t.Fatalf("muted video state = %d, want %d", got, localVideoInactiveState) + } +} + +func TestWhatsAppVideoStartupBufferRetriesEarlyFrames(t *testing.T) { + notReady := errors.New("meowcaller: call has no active video media") + var attempts int + var sent [][]byte + var durations []time.Duration + buffer := whatsAppVideoStartupBuffer{} + + flushed, err := buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{1}, Duration: 33 * time.Millisecond}, func(frame LiveKitVideoFrame) error { + attempts++ + return notReady + }) + if err != notReady { + t.Fatalf("first send error = %v, want notReady", err) + } + if flushed != 0 || buffer.Len() != 1 { + t.Fatalf("flushed=%d buffered=%d, want flushed=0 buffered=1", flushed, buffer.Len()) + } + + flushed, err = buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{2}, Duration: 17 * time.Millisecond}, func(frame LiveKitVideoFrame) error { + attempts++ + sent = append(sent, append([]byte(nil), frame.AccessUnit...)) + durations = append(durations, frame.Duration) + return nil + }) + if err != nil { + t.Fatalf("second send returned error: %v", err) + } + if flushed != 2 || buffer.Len() != 0 { + t.Fatalf("flushed=%d buffered=%d, want flushed=2 buffered=0", flushed, buffer.Len()) + } + if attempts != 3 { + t.Fatalf("attempts=%d, want 3", attempts) + } + if len(sent) != 2 || sent[0][0] != 1 || sent[1][0] != 2 { + t.Fatalf("sent frames = %#v, want [1] then [2]", sent) + } + if len(durations) != 2 || durations[0] != 33*time.Millisecond || durations[1] != 17*time.Millisecond { + t.Fatalf("sent durations = %v, want 33ms then 17ms", durations) + } + if got := buffer.ready; !got { + t.Fatalf("buffer ready = %v, want true", got) + } +} + +func TestWhatsAppVideoStartupBufferCapsPendingFrames(t *testing.T) { + notReady := errors.New("not ready") + buffer := whatsAppVideoStartupBuffer{} + for i := 0; i < maxPendingWhatsAppVideoFrames+3; i++ { + _, _ = buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{byte(i)}}, func(LiveKitVideoFrame) error { + return notReady + }) + } + if buffer.Len() != maxPendingWhatsAppVideoFrames { + t.Fatalf("buffered frames = %d, want %d", buffer.Len(), maxPendingWhatsAppVideoFrames) + } + if got := buffer.frames[0].AccessUnit[0]; got != 3 { + t.Fatalf("oldest retained frame = %d, want 3", got) + } +} diff --git a/pkg/connector/voip/matrixrtc.go b/pkg/connector/voip/matrixrtc.go new file mode 100644 index 0000000..1e0e7f9 --- /dev/null +++ b/pkg/connector/voip/matrixrtc.go @@ -0,0 +1,520 @@ +package voip + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "time" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +const ( + EventTypeGroupCall = "org.matrix.msc3401.call" + EventTypeGroupCallMember = "org.matrix.msc3401.call.member" + EventTypeRTCMembership = "org.matrix.msc4143.rtc.member" + EventTypeRTCNotification = "org.matrix.msc4075.rtc.notification" + EventTypeCallNotify = "org.matrix.msc4075.call.notify" + EventTypeRTCDecline = "org.matrix.msc4310.rtc.decline" + + MatrixRTCApplicationCall = "m.call" + MatrixRTCDefaultSlotID = "m.call#ROOM" + MatrixRTCMembershipV1 = "org.matrix.msc4143.rtc.member" +) + +var supportedMatrixRTCEventTypes = []event.Type{ + {Type: EventTypeGroupCall, Class: event.StateEventType}, + {Type: EventTypeGroupCallMember, Class: event.StateEventType}, + {Type: EventTypeRTCMembership, Class: event.StateEventType}, + {Type: EventTypeRTCMembership, Class: event.MessageEventType}, + {Type: EventTypeRTCNotification, Class: event.MessageEventType}, + {Type: EventTypeCallNotify, Class: event.MessageEventType}, + {Type: EventTypeRTCDecline, Class: event.MessageEventType}, +} + +type MatrixRTCEventKind string + +const ( + MatrixRTCEventKindUnknown MatrixRTCEventKind = "" + MatrixRTCEventKindGroupCall MatrixRTCEventKind = "group_call" + MatrixRTCEventKindGroupCallMember MatrixRTCEventKind = "group_call_member" + MatrixRTCEventKindRTCMembership MatrixRTCEventKind = "rtc_membership" + MatrixRTCEventKindRTCNotification MatrixRTCEventKind = "rtc_notification" + MatrixRTCEventKindLegacyCallNotify MatrixRTCEventKind = "legacy_call_notify" + MatrixRTCEventKindRTCDecline MatrixRTCEventKind = "rtc_decline" +) + +type MatrixRTCEvent struct { + Type event.Type + Kind MatrixRTCEventKind + RoomID id.RoomID + Sender id.UserID + StateKey string + CallID string + DeviceID string + SessionID string + Intent string + LifetimeMS int + FociPreferred []Focus + Raw map[string]any +} + +type MatrixRTCSession struct { + UserID id.UserID + DeviceID string + MemberID string + CallID string + Intent string + Focus Focus + Created time.Time + Expires time.Duration + StickyKey string + NotificationEventID id.EventID +} + +func SupportedMatrixRTCEventTypes() []event.Type { + return append([]event.Type(nil), supportedMatrixRTCEventTypes...) +} + +func ClassifyMatrixRTCEventType(evtType event.Type) MatrixRTCEventKind { + switch evtType.Type { + case EventTypeGroupCall: + return MatrixRTCEventKindGroupCall + case EventTypeGroupCallMember: + return MatrixRTCEventKindGroupCallMember + case EventTypeRTCMembership: + return MatrixRTCEventKindRTCMembership + case EventTypeRTCNotification: + return MatrixRTCEventKindRTCNotification + case EventTypeCallNotify: + return MatrixRTCEventKindLegacyCallNotify + case EventTypeRTCDecline: + return MatrixRTCEventKindRTCDecline + default: + return MatrixRTCEventKindUnknown + } +} + +func ParseMatrixRTCEvent(evt *event.Event) (MatrixRTCEvent, bool) { + if evt == nil { + return MatrixRTCEvent{}, false + } + kind := ClassifyMatrixRTCEventType(evt.Type) + if kind == MatrixRTCEventKindUnknown { + return MatrixRTCEvent{}, false + } + raw := rawMatrixRTCContent(evt) + parsed := MatrixRTCEvent{ + Type: evt.Type, + Kind: kind, + RoomID: evt.RoomID, + Sender: evt.Sender, + Raw: raw, + } + if evt.StateKey != nil { + parsed.StateKey = *evt.StateKey + } + fillMatrixRTCFields(&parsed, raw) + return parsed, true +} + +func rawMatrixRTCContent(evt *event.Event) map[string]any { + if evt.Content.Raw != nil { + return evt.Content.Raw + } + if len(evt.Content.VeryRaw) > 0 { + var raw map[string]any + if err := json.Unmarshal(evt.Content.VeryRaw, &raw); err == nil && raw != nil { + return raw + } + } + if evt.Content.Parsed != nil { + data, err := json.Marshal(evt.Content.Parsed) + if err == nil { + var raw map[string]any + if err = json.Unmarshal(data, &raw); err == nil && raw != nil { + return raw + } + } + } + return map[string]any{} +} + +func fillMatrixRTCFields(parsed *MatrixRTCEvent, raw map[string]any) { + parsed.CallID = firstString(raw, "call_id", "m.call_id", "callId", "callID") + parsed.DeviceID = firstString(raw, "device_id", "m.device_id", "deviceId", "deviceID") + parsed.SessionID = firstString(raw, "session_id", "m.session_id", "sessionId", "sessionID") + parsed.Intent = firstString(raw, "intent", "m.call.intent", "call_intent") + parsed.LifetimeMS = firstInt(raw, "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms") + forEachObject(raw["application"], func(application map[string]any) { + if parsed.Intent == "" { + parsed.Intent = firstString(application, "intent", "m.call.intent", "call_intent") + } + }) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["rtc_transports"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["foci_preferred"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["m.foci_preferred"])...) + + forEachObject(raw["memberships"], func(membership map[string]any) { + if parsed.CallID == "" { + parsed.CallID = firstString(membership, "call_id", "m.call_id", "callId", "callID") + } + if parsed.DeviceID == "" { + parsed.DeviceID = firstString(membership, "device_id", "m.device_id", "deviceId", "deviceID") + } + if parsed.SessionID == "" { + parsed.SessionID = firstString(membership, "session_id", "m.session_id", "sessionId", "sessionID") + } + if parsed.Intent == "" { + parsed.Intent = firstString(membership, "intent", "m.call.intent", "call_intent") + } + if parsed.LifetimeMS == 0 { + parsed.LifetimeMS = firstInt(membership, "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms") + } + forEachObject(membership["application"], func(application map[string]any) { + if parsed.Intent == "" { + parsed.Intent = firstString(application, "intent", "m.call.intent", "call_intent") + } + }) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["rtc_transports"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["foci_preferred"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["m.foci_preferred"])...) + }) + + if parsed.DeviceID == "" { + parsed.DeviceID = parsed.StateKey + } +} + +func firstString(raw map[string]any, keys ...string) string { + for _, key := range keys { + if value, ok := raw[key]; ok { + if str, ok := value.(string); ok { + return str + } + } + } + return "" +} + +func firstInt(raw map[string]any, keys ...string) int { + for _, key := range keys { + value, ok := raw[key] + if !ok { + continue + } + switch typed := value.(type) { + case int: + return typed + case int64: + return int(typed) + case float64: + return int(typed) + case json.Number: + if integer, err := typed.Int64(); err == nil { + return int(integer) + } + } + } + return 0 +} + +func MatrixRTCEventHasJoinContent(evt MatrixRTCEvent) bool { + switch evt.Kind { + case MatrixRTCEventKindRTCMembership, MatrixRTCEventKindGroupCallMember: + return matrixRTCContentHasJoinData(evt.Raw) + default: + return false + } +} + +func matrixRTCContentHasJoinData(raw map[string]any) bool { + if len(raw) == 0 { + return false + } + if matrixRTCModernContentHasJoinData(raw) || matrixRTCLegacyContentHasJoinData(raw) { + return true + } + hasJoin := false + forEachObject(raw["memberships"], func(membership map[string]any) { + if matrixRTCMembershipArrayItemHasJoinData(membership) { + hasJoin = true + } + }) + return hasJoin +} + +func matrixRTCModernContentHasJoinData(raw map[string]any) bool { + if slotID := firstString(raw, "slot_id"); slotID != "" && slotID != MatrixRTCDefaultSlotID { + return false + } + return matrixRTCApplicationIsCall(raw["application"]) && + matrixRTCContentHasMember(raw) && + len(parseFoci(raw["rtc_transports"])) > 0 +} + +func matrixRTCLegacyContentHasJoinData(raw map[string]any) bool { + if !matrixRTCApplicationIsCall(raw["application"]) || + !matrixRTCContentHasIdentifier(raw) || + !matrixRTCContentHasPositiveLifetime(raw) { + return false + } + return len(parseFoci(raw["foci_preferred"])) > 0 || + len(parseFoci(raw["m.foci_preferred"])) > 0 +} + +func matrixRTCMembershipArrayItemHasJoinData(raw map[string]any) bool { + if application, ok := raw["application"]; ok && !matrixRTCApplicationIsCall(application) { + return false + } + if !matrixRTCContentHasIdentifier(raw) || !matrixRTCContentHasPositiveLifetime(raw) { + return false + } + return len(parseFoci(raw["rtc_transports"])) > 0 || + len(parseFoci(raw["foci_preferred"])) > 0 || + len(parseFoci(raw["m.foci_preferred"])) > 0 +} + +func matrixRTCApplicationIsCall(value any) bool { + switch typed := value.(type) { + case string: + return typed == MatrixRTCApplicationCall + case map[string]any: + return firstString(typed, "type", "application") == MatrixRTCApplicationCall + case []any: + for _, item := range typed { + if matrixRTCApplicationIsCall(item) { + return true + } + } + case []map[string]any: + for _, item := range typed { + if matrixRTCApplicationIsCall(item) { + return true + } + } + } + return false +} + +func matrixRTCContentHasMember(raw map[string]any) bool { + hasMember := false + forEachObject(raw["member"], func(member map[string]any) { + if firstString(member, "user_id", "device_id", "id") != "" { + hasMember = true + } + }) + return hasMember +} + +func matrixRTCContentHasIdentifier(raw map[string]any) bool { + return matrixRTCContentHasMember(raw) || + firstString(raw, "membershipID", "membership_id", "device_id", "m.device_id", "deviceId", "deviceID", "session_id", "m.session_id", "sessionId", "sessionID") != "" +} + +func matrixRTCContentHasPositiveLifetime(raw map[string]any) bool { + for _, key := range []string{"expires", "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms"} { + if _, ok := raw[key]; ok { + return firstInt(raw, key) > 0 + } + } + return true +} + +func forEachObject(value any, fn func(map[string]any)) { + switch typed := value.(type) { + case []any: + for _, item := range typed { + if object, ok := item.(map[string]any); ok { + fn(object) + } + } + case []map[string]any: + for _, item := range typed { + fn(item) + } + case map[string]any: + fn(typed) + } +} + +func parseFoci(value any) []Focus { + var output []Focus + forEachObject(value, func(rawFocus map[string]any) { + if firstString(rawFocus, "type") != "livekit" { + return + } + serviceURL := firstString(rawFocus, "livekit_service_url", "livekit_service_url_prefix", "service_url") + if serviceURL == "" { + return + } + output = append(output, Focus{ + Type: "livekit", + LiveKitServiceURL: serviceURL, + }) + }) + return output +} + +func MatrixRTCDeviceID(loginID, waCallID string) string { + sum := sha256.Sum256([]byte(loginID + "\x00" + waCallID)) + return "WA" + hex.EncodeToString(sum[:8]) +} + +func MatrixRTCMemberID(userID id.UserID, deviceID string) string { + if deviceID == "" { + return userID.String() + } + return userID.String() + ":" + deviceID +} + +func MatrixRTCStateKey(userID id.UserID, deviceID string) string { + if deviceID == "" { + return userID.String() + } + return userID.String() + "_" + deviceID +} + +func RTCMembershipEventType(class event.TypeClass) event.Type { + return event.Type{Type: EventTypeRTCMembership, Class: class} +} + +func GroupCallMemberEventType() event.Type { + return event.Type{Type: EventTypeGroupCallMember, Class: event.StateEventType} +} + +func RTCNotificationEventType() event.Type { + return event.Type{Type: EventTypeRTCNotification, Class: event.MessageEventType} +} + +func LegacyCallNotifyEventType() event.Type { + return event.Type{Type: EventTypeCallNotify, Class: event.MessageEventType} +} + +func BuildRTCMembershipContent(session MatrixRTCSession) map[string]any { + deviceID := session.DeviceID + memberID := session.MemberID + if memberID == "" { + memberID = MatrixRTCMemberID(session.UserID, deviceID) + } + stickyKey := session.StickyKey + if stickyKey == "" { + stickyKey = memberID + } + intent := session.Intent + if intent == "" { + intent = "audio" + } + application := map[string]any{ + "type": MatrixRTCApplicationCall, + "m.call.intent": intent, + } + content := map[string]any{ + "slot_id": MatrixRTCDefaultSlotID, + "member": map[string]any{ + "user_id": session.UserID.String(), + "device_id": deviceID, + "id": memberID, + }, + "application": application, + "rtc_transports": []map[string]any{liveKitTransport(session.Focus)}, + "versions": []string{MatrixRTCMembershipV1}, + "sticky_key": stickyKey, + "msc4354_sticky_key": stickyKey, + } + if session.NotificationEventID != "" { + content["m.relates_to"] = map[string]any{ + "rel_type": "m.reference", + "event_id": session.NotificationEventID.String(), + } + } + return content +} + +func BuildLegacyCallMemberContent(session MatrixRTCSession) map[string]any { + deviceID := session.DeviceID + memberID := session.MemberID + if memberID == "" { + memberID = MatrixRTCMemberID(session.UserID, deviceID) + } + intent := session.Intent + if intent == "" { + intent = "audio" + } + created := session.Created + if created.IsZero() { + created = time.Now() + } + expires := session.Expires + if expires <= 0 { + expires = 4 * time.Hour + } + return map[string]any{ + "application": MatrixRTCApplicationCall, + "call_id": "", + "device_id": deviceID, + "focus_active": map[string]any{ + "type": "livekit", + "focus_selection": "multi_sfu", + }, + "foci_preferred": []map[string]any{liveKitTransport(session.Focus)}, + "created_ts": created.UnixMilli(), + "scope": "m.room", + "expires": expires.Milliseconds(), + "m.call.intent": intent, + "membershipID": memberID, + } +} + +func BuildRTCNotificationContent(now time.Time, lifetime time.Duration, intent string) map[string]any { + if now.IsZero() { + now = time.Now() + } + if lifetime <= 0 || lifetime > 90*time.Second { + lifetime = 90 * time.Second + } + if intent == "" { + intent = "audio" + } + return map[string]any{ + "notification_type": "ring", + "sender_ts": now.UnixMilli(), + "lifetime": lifetime.Milliseconds(), + "m.call.intent": intent, + "m.mentions": map[string]any{}, + } +} + +func BuildLegacyCallNotifyContent(callID, intent string) map[string]any { + if intent == "" { + intent = "audio" + } + return map[string]any{ + "application": MatrixRTCApplicationCall, + "notify_type": "ring", + "call_id": callID, + "m.call.intent": intent, + "m.mentions": map[string]any{}, + } +} + +func EmptyMatrixRTCContent(stickyKey string) map[string]any { + if stickyKey == "" { + return map[string]any{} + } + return map[string]any{ + "sticky_key": stickyKey, + "msc4354_sticky_key": stickyKey, + } +} + +func liveKitTransport(focus Focus) map[string]any { + transport := map[string]any{ + "type": "livekit", + } + if focus.LiveKitServiceURL != "" { + transport["livekit_service_url"] = focus.LiveKitServiceURL + } + return transport +} diff --git a/pkg/connector/voip/matrixrtc_test.go b/pkg/connector/voip/matrixrtc_test.go new file mode 100644 index 0000000..42e22c7 --- /dev/null +++ b/pkg/connector/voip/matrixrtc_test.go @@ -0,0 +1,189 @@ +package voip + +import ( + "testing" + "time" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func TestSupportedMatrixRTCEventTypesHaveExplicitClasses(t *testing.T) { + types := SupportedMatrixRTCEventTypes() + if len(types) != 7 { + t.Fatalf("SupportedMatrixRTCEventTypes returned %d types, want 7", len(types)) + } + for _, evtType := range types { + switch evtType.Type { + case EventTypeGroupCall, EventTypeGroupCallMember: + if evtType.Class != event.StateEventType { + t.Fatalf("%s class = %v, want state", evtType.Type, evtType.Class) + } + case EventTypeRTCMembership: + if evtType.Class != event.StateEventType && evtType.Class != event.MessageEventType { + t.Fatalf("%s class = %v, want state or message", evtType.Type, evtType.Class) + } + case EventTypeRTCNotification, EventTypeCallNotify, EventTypeRTCDecline: + if evtType.Class != event.MessageEventType { + t.Fatalf("%s class = %v, want message", evtType.Type, evtType.Class) + } + default: + t.Fatalf("unexpected MatrixRTC event type %s", evtType.Type) + } + } +} + +func TestParseMatrixRTCDeclineEvent(t *testing.T) { + evt := &event.Event{ + Type: event.Type{Type: EventTypeRTCDecline, Class: event.MessageEventType}, + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + Content: event.Content{Raw: map[string]any{ + "call_id": "call-1", + "device_id": "DEVICE", + "session_id": "SESSION", + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize decline event") + } + if parsed.Kind != MatrixRTCEventKindRTCDecline { + t.Fatalf("kind = %q, want %q", parsed.Kind, MatrixRTCEventKindRTCDecline) + } + if parsed.CallID != "call-1" || parsed.DeviceID != "DEVICE" || parsed.SessionID != "SESSION" { + t.Fatalf("unexpected parsed event: %+v", parsed) + } +} + +func TestParseMatrixRTCMembershipEvent(t *testing.T) { + stateKey := "@alice:example.com" + evt := &event.Event{ + Type: event.Type{Type: EventTypeRTCMembership, Class: event.StateEventType}, + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{ + "memberships": []any{map[string]any{ + "call_id": "call-2", + "device_id": "DEVICE", + "session_id": "SESSION", + "lifetime_ms": float64(60000), + "foci_preferred": []any{map[string]any{ + "type": "livekit", + "livekit_service_url": "https://rtc.example.com/jwt", + }}, + }}, + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership event") + } + if parsed.Kind != MatrixRTCEventKindRTCMembership || parsed.StateKey != stateKey { + t.Fatalf("unexpected parsed event metadata: %+v", parsed) + } + if parsed.CallID != "call-2" || parsed.DeviceID != "DEVICE" || parsed.SessionID != "SESSION" { + t.Fatalf("unexpected parsed event identifiers: %+v", parsed) + } + if parsed.LifetimeMS != 60000 { + t.Fatalf("LifetimeMS = %d, want 60000", parsed.LifetimeMS) + } + if len(parsed.FociPreferred) != 1 || parsed.FociPreferred[0].LiveKitServiceURL != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected foci: %+v", parsed.FociPreferred) + } +} + +func TestBuildRTCMembershipContent(t *testing.T) { + session := MatrixRTCSession{ + UserID: "@wa_123:example.com", + DeviceID: "WADEVICE", + Focus: Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + } + content := BuildRTCMembershipContent(session) + if content["slot_id"] != MatrixRTCDefaultSlotID { + t.Fatalf("slot_id = %q, want %q", content["slot_id"], MatrixRTCDefaultSlotID) + } + member := content["member"].(map[string]any) + if member["user_id"] != "@wa_123:example.com" || member["device_id"] != "WADEVICE" { + t.Fatalf("unexpected member: %+v", member) + } + transports := content["rtc_transports"].([]map[string]any) + if len(transports) != 1 || transports[0]["livekit_service_url"] != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected transports: %+v", transports) + } +} + +func TestParseBuiltRTCMembershipContent(t *testing.T) { + content := BuildRTCMembershipContent(MatrixRTCSession{ + UserID: "@wa_123:example.com", + DeviceID: "WADEVICE", + Intent: "audio", + Focus: Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + }) + evt := &event.Event{ + Type: RTCMembershipEventType(event.MessageEventType), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@wa_123:example.com"), + Content: event.Content{Raw: content}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership event") + } + if parsed.Intent != "audio" { + t.Fatalf("Intent = %q, want audio", parsed.Intent) + } + if len(parsed.FociPreferred) != 1 || parsed.FociPreferred[0].LiveKitServiceURL != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected foci: %+v", parsed.FociPreferred) + } + if !MatrixRTCEventHasJoinContent(parsed) { + t.Fatalf("MatrixRTCEventHasJoinContent returned false for a built membership") + } +} + +func TestMatrixRTCEventHasJoinContentRejectsStickyCleanup(t *testing.T) { + evt := MatrixRTCEvent{ + Kind: MatrixRTCEventKindRTCMembership, + Raw: EmptyMatrixRTCContent("sticky"), + } + if MatrixRTCEventHasJoinContent(evt) { + t.Fatalf("MatrixRTCEventHasJoinContent returned true for sticky cleanup content") + } +} + +func TestMatrixRTCEventHasJoinContentRejectsMetadataWithoutFocus(t *testing.T) { + for _, raw := range []map[string]any{ + { + "application": map[string]any{"type": MatrixRTCApplicationCall}, + }, + { + "slot_id": MatrixRTCDefaultSlotID, + "application": map[string]any{"type": MatrixRTCApplicationCall}, + "member": map[string]any{"user_id": "@alice:example.com", "device_id": "DEVICE"}, + }, + } { + evt := MatrixRTCEvent{Kind: MatrixRTCEventKindRTCMembership, Raw: raw} + if MatrixRTCEventHasJoinContent(evt) { + t.Fatalf("MatrixRTCEventHasJoinContent returned true for metadata-only content: %+v", raw) + } + } +} + +func TestBuildRTCNotificationContentCapsLifetime(t *testing.T) { + content := BuildRTCNotificationContent(testTime, 5*time.Minute, "audio") + if content["notification_type"] != "ring" { + t.Fatalf("notification_type = %q, want ring", content["notification_type"]) + } + if content["lifetime"] != int64(90000) { + t.Fatalf("lifetime = %v, want 90000", content["lifetime"]) + } +} + +var testTime = time.Unix(123, 0) diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go new file mode 100644 index 0000000..95e0c27 --- /dev/null +++ b/pkg/connector/voip/video.go @@ -0,0 +1,57 @@ +package voip + +import ( + "errors" + "sync" + "time" + + lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/purpshell/meowcaller" +) + +var ErrVideoSinkClosed = errors.New("voip: video sink closed") + +type LiveKitH264Writer struct { + mu sync.RWMutex + track interface { + WriteSample(media.Sample, *lksdk.SampleWriteOptions) error + } + duration time.Duration + closed bool +} + +func NewLiveKitH264Writer(track interface { + WriteSample(media.Sample, *lksdk.SampleWriteOptions) error +}, duration time.Duration) *LiveKitH264Writer { + if duration <= 0 { + duration = time.Second / 30 + } + return &LiveKitH264Writer{track: track, duration: duration} +} + +func (w *LiveKitH264Writer) WriteVideo(accessUnit []byte) error { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return ErrVideoSinkClosed + } + if w.track == nil || len(accessUnit) == 0 { + return nil + } + sample := media.Sample{ + Data: append([]byte(nil), accessUnit...), + Duration: w.duration, + } + return w.track.WriteSample(sample, nil) +} + +func (w *LiveKitH264Writer) Close() error { + w.mu.Lock() + w.closed = true + w.track = nil + w.mu.Unlock() + return nil +} + +var _ meowcaller.VideoSink = (*LiveKitH264Writer)(nil) diff --git a/pkg/connector/voip/video_buffer.go b/pkg/connector/voip/video_buffer.go new file mode 100644 index 0000000..68d6773 --- /dev/null +++ b/pkg/connector/voip/video_buffer.go @@ -0,0 +1,56 @@ +package voip + +import "time" + +const maxPendingWhatsAppVideoFrames = 64 + +type LiveKitVideoFrame struct { + AccessUnit []byte + Duration time.Duration +} + +type whatsAppVideoStartupBuffer struct { + ready bool + frames []LiveKitVideoFrame +} + +func (b *whatsAppVideoStartupBuffer) Len() int { + if b == nil { + return 0 + } + return len(b.frames) +} + +func (b *whatsAppVideoStartupBuffer) Send(frame LiveKitVideoFrame, send func(LiveKitVideoFrame) error) (int, error) { + if b == nil || len(frame.AccessUnit) == 0 || send == nil { + return 0, nil + } + if b.ready { + return 0, send(frame) + } + b.enqueue(frame) + flushed := 0 + for len(b.frames) > 0 { + if err := send(b.frames[0]); err != nil { + return flushed, err + } + b.frames[0] = LiveKitVideoFrame{} + b.frames = b.frames[1:] + flushed++ + } + b.ready = true + return flushed, nil +} + +func (b *whatsAppVideoStartupBuffer) enqueue(frame LiveKitVideoFrame) { + queued := LiveKitVideoFrame{ + AccessUnit: append([]byte(nil), frame.AccessUnit...), + Duration: frame.Duration, + } + b.frames = append(b.frames, queued) + if len(b.frames) <= maxPendingWhatsAppVideoFrames { + return + } + copy(b.frames, b.frames[len(b.frames)-maxPendingWhatsAppVideoFrames:]) + b.frames = b.frames[:maxPendingWhatsAppVideoFrames] +} diff --git a/pkg/connector/voip_config.go b/pkg/connector/voip_config.go new file mode 100644 index 0000000..ce27737 --- /dev/null +++ b/pkg/connector/voip_config.go @@ -0,0 +1,46 @@ +package connector + +import "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + +func makeVOIPConfig(cfg VOIPConfig) voip.Config { + return voip.Config{ + Enabled: cfg.Enabled, + IncomingPolicy: cfg.IncomingPolicy, + MaxActiveCallsPerLogin: cfg.MaxActiveCallsPerLogin, + MatrixRTC: voip.MatrixRTCConfig{ + LiveKitServiceURL: cfg.MatrixRTC.LiveKitServiceURL, + RequireLiveKitFocus: cfg.MatrixRTC.RequireLiveKitFocus, + MembershipEventCompat: cfg.MatrixRTC.MembershipEventCompat, + NotificationEventCompat: cfg.MatrixRTC.NotificationEventCompat, + UseDelayedEvents: cfg.MatrixRTC.UseDelayedEvents, + ParticipantMode: cfg.MatrixRTC.ParticipantMode, + FallbackParticipantMXID: cfg.MatrixRTC.FallbackParticipantMXID, + }, + LiveKit: voip.LiveKitConfig{ + ConnectTimeout: cfg.LiveKit.ConnectTimeout, + PublishSilenceBeforeWhatsAppAnswer: cfg.LiveKit.PublishSilenceBeforeWhatsAppAnswer, + AutoSubscribe: cfg.LiveKit.AutoSubscribe, + AudioUplinkPolicy: cfg.LiveKit.AudioUplinkPolicy, + SelectedParticipantTimeout: cfg.LiveKit.SelectedParticipantTimeout, + }, + Audio: voip.AudioConfig{ + Enabled: cfg.Audio.Enabled, + JitterBuffer: cfg.Audio.JitterBuffer, + OpusBackend: cfg.Audio.OpusBackend, + SilenceOnUnderrun: cfg.Audio.SilenceOnUnderrun, + MaxMixParticipants: cfg.Audio.MaxMixParticipants, + }, + Video: voip.VideoConfig{ + Enabled: cfg.Video.Enabled, + SelectedSourcePolicy: cfg.Video.SelectedSourcePolicy, + MaxWidth: cfg.Video.MaxWidth, + MaxHeight: cfg.Video.MaxHeight, + MaxFPS: cfg.Video.MaxFPS, + }, + Diagnostics: voip.DiagnosticsConfig{ + HealthcheckFocusOnStartup: cfg.Diagnostics.HealthcheckFocusOnStartup, + EnableMeowcallerDiagnostics: cfg.Diagnostics.EnableMeowcallerDiagnostics, + MediaTraceDir: cfg.Diagnostics.MediaTraceDir, + }, + } +} From 5341e0b0d2c2de997b715a17d33c1b5dbcad2235 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 16 Jul 2026 23:21:05 +0300 Subject: [PATCH 04/44] go.mod: refresh meowcaller module metadata --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 795e050..5276d6d 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 github.com/livekit/protocol v1.49.0 github.com/livekit/server-sdk-go/v2 v2.17.0 + github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 github.com/rs/zerolog v1.35.1 @@ -18,6 +19,7 @@ require ( go.mau.fi/util v0.9.10 go.mau.fi/webp v0.3.0 go.mau.fi/whatsmeow v0.0.0-20260709092057-73fe7355f59f + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/image v0.42.0 golang.org/x/net v0.56.0 golang.org/x/sync v0.21.0 @@ -79,7 +81,6 @@ require ( github.com/pion/opus v0.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.16 // indirect - github.com/pion/rtp v1.10.2 // indirect github.com/pion/sctp v1.10.0 // indirect github.com/pion/sdp/v3 v3.0.18 // indirect github.com/pion/srtp/v3 v3.0.11 // indirect @@ -109,7 +110,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect golang.org/x/mod v0.37.0 // indirect diff --git a/go.sum b/go.sum index 7ddfb5f..e4f2f82 100644 --- a/go.sum +++ b/go.sum @@ -196,10 +196,10 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 h1:ArV0YXyDljr69xRiMrm+F0V6VvZzo/7YEsG06hNAt0c= +github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= -github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 h1:D1asWnhVkAepHAEKSXQYJDG2nK5NHxdo1sRD72u6gSU= -github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52/go.mod h1:/YNSNaB2/qq6ZlFkOtPJV/z5nEWjRC+uBXC+3/LHCGU= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= From 929eb6cefd3e93cb609e1bbb69bdb7515550cc7d Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 16 Jul 2026 23:58:58 +0300 Subject: [PATCH 05/44] go.mod: update meowcaller video transport --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5276d6d..4b15f32 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 + github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index e4f2f82..dbf05c5 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52 h1:ArV0YXyDljr69xRiMrm+F0V6VvZzo/7YEsG06hNAt0c= -github.com/purpshell/meowcaller v0.0.0-20260716175428-b00465cb0e52/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11 h1:UKY1q3U3t4e2izEF9EB5GsdD2nMdmJcvWAgUCGDp4Xs= +github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From a492582892f3f2a80d07bec9bdd0b7066a1d1cf6 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 00:08:32 +0300 Subject: [PATCH 06/44] go.mod: update meowcaller video capability --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4b15f32..b26b155 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11 + github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index dbf05c5..ea10e92 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11 h1:UKY1q3U3t4e2izEF9EB5GsdD2nMdmJcvWAgUCGDp4Xs= -github.com/purpshell/meowcaller v0.0.0-20260716205539-0631faaf0d11/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 h1:ly/0+RFVo1Y1Xt8aNpXloOxHqVf3HPFANTCP5UiYK98= +github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 738e24e41fd23a58b2a53392bb1855e3eefb2aaf Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 00:25:27 +0300 Subject: [PATCH 07/44] video.go: propagate device orientation --- pkg/connector/voip/livekit.go | 14 ++++++++++++++ pkg/connector/voip/manager.go | 1 + pkg/connector/voip/video.go | 22 ++++++++++++++++++++++ pkg/connector/voip/video_test.go | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 pkg/connector/voip/video_test.go diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index 198bbac..acc7823 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -190,6 +190,20 @@ func (p *LiveKitParticipant) SetWhatsAppVideoMuted(muted bool) { p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp video mute state") } +func (p *LiveKitParticipant) SetWhatsAppVideoOrientation(orientation int) { + p.mu.Lock() + video := p.video + p.mu.Unlock() + if video == nil { + return + } + if setLiveKitVideoOrientation(video, orientation) { + p.log.Debug(). + Int("orientation", orientation&0x03). + Msg("Set LiveKit WhatsApp video orientation") + } +} + func (p *LiveKitParticipant) WhatsAppSink() *LiveKitPCMWriter { p.mu.Lock() defer p.mu.Unlock() diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 6fe7bc9..c8142c2 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -423,6 +423,7 @@ func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.Video m.mu.Unlock() if participant != nil { participant.SetWhatsAppVideoMuted(muted) + participant.SetWhatsAppVideoOrientation(state.Orientation) } m.log.Debug(). Str("call_id", callID). diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go index 95e0c27..2f44741 100644 --- a/pkg/connector/voip/video.go +++ b/pkg/connector/voip/video.go @@ -21,6 +21,10 @@ type LiveKitH264Writer struct { closed bool } +type liveKitVideoOrientationSetter interface { + SetVideoOrientation(uint8) +} + func NewLiveKitH264Writer(track interface { WriteSample(media.Sample, *lksdk.SampleWriteOptions) error }, duration time.Duration) *LiveKitH264Writer { @@ -46,6 +50,24 @@ func (w *LiveKitH264Writer) WriteVideo(accessUnit []byte) error { return w.track.WriteSample(sample, nil) } +func (w *LiveKitH264Writer) SetOrientation(orientation int) { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return + } + setLiveKitVideoOrientation(w.track, orientation) +} + +func setLiveKitVideoOrientation(track any, orientation int) bool { + setter, ok := track.(liveKitVideoOrientationSetter) + if !ok { + return false + } + setter.SetVideoOrientation(uint8(orientation) & 0x03) + return true +} + func (w *LiveKitH264Writer) Close() error { w.mu.Lock() w.closed = true diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go new file mode 100644 index 0000000..dc172d2 --- /dev/null +++ b/pkg/connector/voip/video_test.go @@ -0,0 +1,32 @@ +package voip + +import ( + "testing" + "time" + + lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/webrtc/v4/pkg/media" +) + +type orientedSampleTrack struct { + orientation uint8 +} + +func (t *orientedSampleTrack) WriteSample(media.Sample, *lksdk.SampleWriteOptions) error { + return nil +} + +func (t *orientedSampleTrack) SetVideoOrientation(orientation uint8) { + t.orientation = orientation +} + +func TestLiveKitH264WriterSetsVideoOrientation(t *testing.T) { + track := &orientedSampleTrack{} + writer := NewLiveKitH264Writer(track, time.Second/30) + + writer.SetOrientation(5) + + if track.orientation != 1 { + t.Fatalf("track orientation = %d, want 1", track.orientation) + } +} From 8b83612207ea1e44ef1787adb358536a7436c958 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 00:28:45 +0300 Subject: [PATCH 08/44] go.mod: update meowcaller video negotiation --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b26b155..8f79953 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 + github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index ea10e92..b1854f9 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 h1:ly/0+RFVo1Y1Xt8aNpXloOxHqVf3HPFANTCP5UiYK98= github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 h1:y1B49s8icl4hQTpxoaMdgk3XWSJAXAn0WoK27wK2038= +github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From b3784f075d5ec7c51798ed547acfc226fcc3d458 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 09:17:35 +0300 Subject: [PATCH 09/44] go.mod: restore compatible video offer --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8f79953..5f019ed 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 + github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index b1854f9..7e42c46 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 h1:ly/0+RFVo1 github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 h1:y1B49s8icl4hQTpxoaMdgk3XWSJAXAn0WoK27wK2038= github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb h1:VktUaM/cJS0nXFUYB882xp3icPheAq1YNdIWp5Twq8g= +github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 0c424dcbcf90e0ff70ac140654c515aeb6651907 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 10:10:07 +0300 Subject: [PATCH 10/44] livekit.go: request keyframe after peer acceptance --- go.mod | 2 +- go.sum | 2 + pkg/connector/voip/livekit.go | 56 ++++++++++++++++++++++ pkg/connector/voip/manager.go | 81 +++++++++++++++++++++++--------- pkg/connector/voip/video_test.go | 45 ++++++++++++++++++ 5 files changed, 163 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 5f019ed..68f5f6c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb + github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 7e42c46..5cd63f4 100644 --- a/go.sum +++ b/go.sum @@ -202,6 +202,8 @@ github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 h1:y1B49s8icl github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb h1:VktUaM/cJS0nXFUYB882xp3icPheAq1YNdIWp5Twq8g= github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 h1:m2/FsJoVytQhERCn3WP8T01kOQb9EhQdB60YrH1CFE0= +github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index acc7823..afc5ef5 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -34,6 +34,9 @@ type LiveKitParticipant struct { mu sync.Mutex remoteAudio []*lkmedia.PCMRemoteTrack remoteMediaCancel context.CancelFunc + remoteVideoPLI lksdk.PLIWriter + remoteVideoSSRC webrtc.SSRC + remoteVideoKeyframePending bool disconnected bool selectedRemoteParticipant string remoteAudioMuteStateChange func(muted bool) @@ -111,6 +114,48 @@ func (p *LiveKitParticipant) SetRemoteVideoHandlers(selectedParticipant string, p.mu.Unlock() } +func (p *LiveKitParticipant) requestRemoteVideoKeyframe() bool { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return false + } + pli := p.remoteVideoPLI + ssrc := p.remoteVideoSSRC + if pli == nil || ssrc == 0 { + p.remoteVideoKeyframePending = true + p.mu.Unlock() + return false + } + p.remoteVideoKeyframePending = false + p.mu.Unlock() + p.sendRemoteVideoPLI(pli, ssrc) + return true +} + +func (p *LiveKitParticipant) setRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return + } + p.remoteVideoPLI = pli + p.remoteVideoSSRC = ssrc + pending := p.remoteVideoKeyframePending && pli != nil && ssrc != 0 + if pending { + p.remoteVideoKeyframePending = false + } + p.mu.Unlock() + if pending { + p.sendRemoteVideoPLI(pli, ssrc) + } +} + +func (p *LiveKitParticipant) sendRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + pli(ssrc) + p.log.Info().Uint32("ssrc", uint32(ssrc)).Msg("Requested LiveKit H.264 keyframe for WhatsApp peer") +} + func (p *LiveKitParticipant) PublishAudioTrack(name string) error { p.mu.Lock() defer p.mu.Unlock() @@ -325,12 +370,20 @@ func (p *LiveKitParticipant) onVideoTrackSubscribed(ctx context.Context, track * p.handleRemoteVideoMuteState(publication, rp, true) return } + p.setRemoteVideoPLI(rp.WritePLI, track.SSRC()) p.handleRemoteVideoMuteState(publication, rp, publication.IsMuted()) go p.forwardRemoteH264Track(ctx, track, rp) } func (p *LiveKitParticipant) onTrackUnsubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { if track.Kind() == webrtc.RTPCodecTypeVideo { + p.mu.Lock() + if p.remoteVideoSSRC == track.SSRC() { + p.remoteVideoPLI = nil + p.remoteVideoSSRC = 0 + p.remoteVideoKeyframePending = true + } + p.mu.Unlock() p.handleRemoteVideoMuteState(publication, rp, true) } } @@ -469,6 +522,9 @@ func (p *LiveKitParticipant) closeRemoteTracks() { p.mu.Lock() tracks := p.remoteAudio p.remoteAudio = nil + p.remoteVideoPLI = nil + p.remoteVideoSSRC = 0 + p.remoteVideoKeyframePending = false cancel := p.remoteMediaCancel p.remoteMediaCancel = nil p.mu.Unlock() diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index c8142c2..b7ae9a0 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -34,32 +34,34 @@ type Manager struct { client *meowcaller.Client log zerolog.Logger - mu sync.Mutex - calls map[string]*meowcaller.Call - callCreators map[string]types.JID - livekit map[string]*LiveKitParticipant - livekitConnecting map[string]struct{} - matrixAudioMuted map[string]bool - matrixVideoMuted map[string]bool - whatsAppMuted map[string]bool - whatsAppVideoMuted map[string]bool - incomingCallNotify func(*meowcaller.Call) - callEndNotify func(callID, reason string) + mu sync.Mutex + calls map[string]*meowcaller.Call + callCreators map[string]types.JID + livekit map[string]*LiveKitParticipant + livekitConnecting map[string]struct{} + matrixAudioMuted map[string]bool + matrixVideoMuted map[string]bool + whatsAppMuted map[string]bool + whatsAppVideoMuted map[string]bool + videoKeyframePending map[string]bool + incomingCallNotify func(*meowcaller.Call) + callEndNotify func(callID, reason string) } func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Manager { manager := &Manager{ - cfg: cfg, - waClient: waClient, - log: log, - calls: make(map[string]*meowcaller.Call), - callCreators: make(map[string]types.JID), - livekit: make(map[string]*LiveKitParticipant), - livekitConnecting: make(map[string]struct{}), - matrixAudioMuted: make(map[string]bool), - matrixVideoMuted: make(map[string]bool), - whatsAppMuted: make(map[string]bool), - whatsAppVideoMuted: make(map[string]bool), + cfg: cfg, + waClient: waClient, + log: log, + calls: make(map[string]*meowcaller.Call), + callCreators: make(map[string]types.JID), + livekit: make(map[string]*LiveKitParticipant), + livekitConnecting: make(map[string]struct{}), + matrixAudioMuted: make(map[string]bool), + matrixVideoMuted: make(map[string]bool), + whatsAppMuted: make(map[string]bool), + whatsAppVideoMuted: make(map[string]bool), + videoKeyframePending: make(map[string]bool), } if !cfg.Enabled || waClient == nil { return manager @@ -144,6 +146,7 @@ func (m *Manager) AbortAll() { m.matrixVideoMuted = make(map[string]bool) m.whatsAppMuted = make(map[string]bool) m.whatsAppVideoMuted = make(map[string]bool) + m.videoKeyframePending = make(map[string]bool) participants := make([]*LiveKitParticipant, 0, len(m.livekit)) for _, participant := range m.livekit { participants = append(participants, participant) @@ -250,11 +253,26 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth } } m.mu.Lock() + if m.calls[waCallID] != call || call.State() == meowcaller.CallPhaseEnded { + delete(m.livekitConnecting, waCallID) + delete(m.videoKeyframePending, waCallID) + m.mu.Unlock() + call.Receive(nil) + call.ReceiveVideo(nil) + call.Subscribe(nil) + participant.Close() + return ErrCallNotFound + } whatsAppMuted, knownWhatsAppMute := m.whatsAppMuted[waCallID] whatsAppVideoMuted, knownWhatsAppVideoMute := m.whatsAppVideoMuted[waCallID] + keyframePending := m.videoKeyframePending[waCallID] + delete(m.videoKeyframePending, waCallID) delete(m.livekitConnecting, waCallID) m.livekit[waCallID] = participant m.mu.Unlock() + if videoEnabled && keyframePending { + participant.requestRemoteVideoKeyframe() + } if knownWhatsAppMute { participant.SetWhatsAppAudioMuted(whatsAppMuted) } @@ -603,6 +621,7 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { delete(m.matrixVideoMuted, call.ID()) delete(m.whatsAppMuted, call.ID()) delete(m.whatsAppVideoMuted, call.ID()) + delete(m.videoKeyframePending, call.ID()) participant := m.livekit[call.ID()] delete(m.livekit, call.ID()) delete(m.livekitConnecting, call.ID()) @@ -619,6 +638,11 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { call.OnStateChange(func(phase meowcaller.CallPhase) { m.log.Debug().Str("call_id", call.ID()).Int("phase", int(phase)).Msg("WhatsApp VOIP call state changed") }) + call.OnPeerAccept(func() { + if call.IsVideo() { + m.requestLiveKitVideoKeyframe(call.ID()) + } + }) call.OnMuteState(func(muted bool) { m.handleWhatsAppAudioMuteState(call.ID(), muted) }) @@ -627,6 +651,19 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { }) } +func (m *Manager) requestLiveKitVideoKeyframe(callID string) { + m.mu.Lock() + call := m.calls[callID] + participant := m.livekit[callID] + if call != nil && participant == nil { + m.videoKeyframePending[callID] = true + } + m.mu.Unlock() + if call != nil && participant != nil { + participant.requestRemoteVideoKeyframe() + } +} + func (m *Manager) callCreatorFor(call *meowcaller.Call) types.JID { if m == nil || call == nil { return types.EmptyJID diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go index dc172d2..8bb097f 100644 --- a/pkg/connector/voip/video_test.go +++ b/pkg/connector/voip/video_test.go @@ -5,7 +5,9 @@ import ( "time" lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" + "github.com/purpshell/meowcaller" ) type orientedSampleTrack struct { @@ -30,3 +32,46 @@ func TestLiveKitH264WriterSetsVideoOrientation(t *testing.T) { t.Fatalf("track orientation = %d, want 1", track.orientation) } } + +func TestLiveKitParticipantRequestsRemoteVideoKeyframe(t *testing.T) { + const wantSSRC = webrtc.SSRC(0x12345678) + var gotSSRC webrtc.SSRC + participant := &LiveKitParticipant{} + + if participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned true before track subscription") + } + participant.setRemoteVideoPLI(func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, wantSSRC) + if gotSSRC != wantSSRC { + t.Fatalf("deferred PLI SSRC = %#x, want %#x", gotSSRC, wantSSRC) + } + + gotSSRC = 0 + if !participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned false with a subscribed track") + } + if gotSSRC != wantSSRC { + t.Fatalf("immediate PLI SSRC = %#x, want %#x", gotSSRC, wantSSRC) + } +} + +func TestManagerDefersVideoKeyframeOnlyForTrackedCall(t *testing.T) { + manager := &Manager{ + calls: make(map[string]*meowcaller.Call), + livekit: make(map[string]*LiveKitParticipant), + videoKeyframePending: make(map[string]bool), + } + + manager.requestLiveKitVideoKeyframe("ended") + if manager.videoKeyframePending["ended"] { + t.Fatal("keyframe request was retained for an untracked call") + } + + manager.calls["active"] = &meowcaller.Call{} + manager.requestLiveKitVideoKeyframe("active") + if !manager.videoKeyframePending["active"] { + t.Fatal("keyframe request was not retained for a tracked call") + } +} From 65571f5d6cadc34c62f7a69a4a9900b4f4f32558 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 10:47:52 +0300 Subject: [PATCH 11/44] (video.go): repeat H264 parameter sets --- pkg/connector/voip/livekit.go | 34 ++++++++++++- pkg/connector/voip/video.go | 83 ++++++++++++++++++++++++++++++++ pkg/connector/voip/video_test.go | 54 +++++++++++++++++++++ 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index afc5ef5..b4f1c4a 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -37,6 +37,7 @@ type LiveKitParticipant struct { remoteVideoPLI lksdk.PLIWriter remoteVideoSSRC webrtc.SSRC remoteVideoKeyframePending bool + remoteVideoKeyframeAwaited bool disconnected bool selectedRemoteParticipant string remoteAudioMuteStateChange func(muted bool) @@ -152,6 +153,11 @@ func (p *LiveKitParticipant) setRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc. } func (p *LiveKitParticipant) sendRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.mu.Lock() + if !p.disconnected { + p.remoteVideoKeyframeAwaited = true + } + p.mu.Unlock() pli(ssrc) p.log.Info().Uint32("ssrc", uint32(ssrc)).Msg("Requested LiveKit H.264 keyframe for WhatsApp peer") } @@ -464,9 +470,12 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * &codecs.H264Packet{}, track.Codec().ClockRate, ) + var parameterSets h264ParameterSetRepeater + loggedIDR := false p.log.Info(). Str("participant", string(rp.Identity())). Str("track_id", track.ID()). + Str("fmtp", track.Codec().SDPFmtpLine). Msg("Started forwarding LiveKit H.264 video to WhatsApp") for { if ctx.Err() != nil { @@ -488,6 +497,28 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * if len(sample.Data) == 0 { continue } + accessUnit, repeatedParameterSets := parameterSets.Normalize(sample.Data) + nalTypes, profileLevelID, hasIDR, hasSPS, hasPPS := h264AccessUnitMetadata(accessUnit) + p.mu.Lock() + afterPLI := hasIDR && p.remoteVideoKeyframeAwaited + if afterPLI { + p.remoteVideoKeyframeAwaited = false + } + p.mu.Unlock() + if hasIDR && (!loggedIDR || afterPLI || repeatedParameterSets) { + p.log.Info(). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Ints("nal_types", nalTypes). + Str("profile_level_id", profileLevelID). + Int("bytes", len(accessUnit)). + Bool("has_sps", hasSPS). + Bool("has_pps", hasPPS). + Bool("after_pli", afterPLI). + Bool("repeated_parameter_sets", repeatedParameterSets). + Msg("Forwarding decoder-safe LiveKit H.264 keyframe to WhatsApp") + loggedIDR = true + } p.mu.Lock() handler := p.remoteVideoFrame p.mu.Unlock() @@ -495,7 +526,7 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * continue } if err = handler(LiveKitVideoFrame{ - AccessUnit: sample.Data, + AccessUnit: accessUnit, Duration: sample.Duration, }); err != nil { p.log.Warn(). @@ -525,6 +556,7 @@ func (p *LiveKitParticipant) closeRemoteTracks() { p.remoteVideoPLI = nil p.remoteVideoSSRC = 0 p.remoteVideoKeyframePending = false + p.remoteVideoKeyframeAwaited = false cancel := p.remoteMediaCancel p.remoteMediaCancel = nil p.mu.Unlock() diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go index 2f44741..d97264b 100644 --- a/pkg/connector/voip/video.go +++ b/pkg/connector/voip/video.go @@ -2,12 +2,14 @@ package voip import ( "errors" + "fmt" "sync" "time" lksdk "github.com/livekit/server-sdk-go/v2" "github.com/pion/webrtc/v4/pkg/media" "github.com/purpshell/meowcaller" + wartp "github.com/purpshell/meowcaller/rtp" ) var ErrVideoSinkClosed = errors.New("voip: video sink closed") @@ -25,6 +27,87 @@ type liveKitVideoOrientationSetter interface { SetVideoOrientation(uint8) } +type h264ParameterSetRepeater struct { + sps []byte + pps []byte +} + +func (r *h264ParameterSetRepeater) Normalize(accessUnit []byte) ([]byte, bool) { + nalus := wartp.SplitAnnexB(accessUnit) + if len(nalus) == 0 { + return accessUnit, false + } + var currentSPS, currentPPS []byte + hasIDR := false + for _, nalu := range nalus { + if len(nalu) == 0 { + continue + } + switch nalu[0] & 0x1f { + case 5: + hasIDR = true + case 7: + currentSPS = nalu + r.sps = append(r.sps[:0], nalu...) + case 8: + currentPPS = nalu + r.pps = append(r.pps[:0], nalu...) + } + } + if !hasIDR || (currentSPS != nil && currentPPS != nil) { + return accessUnit, false + } + sps := currentSPS + if sps == nil { + sps = r.sps + } + pps := currentPPS + if pps == nil { + pps = r.pps + } + if len(sps) == 0 || len(pps) == 0 { + return accessUnit, false + } + + normalized := make([]byte, 0, len(accessUnit)+len(sps)+len(pps)+8) + normalized = appendAnnexBNAL(normalized, sps) + normalized = appendAnnexBNAL(normalized, pps) + for _, nalu := range nalus { + if len(nalu) == 0 || nalu[0]&0x1f == 7 || nalu[0]&0x1f == 8 { + continue + } + normalized = appendAnnexBNAL(normalized, nalu) + } + return normalized, true +} + +func appendAnnexBNAL(dst, nalu []byte) []byte { + dst = append(dst, 0, 0, 0, 1) + return append(dst, nalu...) +} + +func h264AccessUnitMetadata(accessUnit []byte) (nalTypes []int, profileLevelID string, hasIDR, hasSPS, hasPPS bool) { + for _, nalu := range wartp.SplitAnnexB(accessUnit) { + if len(nalu) == 0 { + continue + } + nalType := int(nalu[0] & 0x1f) + nalTypes = append(nalTypes, nalType) + switch nalType { + case 5: + hasIDR = true + case 7: + hasSPS = true + if len(nalu) >= 4 { + profileLevelID = fmt.Sprintf("%02x%02x%02x", nalu[1], nalu[2], nalu[3]) + } + case 8: + hasPPS = true + } + } + return +} + func NewLiveKitH264Writer(track interface { WriteSample(media.Sample, *lksdk.SampleWriteOptions) error }, duration time.Duration) *LiveKitH264Writer { diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go index 8bb097f..504716d 100644 --- a/pkg/connector/voip/video_test.go +++ b/pkg/connector/voip/video_test.go @@ -1,6 +1,7 @@ package voip import ( + "bytes" "testing" "time" @@ -10,6 +11,59 @@ import ( "github.com/purpshell/meowcaller" ) +func annexBNAL(nalu ...byte) []byte { + return append([]byte{0, 0, 0, 1}, nalu...) +} + +func TestH264ParameterSetRepeaterAddsCachedHeadersToIDR(t *testing.T) { + repeater := h264ParameterSetRepeater{} + sps := annexBNAL(0x67, 0x42, 0xe0, 0x1f) + pps := annexBNAL(0x68, 0xce, 0x06, 0xe2) + repeater.Normalize(append(append([]byte{}, sps...), pps...)) + + idr := annexBNAL(0x65, 0x88, 0x84) + got, repeated := repeater.Normalize(idr) + want := append(append(append([]byte{}, sps...), pps...), idr...) + if !repeated { + t.Fatal("Normalize did not report repeated parameter sets") + } + if !bytes.Equal(got, want) { + t.Fatalf("normalized IDR = %x, want %x", got, want) + } +} + +func TestH264ParameterSetRepeaterPreservesCompleteIDR(t *testing.T) { + repeater := h264ParameterSetRepeater{} + au := append(append(annexBNAL(0x67, 0x42, 0xe0, 0x1f), annexBNAL(0x68, 0xce, 0x06, 0xe2)...), annexBNAL(0x65, 0x88, 0x84)...) + + got, repeated := repeater.Normalize(au) + if repeated { + t.Fatal("Normalize reported repeating already-present parameter sets") + } + if !bytes.Equal(got, au) { + t.Fatalf("complete IDR changed: got %x, want %x", got, au) + } +} + +func TestH264ParameterSetRepeaterUsesCurrentAndCachedHeadersInDecodeOrder(t *testing.T) { + repeater := h264ParameterSetRepeater{} + oldSPS := annexBNAL(0x67, 0x42, 0xe0, 0x1f) + pps := annexBNAL(0x68, 0xce, 0x06, 0xe2) + repeater.Normalize(append(append([]byte{}, oldSPS...), pps...)) + + newSPS := annexBNAL(0x67, 0x42, 0xe0, 0x20) + idr := annexBNAL(0x65, 0x99) + au := append(append([]byte{}, newSPS...), idr...) + got, repeated := repeater.Normalize(au) + want := append(append(append([]byte{}, newSPS...), pps...), idr...) + if !repeated { + t.Fatal("Normalize did not report filling the missing PPS") + } + if !bytes.Equal(got, want) { + t.Fatalf("normalized partial IDR = %x, want %x", got, want) + } +} + type orientedSampleTrack struct { orientation uint8 } From 07e19a826d7de11f9a6a06fb65aa58f49c953c14 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:33:11 +0300 Subject: [PATCH 12/44] go.mod: update meowcaller video lifecycle --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 68f5f6c..fcecc36 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 + github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 5cd63f4..dd1f794 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb h1:VktUaM/cJS github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 h1:m2/FsJoVytQhERCn3WP8T01kOQb9EhQdB60YrH1CFE0= github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 h1:d5Qu51VoLMgRj7qlRUBOjTsUQSYyyUPSQj7To/Qig3c= +github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From e510baa1f4c47076ab6284dab7209cecf122a38e Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:33:11 +0300 Subject: [PATCH 13/44] manager.go: forward video keyframe requests --- pkg/connector/voip/manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index b7ae9a0..44a19fa 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -643,6 +643,9 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { m.requestLiveKitVideoKeyframe(call.ID()) } }) + call.OnVideoKeyframeRequest(func() { + m.requestLiveKitVideoKeyframe(call.ID()) + }) call.OnMuteState(func(muted bool) { m.handleWhatsAppAudioMuteState(call.ID(), muted) }) From 116012b27d356c365f1d19ed8b1339ffc18cebca Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:36:37 +0300 Subject: [PATCH 14/44] go.mod: update meowcaller revision --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fcecc36..f4374d0 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 + github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index dd1f794..bd73043 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 h1:m2/FsJoVyt github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 h1:d5Qu51VoLMgRj7qlRUBOjTsUQSYyyUPSQj7To/Qig3c= github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a h1:/RrPY0TupZQSZ6MTz41bbpxUnyApLJUzfT/StrL2Yjs= +github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 7466d5e7ff876b10524e57fcf6beb771bc2bf340 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:48:07 +0300 Subject: [PATCH 15/44] go.mod: update meowcaller directional video --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f4374d0..499d8b4 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a + github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index bd73043..8f03e3c 100644 --- a/go.sum +++ b/go.sum @@ -208,6 +208,8 @@ github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 h1:d5Qu51VoLM github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a h1:/RrPY0TupZQSZ6MTz41bbpxUnyApLJUzfT/StrL2Yjs= github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8 h1:4mJDTzyk2CiX1G9/Wup2jMpZxDXdsWjbiDmuQcwc+xI= +github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 4b3aaac41c69c30f80d3b7b9f302f5ce011729d3 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:48:07 +0300 Subject: [PATCH 16/44] manager.go: separate video flow state --- pkg/connector/voip/manager.go | 144 ++++++++++++----------------- pkg/connector/voip/manager_test.go | 44 ++++++++- 2 files changed, 98 insertions(+), 90 deletions(-) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 44a19fa..032e197 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -16,9 +16,17 @@ import ( ) const ( - localUnmuteState = "0" - localMuteState = "1" - localVideoInactiveState = 0 + localUnmuteState = "0" + localMuteState = "1" +) + +type matrixVideoAction uint8 + +const ( + matrixVideoNone matrixVideoAction = iota + matrixVideoDisable + matrixVideoEnable + matrixVideoUpgrade ) var localMuteRetryIntervals = []time.Duration{ @@ -190,7 +198,7 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth participant.SetRemoteAudioMuteHandler(selectedRemoteParticipantID, func(muted bool) { m.handleMatrixAudioMuteState(call, muted) }) - videoEnabled := m.cfg.Video.Enabled && call.IsVideo() + videoEnabled := m.cfg.Video.Enabled if videoEnabled { var videoBuffer whatsAppVideoStartupBuffer var videoBufferLock sync.Mutex @@ -276,16 +284,16 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth if knownWhatsAppMute { participant.SetWhatsAppAudioMuted(whatsAppMuted) } - if knownWhatsAppVideoMute { + if videoEnabled { + if !knownWhatsAppVideoMute { + whatsAppVideoMuted = !call.IsReceivingVideo() + } participant.SetWhatsAppVideoMuted(whatsAppVideoMuted) } if answeredIncoming { m.log.Debug().Str("call_id", waCallID).Msg("Answered incoming WhatsApp call before sending local unmute") } go m.sendLocalMuteStateRetries(call) - if videoEnabled { - go m.sendLocalVideoStateRetries(call) - } return nil } @@ -321,38 +329,6 @@ func (m *Manager) sendLocalMuteStateRetries(call *meowcaller.Call) { } } -func (m *Manager) sendLocalVideoStateRetries(call *meowcaller.Call) { - if m == nil || m.waClient == nil || call == nil || !call.IsVideo() { - return - } - for attempt, interval := range localMuteRetryIntervals { - time.Sleep(interval) - if call.State() == meowcaller.CallPhaseEnded { - return - } - muted := m.currentMatrixVideoMuted(call.ID()) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err := m.sendLocalVideoState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localVideoStateFor(muted)) - cancel() - if err != nil { - m.log.Warn(). - Err(err). - Str("call_id", call.ID()). - Stringer("peer_jid", call.Peer()). - Bool("muted", muted). - Int("attempt", attempt+1). - Msg("Failed to send WhatsApp local video state") - continue - } - m.log.Debug(). - Str("call_id", call.ID()). - Stringer("peer_jid", call.Peer()). - Bool("muted", muted). - Int("attempt", attempt+1). - Msg("Sent WhatsApp local video state") - } -} - func (m *Manager) handleMatrixAudioMuteState(call *meowcaller.Call, muted bool) { if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { return @@ -384,7 +360,7 @@ func (m *Manager) handleMatrixAudioMuteState(call *meowcaller.Call, muted bool) } func (m *Manager) handleMatrixVideoMuteState(call *meowcaller.Call, muted bool) { - if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded || !call.IsVideo() { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { return } m.mu.Lock() @@ -394,9 +370,7 @@ func (m *Manager) handleMatrixVideoMuteState(call *meowcaller.Call, muted bool) if known && previous == muted { return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err := m.sendLocalVideoState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localVideoStateFor(muted)) - cancel() + err := applyMatrixVideoState(call, muted) if err != nil { m.log.Warn(). Err(err). @@ -434,18 +408,23 @@ func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.Video if m == nil || callID == "" { return } - muted := !state.Active && !state.Upgrade + muted, muteChanged := remoteVideoMuteForState(state) m.mu.Lock() - m.whatsAppVideoMuted[callID] = muted + if muteChanged { + m.whatsAppVideoMuted[callID] = muted + } participant := m.livekit[callID] m.mu.Unlock() if participant != nil { - participant.SetWhatsAppVideoMuted(muted) + if muteChanged { + participant.SetWhatsAppVideoMuted(muted) + } participant.SetWhatsAppVideoOrientation(state.Orientation) } m.log.Debug(). Str("call_id", callID). Bool("muted", muted). + Bool("mute_changed", muteChanged). Bool("active", state.Active). Bool("upgrade", state.Upgrade). Int("orientation", state.Orientation). @@ -453,6 +432,17 @@ func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.Video Msg("Observed WhatsApp remote video state") } +func remoteVideoMuteForState(state meowcaller.VideoState) (muted, changed bool) { + switch state.Raw { + case signaling.VideoStateEnabled: + return false, true + case signaling.VideoStateDisabled, signaling.VideoStateStopped: + return true, true + default: + return false, false + } +} + func (m *Manager) currentMatrixAudioMuted(callID string) bool { if m == nil { return false @@ -463,16 +453,6 @@ func (m *Manager) currentMatrixAudioMuted(callID string) bool { return muted } -func (m *Manager) currentMatrixVideoMuted(callID string) bool { - if m == nil { - return false - } - m.mu.Lock() - muted := m.matrixVideoMuted[callID] - m.mu.Unlock() - return muted -} - func localMuteStateFor(muted bool) string { if muted { return localMuteState @@ -480,11 +460,30 @@ func localMuteStateFor(muted bool) string { return localUnmuteState } -func localVideoStateFor(muted bool) int { +func matrixVideoActionFor(muted, sending, receiving bool) matrixVideoAction { if muted { - return localVideoInactiveState + if sending { + return matrixVideoDisable + } + return matrixVideoNone + } + if sending || receiving { + return matrixVideoEnable + } + return matrixVideoUpgrade +} + +func applyMatrixVideoState(call *meowcaller.Call, muted bool) error { + switch matrixVideoActionFor(muted, call.IsSendingVideo(), call.IsReceivingVideo()) { + case matrixVideoDisable: + return call.SetVideoEnabled(false) + case matrixVideoEnable: + return call.SetVideoEnabled(true) + case matrixVideoUpgrade: + return call.StartVideo() + default: + return nil } - return signaling.VideoStateActive } func (m *Manager) sendLocalMuteState(ctx context.Context, callID string, peer, callCreator types.JID, muteState string) error { @@ -508,31 +507,6 @@ func (m *Manager) sendLocalMuteState(ctx context.Context, callID string, peer, c return nil } -func (m *Manager) sendLocalVideoState(ctx context.Context, callID string, peer, callCreator types.JID, videoState int) error { - if m == nil || m.waClient == nil { - return fmt.Errorf("whatsapp client is not available") - } - if callID == "" { - return fmt.Errorf("call ID is empty") - } - if peer.IsEmpty() { - return fmt.Errorf("peer JID is empty") - } - if callCreator.IsEmpty() { - return fmt.Errorf("call creator JID is empty") - } - codec := "" - if videoState == signaling.VideoStateActive { - codec = signaling.VideoStateDecH264 - } - node := signaling.BuildVideoState(callID, peer, callCreator, string(m.waClient.GenerateMessageID()), videoState, 0, codec) - //lint:ignore SA1019 low-level call signaling is not exposed by whatsmeow's public API - if err := m.waClient.DangerousInternals().SendNode(ctx, node); err != nil { - return fmt.Errorf("send video state: %w", err) - } - return nil -} - func buildLocalMuteV2Node(callID string, peer, callCreator types.JID, wrapperID, muteState string) waBinary.Node { node := signaling.BuildMuteV2(callID, peer, callCreator, muteState) if wrapperID != "" { diff --git a/pkg/connector/voip/manager_test.go b/pkg/connector/voip/manager_test.go index 0f39de0..53b405b 100644 --- a/pkg/connector/voip/manager_test.go +++ b/pkg/connector/voip/manager_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/purpshell/meowcaller" "github.com/purpshell/meowcaller/signaling" "go.mau.fi/whatsmeow/types" ) @@ -51,12 +52,45 @@ func TestLocalMuteStateFor(t *testing.T) { } } -func TestLocalVideoStateFor(t *testing.T) { - if got := localVideoStateFor(false); got != signaling.VideoStateActive { - t.Fatalf("unmuted video state = %d, want %d", got, signaling.VideoStateActive) +func TestMatrixVideoActionFor(t *testing.T) { + tests := []struct { + muted, sending, receiving bool + want matrixVideoAction + }{ + {muted: true, sending: true, receiving: true, want: matrixVideoDisable}, + {muted: true, sending: false, receiving: true, want: matrixVideoNone}, + {muted: false, sending: true, receiving: false, want: matrixVideoEnable}, + {muted: false, sending: false, receiving: true, want: matrixVideoEnable}, + {muted: false, sending: false, receiving: false, want: matrixVideoUpgrade}, } - if got := localVideoStateFor(true); got != localVideoInactiveState { - t.Fatalf("muted video state = %d, want %d", got, localVideoInactiveState) + for _, tc := range tests { + if got := matrixVideoActionFor(tc.muted, tc.sending, tc.receiving); got != tc.want { + t.Errorf("muted:%v sending:%v receiving:%v => %d, want %d", + tc.muted, tc.sending, tc.receiving, got, tc.want) + } + } +} + +func TestRemoteVideoMuteForStateOnlyChangesPeerOwnedFlow(t *testing.T) { + tests := []struct { + state int + muted bool + changed bool + }{ + {signaling.VideoStateEnabled, false, true}, + {signaling.VideoStateDisabled, true, true}, + {signaling.VideoStateStopped, true, true}, + {signaling.VideoStateUpgradeRequestV2, false, false}, + {signaling.VideoStateUpgradeAccept, false, false}, + {signaling.VideoStateUpgradeReject, false, false}, + {signaling.VideoStateUpgradeCancel, false, false}, + } + for _, tc := range tests { + muted, changed := remoteVideoMuteForState(meowcaller.VideoState{Raw: tc.state}) + if muted != tc.muted || changed != tc.changed { + t.Errorf("state %d => muted:%v changed:%v, want muted:%v changed:%v", + tc.state, muted, changed, tc.muted, tc.changed) + } } } From be0ce2e49cfde9586f2874cbba4cffced6d6195d Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Fri, 17 Jul 2026 12:57:22 +0300 Subject: [PATCH 17/44] go.mod: update meowcaller web video state --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 499d8b4..baf4a5d 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8 + github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 8f03e3c..2c1e90d 100644 --- a/go.sum +++ b/go.sum @@ -210,6 +210,8 @@ github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a h1:/RrPY0TupZ github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8 h1:4mJDTzyk2CiX1G9/Wup2jMpZxDXdsWjbiDmuQcwc+xI= github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a h1:6FTf0Npxq5LfJG07p8WPDhUeRePTKykycm1LmO2nTtU= +github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 666fc786097dd67e057b914691ceac13fbaab4c1 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Tue, 21 Jul 2026 23:02:46 +0200 Subject: [PATCH 18/44] manager.go: keep video toggles directional --- pkg/connector/voip/manager.go | 23 ++++++++++------------- pkg/connector/voip/manager_test.go | 9 +++++---- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 032e197..d1178a4 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -24,9 +24,8 @@ type matrixVideoAction uint8 const ( matrixVideoNone matrixVideoAction = iota - matrixVideoDisable - matrixVideoEnable - matrixVideoUpgrade + matrixVideoStop + matrixVideoStart ) var localMuteRetryIntervals = []time.Duration{ @@ -460,26 +459,24 @@ func localMuteStateFor(muted bool) string { return localUnmuteState } -func matrixVideoActionFor(muted, sending, receiving bool) matrixVideoAction { +func matrixVideoActionFor(muted, sending, _ bool) matrixVideoAction { if muted { if sending { - return matrixVideoDisable + return matrixVideoStop } return matrixVideoNone } - if sending || receiving { - return matrixVideoEnable + if sending { + return matrixVideoNone } - return matrixVideoUpgrade + return matrixVideoStart } func applyMatrixVideoState(call *meowcaller.Call, muted bool) error { switch matrixVideoActionFor(muted, call.IsSendingVideo(), call.IsReceivingVideo()) { - case matrixVideoDisable: - return call.SetVideoEnabled(false) - case matrixVideoEnable: - return call.SetVideoEnabled(true) - case matrixVideoUpgrade: + case matrixVideoStop: + return call.StopVideo() + case matrixVideoStart: return call.StartVideo() default: return nil diff --git a/pkg/connector/voip/manager_test.go b/pkg/connector/voip/manager_test.go index 53b405b..2f5e6be 100644 --- a/pkg/connector/voip/manager_test.go +++ b/pkg/connector/voip/manager_test.go @@ -57,11 +57,12 @@ func TestMatrixVideoActionFor(t *testing.T) { muted, sending, receiving bool want matrixVideoAction }{ - {muted: true, sending: true, receiving: true, want: matrixVideoDisable}, + {muted: true, sending: true, receiving: true, want: matrixVideoStop}, {muted: true, sending: false, receiving: true, want: matrixVideoNone}, - {muted: false, sending: true, receiving: false, want: matrixVideoEnable}, - {muted: false, sending: false, receiving: true, want: matrixVideoEnable}, - {muted: false, sending: false, receiving: false, want: matrixVideoUpgrade}, + {muted: false, sending: true, receiving: false, want: matrixVideoNone}, + {muted: false, sending: true, receiving: true, want: matrixVideoNone}, + {muted: false, sending: false, receiving: true, want: matrixVideoStart}, + {muted: false, sending: false, receiving: false, want: matrixVideoStart}, } for _, tc := range tests { if got := matrixVideoActionFor(tc.muted, tc.sending, tc.receiving); got != tc.want { From d29d808ebe22901f15ad1d06bd4409c0c5e4cf2f Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 12:11:41 +0200 Subject: [PATCH 19/44] go.mod: update meowcaller video startup --- go.mod | 2 +- go.sum | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index baf4a5d..dec8768 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a + github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 2c1e90d..382bbf6 100644 --- a/go.sum +++ b/go.sum @@ -196,22 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23 h1:ly/0+RFVo1Y1Xt8aNpXloOxHqVf3HPFANTCP5UiYK98= -github.com/purpshell/meowcaller v0.0.0-20260716210728-21af17da0b23/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283 h1:y1B49s8icl4hQTpxoaMdgk3XWSJAXAn0WoK27wK2038= -github.com/purpshell/meowcaller v0.0.0-20260716212626-6cae75f90283/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb h1:VktUaM/cJS0nXFUYB882xp3icPheAq1YNdIWp5Twq8g= -github.com/purpshell/meowcaller v0.0.0-20260717061552-7bdcc1261edb/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954 h1:m2/FsJoVytQhERCn3WP8T01kOQb9EhQdB60YrH1CFE0= -github.com/purpshell/meowcaller v0.0.0-20260717070627-340bd535b954/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98 h1:d5Qu51VoLMgRj7qlRUBOjTsUQSYyyUPSQj7To/Qig3c= -github.com/purpshell/meowcaller v0.0.0-20260717093103-2ef46e5a7b98/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a h1:/RrPY0TupZQSZ6MTz41bbpxUnyApLJUzfT/StrL2Yjs= -github.com/purpshell/meowcaller v0.0.0-20260717093525-595773cd0a0a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8 h1:4mJDTzyk2CiX1G9/Wup2jMpZxDXdsWjbiDmuQcwc+xI= -github.com/purpshell/meowcaller v0.0.0-20260717094426-81bc68b10bc8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a h1:6FTf0Npxq5LfJG07p8WPDhUeRePTKykycm1LmO2nTtU= -github.com/purpshell/meowcaller v0.0.0-20260717095554-216018153f2a/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3 h1:Y9BtW0255YOQd5FT51xlfQfuqAvMgZPjud0bOhHl1tw= +github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 1229760bf192277789f3dd09eccb426cc370efc1 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 12:37:11 +0200 Subject: [PATCH 20/44] go.mod: update meowcaller video metadata --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dec8768..a779140 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3 + github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 382bbf6..9244011 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3 h1:Y9BtW0255YOQd5FT51xlfQfuqAvMgZPjud0bOhHl1tw= github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561 h1:Y+H4pg7SWXBha0/aBpcVf+WkkSUEfhocyIicsteRj9g= +github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From d49b20735d6ac6ec2fc53d402f59e6109903bf12 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 12:47:02 +0200 Subject: [PATCH 21/44] go.mod: update meowcaller diagnostics --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index a779140..9f45718 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561 + github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 9244011..42e3701 100644 --- a/go.sum +++ b/go.sum @@ -196,10 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3 h1:Y9BtW0255YOQd5FT51xlfQfuqAvMgZPjud0bOhHl1tw= -github.com/purpshell/meowcaller v0.0.0-20260722100956-a04997bd95e3/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561 h1:Y+H4pg7SWXBha0/aBpcVf+WkkSUEfhocyIicsteRj9g= -github.com/purpshell/meowcaller v0.0.0-20260722103527-03cd550f7561/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c h1:jgQmtPuX3iXsLN14xk7yhoi55+QydHOqpn8BYfeqc9s= +github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 4c45a219190138b8e65c6260bc00269ce2770be3 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 12:56:04 +0200 Subject: [PATCH 22/44] go.mod: update meowcaller video packetization --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9f45718..0f2eb6d 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c + github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 42e3701..6075c4a 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c h1:jgQmtPuX3iXsLN14xk7yhoi55+QydHOqpn8BYfeqc9s= -github.com/purpshell/meowcaller v0.0.0-20260722104514-1978e63bab2c/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501 h1:z7LKV57PGrpiLnHLMgONQ4TZvHD2led3Iyqaks5vd3I= +github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 638205226a6f4d59d7482991f9a2fd8143074f82 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 13:20:11 +0200 Subject: [PATCH 23/44] go.mod: update meowcaller audio feedback --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0f2eb6d..3a7a3e7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501 + github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 6075c4a..c4bb3c1 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501 h1:z7LKV57PGrpiLnHLMgONQ4TZvHD2led3Iyqaks5vd3I= -github.com/purpshell/meowcaller v0.0.0-20260722105532-2d1c5bfef501/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac h1:+NNteVq7Vllr0fPMQYbbBLyptvVICBbX8xxjBjRCRV0= +github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 6f0e9ba9ffa7e74c6a39143a86c1c721ba5b735d Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 13:40:25 +0200 Subject: [PATCH 24/44] go.mod: update meowcaller low-rate audio --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3a7a3e7..9bc8c59 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac + github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index c4bb3c1..cca9464 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac h1:+NNteVq7Vllr0fPMQYbbBLyptvVICBbX8xxjBjRCRV0= github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0 h1:aiBJNRKNnBNYZ84bU2tSTWsf1pqAKbIBiiLP6lK4hWo= +github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From b91ed6f5b6e3db742ef77153bae69dad57760db7 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 14:00:54 +0200 Subject: [PATCH 25/44] go.mod: update meowcaller audio playout --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9bc8c59..4f79abc 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0 + github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index cca9464..b994e3c 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac h1:+NNteVq7Vl github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0 h1:aiBJNRKNnBNYZ84bU2tSTWsf1pqAKbIBiiLP6lK4hWo= github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7 h1:Y2OrAuxxoA0DPzqHrASJpMnhFjWqpJrfLmZlUGNptDw= +github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 6f23905640ee77adc8da67cebdb9f1f30f7824f7 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 14:18:25 +0200 Subject: [PATCH 26/44] go.mod: update meowcaller audio playout --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4f79abc..447f920 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7 + github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index b994e3c..287b2a4 100644 --- a/go.sum +++ b/go.sum @@ -196,12 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac h1:+NNteVq7Vllr0fPMQYbbBLyptvVICBbX8xxjBjRCRV0= -github.com/purpshell/meowcaller v0.0.0-20260722111820-06beb2b3b2ac/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0 h1:aiBJNRKNnBNYZ84bU2tSTWsf1pqAKbIBiiLP6lK4hWo= -github.com/purpshell/meowcaller v0.0.0-20260722113904-464b47ee55f0/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7 h1:Y2OrAuxxoA0DPzqHrASJpMnhFjWqpJrfLmZlUGNptDw= -github.com/purpshell/meowcaller v0.0.0-20260722115919-6612437100b7/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e h1:NskGTlKxZlWGbXHm2piXRkOOhTQhKby+u2bF0NNubp8= +github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 830003be9b7075443320656c129e4f68e25f0cd8 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 15:13:10 +0200 Subject: [PATCH 27/44] go.mod: update meowcaller low-rate audio --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 447f920..a4cfe9d 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e + github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 287b2a4..9206e6b 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e h1:NskGTlKxZlWGbXHm2piXRkOOhTQhKby+u2bF0NNubp8= github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c h1:85lucaWq/0HcTJxJb2XbE3AM+PYQZJOnrWnjmF23/ZI= +github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From d887137a81ca39cd24afe60d4e9f18e5be2f69ef Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 15:30:35 +0200 Subject: [PATCH 28/44] pkg/connector/voip/video.go: forward WhatsApp orientation to LiveKit --- pkg/connector/voip/livekit.go | 4 +- pkg/connector/voip/video.go | 133 +++++++++++++++++++++++++++++++ pkg/connector/voip/video_test.go | 31 +++++++ 3 files changed, 166 insertions(+), 2 deletions(-) diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index b4f1c4a..76293f7 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -28,7 +28,7 @@ type LiveKitParticipant struct { audio *lkmedia.PCMLocalTrack audioPub *lksdk.LocalTrackPublication audioSrc *MeowcallerAudioSource - video *lksdk.LocalTrack + video *liveKitVideoTrack videoPub *lksdk.LocalTrackPublication mu sync.Mutex @@ -197,7 +197,7 @@ func (p *LiveKitParticipant) PublishVideoTrack(name string) error { if p.video != nil { return nil } - track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + track, err := newLiveKitVideoTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) if err != nil { return err } diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go index d97264b..8e63cbf 100644 --- a/pkg/connector/voip/video.go +++ b/pkg/connector/voip/video.go @@ -7,9 +7,13 @@ import ( "time" lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" "github.com/purpshell/meowcaller" wartp "github.com/purpshell/meowcaller/rtp" + "go.mau.fi/util/random" ) var ErrVideoSinkClosed = errors.New("voip: video sink closed") @@ -27,6 +31,135 @@ type liveKitVideoOrientationSetter interface { SetVideoOrientation(uint8) } +const ( + liveKitVideoMTU = 1200 + liveKitVideoOrientationURI = "urn:3gpp:video-orientation" + liveKitVideoOrientationBytes = 8 +) + +// liveKitVideoTrack adds CVO to LiveKit's sample track. The SDK does not expose +// video-orientation in SampleWriteOptions, so this track packetizes H.264 itself. +type liveKitVideoTrack struct { + rtpTrack *webrtc.TrackLocalStaticRTP + codec webrtc.RTPCodecCapability + + mu sync.Mutex + packetizer rtp.Packetizer + orientation uint8 + orientationExtensionID uint8 + closed bool +} + +func newLiveKitVideoTrack(codec webrtc.RTPCodecCapability) (*liveKitVideoTrack, error) { + id := "whatsapp-video-" + random.String(12) + track, err := webrtc.NewTrackLocalStaticRTP(codec, id, id) + if err != nil { + return nil, err + } + return &liveKitVideoTrack{rtpTrack: track, codec: codec}, nil +} + +func (t *liveKitVideoTrack) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { + codec, err := t.rtpTrack.Bind(ctx) + if err != nil { + return codec, err + } + + var orientationExtensionID uint8 + for _, extension := range ctx.HeaderExtensions() { + if extension.URI == liveKitVideoOrientationURI { + orientationExtensionID = uint8(extension.ID) + break + } + } + t.mu.Lock() + t.orientationExtensionID = orientationExtensionID + t.packetizer = rtp.NewPacketizer( + liveKitVideoMTU-liveKitVideoOrientationBytes, + 0, + 0, + &codecs.H264Payloader{}, + rtp.NewRandomSequencer(), + codec.ClockRate, + ) + t.mu.Unlock() + return codec, nil +} + +func (t *liveKitVideoTrack) Unbind(ctx webrtc.TrackLocalContext) error { + t.mu.Lock() + t.packetizer = nil + t.orientationExtensionID = 0 + t.mu.Unlock() + return t.rtpTrack.Unbind(ctx) +} + +func (t *liveKitVideoTrack) WriteSample(sample media.Sample, _ *lksdk.SampleWriteOptions) error { + return t.writeSampleRTP(sample, func(packet *rtp.Packet) error { + return t.rtpTrack.WriteRTP(packet) + }) +} + +func (t *liveKitVideoTrack) writeSampleRTP(sample media.Sample, write func(*rtp.Packet) error) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closed || t.packetizer == nil || len(sample.Data) == 0 { + return nil + } + + samples := uint32(sample.Duration.Seconds() * float64(liveKitH264ClockRate)) + for _, packet := range t.packetizer.Packetize(sample.Data, samples) { + if t.orientationExtensionID != 0 { + if err := packet.SetExtension(t.orientationExtensionID, []byte{t.orientation & 0x03}); err != nil { + return err + } + } + if err := write(packet); err != nil { + return err + } + } + return nil +} + +func (t *liveKitVideoTrack) SetVideoOrientation(orientation uint8) { + t.mu.Lock() + t.orientation = orientation & 0x03 + t.mu.Unlock() +} + +func (t *liveKitVideoTrack) Codec() webrtc.RTPCodecCapability { + return t.codec +} + +func (t *liveKitVideoTrack) ID() string { + return t.rtpTrack.ID() +} + +func (t *liveKitVideoTrack) RID() string { + return t.rtpTrack.RID() +} + +func (t *liveKitVideoTrack) StreamID() string { + return t.rtpTrack.StreamID() +} + +func (t *liveKitVideoTrack) Kind() webrtc.RTPCodecType { + return t.rtpTrack.Kind() +} + +func (t *liveKitVideoTrack) Close() error { + t.mu.Lock() + t.closed = true + t.packetizer = nil + t.mu.Unlock() + return nil +} + +var _ liveKitVideoOrientationSetter = (*liveKitVideoTrack)(nil) +var _ lksdk.LocalTrackWithClose = (*liveKitVideoTrack)(nil) +var _ lksdk.TrackLocalWithCodec = (*liveKitVideoTrack)(nil) +var _ webrtc.TrackLocal = (*liveKitVideoTrack)(nil) + type h264ParameterSetRepeater struct { sps []byte pps []byte diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go index 504716d..23bc536 100644 --- a/pkg/connector/voip/video_test.go +++ b/pkg/connector/voip/video_test.go @@ -6,6 +6,7 @@ import ( "time" lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/rtp" "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" "github.com/purpshell/meowcaller" @@ -87,6 +88,36 @@ func TestLiveKitH264WriterSetsVideoOrientation(t *testing.T) { } } +func TestLiveKitVideoTrackWritesOrientationExtension(t *testing.T) { + packetizer := rtp.NewPacketizer(1192, 96, 1234, &recordingPayloader{}, rtp.NewFixedSequencer(1), liveKitH264ClockRate) + track := &liveKitVideoTrack{ + packetizer: packetizer, + orientation: 1, + orientationExtensionID: 13, + } + var packets []*rtp.Packet + write := func(packet *rtp.Packet) error { + packets = append(packets, packet.Clone()) + return nil + } + + if err := track.writeSampleRTP(media.Sample{Data: []byte{1, 2, 3}, Duration: time.Second / 30}, write); err != nil { + t.Fatalf("writeSampleRTP: %v", err) + } + if len(packets) != 1 { + t.Fatalf("packet count = %d, want 1", len(packets)) + } + if got := packets[0].GetExtension(13); !bytes.Equal(got, []byte{1}) { + t.Fatalf("orientation extension = %x, want 01", got) + } +} + +type recordingPayloader struct{} + +func (*recordingPayloader) Payload(_ uint16, payload []byte) [][]byte { + return [][]byte{payload} +} + func TestLiveKitParticipantRequestsRemoteVideoKeyframe(t *testing.T) { const wantSSRC = webrtc.SSRC(0x12345678) var gotSSRC webrtc.SSRC From a2dfba2473154c209b5c4d5daa294044ff7e3248 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 15:41:23 +0200 Subject: [PATCH 29/44] go.mod: update meowcaller video orientation --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index a4cfe9d..ca3befa 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c + github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index 9206e6b..c7de565 100644 --- a/go.sum +++ b/go.sum @@ -196,10 +196,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e h1:NskGTlKxZlWGbXHm2piXRkOOhTQhKby+u2bF0NNubp8= -github.com/purpshell/meowcaller v0.0.0-20260722121634-54bbcad94d4e/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= -github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c h1:85lucaWq/0HcTJxJb2XbE3AM+PYQZJOnrWnjmF23/ZI= -github.com/purpshell/meowcaller v0.0.0-20260722131108-cca0ef73986c/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527 h1:RDtFZuvgsg2KZtyY1bZLmG/xg42+OukL86O+OFFIMhI= +github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 7e9cdabfc87b9d39b97b04c1d2e9e2bb0117594e Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 16:18:07 +0200 Subject: [PATCH 30/44] video.go: remove unsupported orientation forwarding --- pkg/connector/voip/livekit.go | 8 +- pkg/connector/voip/video.go | 133 ------------------------------- pkg/connector/voip/video_test.go | 31 ------- 3 files changed, 3 insertions(+), 169 deletions(-) diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index 76293f7..8dbde38 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -28,7 +28,7 @@ type LiveKitParticipant struct { audio *lkmedia.PCMLocalTrack audioPub *lksdk.LocalTrackPublication audioSrc *MeowcallerAudioSource - video *liveKitVideoTrack + video *lksdk.LocalTrack videoPub *lksdk.LocalTrackPublication mu sync.Mutex @@ -197,7 +197,7 @@ func (p *LiveKitParticipant) PublishVideoTrack(name string) error { if p.video != nil { return nil } - track, err := newLiveKitVideoTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) if err != nil { return err } @@ -249,9 +249,7 @@ func (p *LiveKitParticipant) SetWhatsAppVideoOrientation(orientation int) { return } if setLiveKitVideoOrientation(video, orientation) { - p.log.Debug(). - Int("orientation", orientation&0x03). - Msg("Set LiveKit WhatsApp video orientation") + p.log.Debug().Int("orientation", orientation&0x03).Msg("Set LiveKit WhatsApp video orientation") } } diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go index 8e63cbf..d97264b 100644 --- a/pkg/connector/voip/video.go +++ b/pkg/connector/voip/video.go @@ -7,13 +7,9 @@ import ( "time" lksdk "github.com/livekit/server-sdk-go/v2" - "github.com/pion/rtp" - "github.com/pion/rtp/codecs" - "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" "github.com/purpshell/meowcaller" wartp "github.com/purpshell/meowcaller/rtp" - "go.mau.fi/util/random" ) var ErrVideoSinkClosed = errors.New("voip: video sink closed") @@ -31,135 +27,6 @@ type liveKitVideoOrientationSetter interface { SetVideoOrientation(uint8) } -const ( - liveKitVideoMTU = 1200 - liveKitVideoOrientationURI = "urn:3gpp:video-orientation" - liveKitVideoOrientationBytes = 8 -) - -// liveKitVideoTrack adds CVO to LiveKit's sample track. The SDK does not expose -// video-orientation in SampleWriteOptions, so this track packetizes H.264 itself. -type liveKitVideoTrack struct { - rtpTrack *webrtc.TrackLocalStaticRTP - codec webrtc.RTPCodecCapability - - mu sync.Mutex - packetizer rtp.Packetizer - orientation uint8 - orientationExtensionID uint8 - closed bool -} - -func newLiveKitVideoTrack(codec webrtc.RTPCodecCapability) (*liveKitVideoTrack, error) { - id := "whatsapp-video-" + random.String(12) - track, err := webrtc.NewTrackLocalStaticRTP(codec, id, id) - if err != nil { - return nil, err - } - return &liveKitVideoTrack{rtpTrack: track, codec: codec}, nil -} - -func (t *liveKitVideoTrack) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { - codec, err := t.rtpTrack.Bind(ctx) - if err != nil { - return codec, err - } - - var orientationExtensionID uint8 - for _, extension := range ctx.HeaderExtensions() { - if extension.URI == liveKitVideoOrientationURI { - orientationExtensionID = uint8(extension.ID) - break - } - } - t.mu.Lock() - t.orientationExtensionID = orientationExtensionID - t.packetizer = rtp.NewPacketizer( - liveKitVideoMTU-liveKitVideoOrientationBytes, - 0, - 0, - &codecs.H264Payloader{}, - rtp.NewRandomSequencer(), - codec.ClockRate, - ) - t.mu.Unlock() - return codec, nil -} - -func (t *liveKitVideoTrack) Unbind(ctx webrtc.TrackLocalContext) error { - t.mu.Lock() - t.packetizer = nil - t.orientationExtensionID = 0 - t.mu.Unlock() - return t.rtpTrack.Unbind(ctx) -} - -func (t *liveKitVideoTrack) WriteSample(sample media.Sample, _ *lksdk.SampleWriteOptions) error { - return t.writeSampleRTP(sample, func(packet *rtp.Packet) error { - return t.rtpTrack.WriteRTP(packet) - }) -} - -func (t *liveKitVideoTrack) writeSampleRTP(sample media.Sample, write func(*rtp.Packet) error) error { - t.mu.Lock() - defer t.mu.Unlock() - if t.closed || t.packetizer == nil || len(sample.Data) == 0 { - return nil - } - - samples := uint32(sample.Duration.Seconds() * float64(liveKitH264ClockRate)) - for _, packet := range t.packetizer.Packetize(sample.Data, samples) { - if t.orientationExtensionID != 0 { - if err := packet.SetExtension(t.orientationExtensionID, []byte{t.orientation & 0x03}); err != nil { - return err - } - } - if err := write(packet); err != nil { - return err - } - } - return nil -} - -func (t *liveKitVideoTrack) SetVideoOrientation(orientation uint8) { - t.mu.Lock() - t.orientation = orientation & 0x03 - t.mu.Unlock() -} - -func (t *liveKitVideoTrack) Codec() webrtc.RTPCodecCapability { - return t.codec -} - -func (t *liveKitVideoTrack) ID() string { - return t.rtpTrack.ID() -} - -func (t *liveKitVideoTrack) RID() string { - return t.rtpTrack.RID() -} - -func (t *liveKitVideoTrack) StreamID() string { - return t.rtpTrack.StreamID() -} - -func (t *liveKitVideoTrack) Kind() webrtc.RTPCodecType { - return t.rtpTrack.Kind() -} - -func (t *liveKitVideoTrack) Close() error { - t.mu.Lock() - t.closed = true - t.packetizer = nil - t.mu.Unlock() - return nil -} - -var _ liveKitVideoOrientationSetter = (*liveKitVideoTrack)(nil) -var _ lksdk.LocalTrackWithClose = (*liveKitVideoTrack)(nil) -var _ lksdk.TrackLocalWithCodec = (*liveKitVideoTrack)(nil) -var _ webrtc.TrackLocal = (*liveKitVideoTrack)(nil) - type h264ParameterSetRepeater struct { sps []byte pps []byte diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go index 23bc536..504716d 100644 --- a/pkg/connector/voip/video_test.go +++ b/pkg/connector/voip/video_test.go @@ -6,7 +6,6 @@ import ( "time" lksdk "github.com/livekit/server-sdk-go/v2" - "github.com/pion/rtp" "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" "github.com/purpshell/meowcaller" @@ -88,36 +87,6 @@ func TestLiveKitH264WriterSetsVideoOrientation(t *testing.T) { } } -func TestLiveKitVideoTrackWritesOrientationExtension(t *testing.T) { - packetizer := rtp.NewPacketizer(1192, 96, 1234, &recordingPayloader{}, rtp.NewFixedSequencer(1), liveKitH264ClockRate) - track := &liveKitVideoTrack{ - packetizer: packetizer, - orientation: 1, - orientationExtensionID: 13, - } - var packets []*rtp.Packet - write := func(packet *rtp.Packet) error { - packets = append(packets, packet.Clone()) - return nil - } - - if err := track.writeSampleRTP(media.Sample{Data: []byte{1, 2, 3}, Duration: time.Second / 30}, write); err != nil { - t.Fatalf("writeSampleRTP: %v", err) - } - if len(packets) != 1 { - t.Fatalf("packet count = %d, want 1", len(packets)) - } - if got := packets[0].GetExtension(13); !bytes.Equal(got, []byte{1}) { - t.Fatalf("orientation extension = %x, want 01", got) - } -} - -type recordingPayloader struct{} - -func (*recordingPayloader) Payload(_ uint16, payload []byte) [][]byte { - return [][]byte{payload} -} - func TestLiveKitParticipantRequestsRemoteVideoKeyframe(t *testing.T) { const wantSSRC = webrtc.SSRC(0x12345678) var gotSSRC webrtc.SSRC From aed702e5a116174dd4602a47ca4bb033ad6bcda4 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 16:29:32 +0200 Subject: [PATCH 31/44] manager.go: accept peer video upgrades --- pkg/connector/voip/manager.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index d1178a4..2c6598d 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -412,8 +412,16 @@ func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.Video if muteChanged { m.whatsAppVideoMuted[callID] = muted } + call := m.calls[callID] participant := m.livekit[callID] m.mu.Unlock() + if state.Upgrade && call != nil { + if err := call.AcceptVideo(); err != nil { + m.log.Warn().Err(err).Str("call_id", callID).Int("raw_state", state.Raw).Msg("Failed to accept WhatsApp peer video upgrade") + } else { + m.log.Info().Str("call_id", callID).Int("raw_state", state.Raw).Msg("Accepted WhatsApp peer video upgrade") + } + } if participant != nil { if muteChanged { participant.SetWhatsAppVideoMuted(muted) From 9810204dfe0b25335017d9e0043d163d065ea7f2 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Wed, 22 Jul 2026 16:58:32 +0200 Subject: [PATCH 32/44] go.mod: update meowcaller peer video upgrades --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ca3befa..b3473e4 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527 + github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.10 diff --git a/go.sum b/go.sum index c7de565..0875490 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527 h1:RDtFZuvgsg2KZtyY1bZLmG/xg42+OukL86O+OFFIMhI= github.com/purpshell/meowcaller v0.0.0-20260722133919-2ca8d7d30527/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8 h1:sVe8DiEWPu+oEkhbLUMPrTYXxaRQ/V3tUZld/5y2jYE= +github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= From 9b5c322351cc523f2cc8334927df05d266dbba81 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 04:13:45 +0200 Subject: [PATCH 33/44] matrixrtc: dial WhatsApp group calls --- go.mod | 4 ++-- go.sum | 4 ++++ pkg/connector/matrixrtc_outgoing.go | 11 ++++++++--- pkg/connector/matrixrtc_test.go | 8 ++++++++ pkg/connector/voip/manager.go | 16 ++++++++++++++++ 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 94babdc..9ba1a79 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,12 @@ require ( github.com/livekit/server-sdk-go/v2 v2.17.0 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 - github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8 + github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391 go.mau.fi/webp v0.3.0 - go.mau.fi/whatsmeow v0.0.0-20260720135917-a2381054887e + go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933 go.yaml.in/yaml/v3 v3.0.4 golang.org/x/image v0.44.0 golang.org/x/net v0.57.0 diff --git a/go.sum b/go.sum index d63736a..eba43aa 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8 h1:sVe8DiEWPu+oEkhbLUMPrTYXxaRQ/V3tUZld/5y2jYE= github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= +github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 h1:SnLX76CnagumooXRm63BK9Rn2/e/Th6aWWJJKTqOk2k= +github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807/go.mod h1:kSME01MaSkwul6tSmExmgBWUOmfkw3DNpfnozsn01eE= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= @@ -249,6 +251,8 @@ go.mau.fi/webp v0.3.0 h1:gVHQZtz21Ziwj+CDuklbX9mqpsnDIFKxs/BJyV7iZzA= go.mau.fi/webp v0.3.0/go.mod h1:rlZFTev+dYxhvk+XNBP/5GcTt4gXmzAB4DU0aGUYIQo= go.mau.fi/whatsmeow v0.0.0-20260720135917-a2381054887e h1:Kn2XQTpYRploXqtmJKjeX4ZYGevhIeQkO2PWGPGdwrY= go.mau.fi/whatsmeow v0.0.0-20260720135917-a2381054887e/go.mod h1:Iy/xVSuVU2payR26MB1hv0UZUWRraEn4qKZ7+VRHulg= +go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933 h1:7skZGs9q+rWKqYHok4ZufzhKpf6GmTKZnTjSm0aDtus= +go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933/go.mod h1:Iy/xVSuVU2payR26MB1hv0UZUWRraEn4qKZ7+VRHulg= go.mau.fi/zeroconfig v0.2.0 h1:e/OGEERqVRRKlgaro7E6bh8xXiKFSXB3eNNIud7FUjU= go.mau.fi/zeroconfig v0.2.0/go.mod h1:J0Vn0prHNOm493oZoQ84kq83ZaNCYZnq+noI1b1eN8w= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index 73fbdd4..ddd39aa 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -203,7 +203,7 @@ func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal return err } if !matrixRTCPortalSupportsWhatsAppCalls(peer) { - return fmt.Errorf("MatrixRTC WhatsApp calls are only supported in 1:1 portals, not %s", peer.Server) + return fmt.Errorf("MatrixRTC WhatsApp calls are not supported in %s portals", peer.Server) } mediaKind, downgradedMedia := matrixRTCOutboundMediaKind(trigger) if mediaKind == "" { @@ -237,7 +237,12 @@ func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal return err } - call, err := wa.VOIP.Dial(ctx, peer.ToNonAD().String(), mediaKind == "video") + var call *meowcaller.Call + if peer.Server == types.GroupServer { + call, err = wa.VOIP.DialGroupByID(ctx, peer.ToNonAD().String(), mediaKind == "video") + } else { + call, err = wa.VOIP.Dial(ctx, peer.ToNonAD().String(), mediaKind == "video") + } if err != nil { return err } @@ -578,7 +583,7 @@ func (wa *WhatsAppClient) matrixRTCIntentForMXID(ctx context.Context, mxid id.Us func matrixRTCPortalSupportsWhatsAppCalls(peer types.JID) bool { switch peer.Server { - case types.DefaultUserServer, types.HiddenUserServer: + case types.DefaultUserServer, types.HiddenUserServer, types.GroupServer: return true default: return false diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index aa1d8c6..c9a9405 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -179,6 +180,13 @@ func TestShouldEndMatrixRTCCallFromMembershipRejectsOtherParticipant(t *testing. } } +func TestMatrixRTCPortalSupportsWhatsAppGroupCalls(t *testing.T) { + peer := types.NewJID("120363000000000000", types.GroupServer) + if !matrixRTCPortalSupportsWhatsAppCalls(peer) { + t.Fatal("matrixRTCPortalSupportsWhatsAppCalls rejected a WhatsApp group portal") + } +} + func TestReserveMatrixRTCOutboundStartSuppressesDuplicates(t *testing.T) { wa := &WhatsAppConnector{} if !wa.reserveMatrixRTCOutboundStart("!room:example.com") { diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 2c6598d..e1bbd8d 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -138,6 +138,22 @@ func (m *Manager) Dial(ctx context.Context, target string, video ...bool) (*meow return call, nil } +func (m *Manager) DialGroupByID(ctx context.Context, groupID string, video ...bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + opts := meowcaller.GroupCallOptions{} + if len(video) > 0 { + opts.Video = video[0] + } + call, err := m.client.GroupCallByIDWithOptions(ctx, groupID, opts) + if err != nil { + return nil, err + } + m.trackCall(call, m.ownCallCreator()) + return call, nil +} + func (m *Manager) AbortAll() { if m == nil { return From 989464a146ea076971682e0905af1b6f5a6b59ea Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 04:34:36 +0200 Subject: [PATCH 34/44] voip: bridge group video and screen sharing --- pkg/connector/voip/group_media.go | 243 +++++++++++++++++++++++++ pkg/connector/voip/group_media_test.go | 133 ++++++++++++++ pkg/connector/voip/livekit.go | 234 +++++++++++++++++++----- pkg/connector/voip/livekit_test.go | 30 +++ pkg/connector/voip/manager.go | 107 ++++++++++- pkg/connector/voip/video_test.go | 25 +++ 6 files changed, 721 insertions(+), 51 deletions(-) create mode 100644 pkg/connector/voip/group_media.go create mode 100644 pkg/connector/voip/group_media_test.go create mode 100644 pkg/connector/voip/livekit_test.go diff --git a/pkg/connector/voip/group_media.go b/pkg/connector/voip/group_media.go new file mode 100644 index 0000000..63d29e3 --- /dev/null +++ b/pkg/connector/voip/group_media.go @@ -0,0 +1,243 @@ +package voip + +import ( + "sync" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" +) + +type whatsAppVideoSink interface { + WriteVideo([]byte) error + SetOrientation(int) +} + +type whatsAppVideoRouter struct { + mu sync.Mutex + + camera whatsAppVideoSink + screen whatsAppVideoSink + setCameraMute func(bool) + setScreenMute func(bool) + + group bool + connected map[string]struct{} + screenSharers map[string]struct{} + selectedCamera string + selectedScreen string +} + +func newWhatsAppVideoRouter( + camera, screen whatsAppVideoSink, + setCameraMute, setScreenMute func(bool), +) *whatsAppVideoRouter { + return &whatsAppVideoRouter{ + camera: camera, + screen: screen, + setCameraMute: setCameraMute, + setScreenMute: setScreenMute, + connected: make(map[string]struct{}), + screenSharers: make(map[string]struct{}), + } +} + +func (r *whatsAppVideoRouter) SetGroupState(state meowcaller.GroupCallState) { + if r == nil { + return + } + connected := make(map[string]struct{}) + for _, participant := range state.Participants { + if participant.State != "connected" { + continue + } + addVideoParticipantIdentity(connected, participant.JID) + addVideoParticipantIdentity(connected, participant.PN) + for _, device := range participant.Devices { + addVideoParticipantIdentity(connected, device.JID) + } + } + + r.mu.Lock() + r.group = true + r.connected = connected + cameraRemoved := r.selectedCamera != "" + if cameraRemoved { + _, cameraRemoved = connected[r.selectedCamera] + cameraRemoved = !cameraRemoved + } + if cameraRemoved { + r.selectedCamera = "" + } + screenRemoved := r.selectedScreen != "" + if screenRemoved { + _, screenRemoved = connected[r.selectedScreen] + screenRemoved = !screenRemoved + } + if screenRemoved { + delete(r.screenSharers, r.selectedScreen) + r.selectedScreen = "" + } + setCameraMute := r.setCameraMute + setScreenMute := r.setScreenMute + r.mu.Unlock() + + if cameraRemoved && setCameraMute != nil { + setCameraMute(true) + } + if screenRemoved && setScreenMute != nil { + setScreenMute(true) + } +} + +func (r *whatsAppVideoRouter) SetScreenShare(state meowcaller.ScreenShareState) { + if r == nil || state.Participant.IsEmpty() { + return + } + participant := videoParticipantIdentity(state.Participant) + r.mu.Lock() + if state.Active { + r.screenSharers[participant] = struct{}{} + if r.selectedScreen == "" { + r.selectedScreen = participant + } + } else { + delete(r.screenSharers, participant) + if r.selectedScreen == participant { + r.selectedScreen = "" + } + } + selected := r.selectedScreen + setScreenMute := r.setScreenMute + r.mu.Unlock() + + if setScreenMute != nil { + setScreenMute(selected == "") + } +} + +func (r *whatsAppVideoRouter) WriteParticipantFrame(frame meowcaller.ParticipantVideoFrame) { + if r == nil || len(frame.AccessUnit) == 0 { + return + } + identity := participantVideoFrameIdentity(frame) + r.mu.Lock() + _, sharing := r.screenSharers[identity] + if sharing { + if r.selectedScreen == "" { + r.selectedScreen = identity + } + if r.selectedScreen != identity { + r.mu.Unlock() + return + } + sink := r.screen + setMuted := r.setScreenMute + r.mu.Unlock() + if setMuted != nil { + setMuted(false) + } + writeWhatsAppVideoFrame(sink, frame) + return + } + + if r.group { + if r.selectedCamera == "" { + r.selectedCamera = identity + } + if r.selectedCamera != identity { + r.mu.Unlock() + return + } + } + sink := r.camera + setMuted := r.setCameraMute + r.mu.Unlock() + if setMuted != nil { + setMuted(false) + } + writeWhatsAppVideoFrame(sink, frame) +} + +func writeWhatsAppVideoFrame(sink whatsAppVideoSink, frame meowcaller.ParticipantVideoFrame) { + if sink == nil { + return + } + sink.SetOrientation(frame.Orientation) + _ = sink.WriteVideo(frame.AccessUnit) +} + +func participantVideoFrameIdentity(frame meowcaller.ParticipantVideoFrame) string { + if !frame.Sender.IsEmpty() { + return videoParticipantIdentity(frame.Sender) + } + if !frame.Device.IsEmpty() { + return videoParticipantIdentity(frame.Device) + } + return frame.ParticipantID +} + +func addVideoParticipantIdentity(target map[string]struct{}, jid types.JID) { + if !jid.IsEmpty() { + target[videoParticipantIdentity(jid)] = struct{}{} + } +} + +func videoParticipantIdentity(jid types.JID) string { + return jid.ToNonAD().String() +} + +type matrixVideoSourceRouter struct { + mu sync.RWMutex + screenSharing bool + write func(LiveKitVideoFrame) error +} + +func newMatrixVideoSourceRouter(write func(LiveKitVideoFrame) error) *matrixVideoSourceRouter { + return &matrixVideoSourceRouter{write: write} +} + +func (r *matrixVideoSourceRouter) SetScreenSharing(active bool) { + if r == nil { + return + } + r.mu.Lock() + r.screenSharing = active + r.mu.Unlock() +} + +func (r *matrixVideoSourceRouter) ScreenSharing() bool { + if r == nil { + return false + } + r.mu.RLock() + defer r.mu.RUnlock() + return r.screenSharing +} + +func (r *matrixVideoSourceRouter) WriteCamera(frame LiveKitVideoFrame) error { + if r == nil { + return nil + } + r.mu.RLock() + screenSharing := r.screenSharing + write := r.write + r.mu.RUnlock() + if screenSharing || write == nil { + return nil + } + return write(frame) +} + +func (r *matrixVideoSourceRouter) WriteScreen(frame LiveKitVideoFrame) error { + if r == nil { + return nil + } + r.mu.RLock() + screenSharing := r.screenSharing + write := r.write + r.mu.RUnlock() + if !screenSharing || write == nil { + return nil + } + return write(frame) +} diff --git a/pkg/connector/voip/group_media_test.go b/pkg/connector/voip/group_media_test.go new file mode 100644 index 0000000..344388c --- /dev/null +++ b/pkg/connector/voip/group_media_test.go @@ -0,0 +1,133 @@ +package voip + +import ( + "testing" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" +) + +type recordingGroupVideoSink struct { + frames [][]byte + orientations []int + muted []bool +} + +func (s *recordingGroupVideoSink) WriteVideo(frame []byte) error { + s.frames = append(s.frames, append([]byte(nil), frame...)) + return nil +} + +func (s *recordingGroupVideoSink) SetOrientation(orientation int) { + s.orientations = append(s.orientations, orientation) +} + +func (s *recordingGroupVideoSink) setMuted(muted bool) { + s.muted = append(s.muted, muted) +} + +func TestWhatsAppVideoRouterKeepsOneStableGroupCamera(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + alice := types.NewJID("111", types.DefaultUserServer) + bob := types.NewJID("222", types.DefaultUserServer) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + {JID: alice, State: "connected"}, + {JID: bob, State: "connected"}, + }, + }) + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + Orientation: 1, + AccessUnit: []byte{0x01}, + }) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: bob.String(), + Sender: bob, + Orientation: 2, + AccessUnit: []byte{0x02}, + }) + + if len(camera.frames) != 1 || camera.frames[0][0] != 0x01 { + t.Fatalf("camera frames = %v, want only the first connected participant", camera.frames) + } + if len(camera.orientations) != 1 || camera.orientations[0] != 1 { + t.Fatalf("camera orientations = %v, want [1]", camera.orientations) + } +} + +func TestWhatsAppVideoRouterSeparatesScreenShareFromCamera(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + alice := types.NewJID("111", types.DefaultUserServer) + router.SetScreenShare(meowcaller.ScreenShareState{Participant: alice, Active: true}) + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + Orientation: 3, + AccessUnit: []byte{0x03}, + }) + + if len(camera.frames) != 0 { + t.Fatalf("camera received screen-share frames: %v", camera.frames) + } + if len(screen.frames) != 1 || screen.frames[0][0] != 0x03 { + t.Fatalf("screen frames = %v, want one screen-share frame", screen.frames) + } + if len(screen.orientations) != 1 || screen.orientations[0] != 3 { + t.Fatalf("screen orientations = %v, want [3]", screen.orientations) + } + if len(screen.muted) == 0 || screen.muted[len(screen.muted)-1] { + t.Fatalf("screen mute transitions = %v, want unmuted", screen.muted) + } + + router.SetScreenShare(meowcaller.ScreenShareState{Participant: alice, Active: false}) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + AccessUnit: []byte{0x04}, + }) + if len(camera.frames) != 1 || camera.frames[0][0] != 0x04 { + t.Fatalf("camera frames after screen-share stop = %v, want camera frame", camera.frames) + } + if !screen.muted[len(screen.muted)-1] { + t.Fatalf("screen mute transitions = %v, want muted after stop", screen.muted) + } +} + +func TestMatrixVideoRouterForwardsOnlyTheActiveSource(t *testing.T) { + var got []byte + router := newMatrixVideoSourceRouter(func(frame LiveKitVideoFrame) error { + got = append(got, frame.AccessUnit...) + return nil + }) + + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x01}}); err != nil { + t.Fatal(err) + } + router.SetScreenSharing(true) + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x02}}); err != nil { + t.Fatal(err) + } + if err := router.WriteScreen(LiveKitVideoFrame{AccessUnit: []byte{0x03}}); err != nil { + t.Fatal(err) + } + router.SetScreenSharing(false) + if err := router.WriteScreen(LiveKitVideoFrame{AccessUnit: []byte{0x04}}); err != nil { + t.Fatal(err) + } + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x05}}); err != nil { + t.Fatal(err) + } + + want := []byte{0x01, 0x03, 0x05} + if string(got) != string(want) { + t.Fatalf("forwarded frames = %v, want %v", got, want) + } +} diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go index 8dbde38..48511c8 100644 --- a/pkg/connector/voip/livekit.go +++ b/pkg/connector/voip/livekit.go @@ -21,30 +21,44 @@ import ( ) type LiveKitParticipant struct { - cfg LiveKitConfig - videoCfg VideoConfig - log zerolog.Logger - room *lksdk.Room - audio *lkmedia.PCMLocalTrack - audioPub *lksdk.LocalTrackPublication - audioSrc *MeowcallerAudioSource - video *lksdk.LocalTrack - videoPub *lksdk.LocalTrackPublication + cfg LiveKitConfig + videoCfg VideoConfig + log zerolog.Logger + room *lksdk.Room + audio *lkmedia.PCMLocalTrack + audioPub *lksdk.LocalTrackPublication + audioSrc *MeowcallerAudioSource + video *lksdk.LocalTrack + videoPub *lksdk.LocalTrackPublication + screen *lksdk.LocalTrack + screenPub *lksdk.LocalTrackPublication - mu sync.Mutex - remoteAudio []*lkmedia.PCMRemoteTrack - remoteMediaCancel context.CancelFunc - remoteVideoPLI lksdk.PLIWriter - remoteVideoSSRC webrtc.SSRC - remoteVideoKeyframePending bool - remoteVideoKeyframeAwaited bool - disconnected bool - selectedRemoteParticipant string - remoteAudioMuteStateChange func(muted bool) - remoteVideoFrame func(frame LiveKitVideoFrame) error - remoteVideoMuteStateChange func(muted bool) + mu sync.Mutex + remoteAudio []*lkmedia.PCMRemoteTrack + remoteMediaCancel context.CancelFunc + remoteVideoPLI lksdk.PLIWriter + remoteVideoSSRC webrtc.SSRC + remoteScreenPLI lksdk.PLIWriter + remoteScreenSSRC webrtc.SSRC + remoteScreenActive bool + remoteVideoKeyframePending [2]bool + remoteVideoKeyframeAwaited [2]bool + disconnected bool + selectedRemoteParticipant string + remoteAudioMuteStateChange func(muted bool) + remoteVideoFrame func(frame LiveKitVideoFrame) error + remoteVideoMuteStateChange func(muted bool) + remoteScreenFrame func(frame LiveKitVideoFrame) error + remoteScreenMuteStateChange func(muted bool) } +type liveKitVideoSource uint8 + +const ( + liveKitVideoSourceCamera liveKitVideoSource = iota + liveKitVideoSourceScreenShare +) + func ConnectLiveKitParticipant(ctx context.Context, authResp *LiveKitAuthResponse, cfg LiveKitConfig, videoCfg VideoConfig, log zerolog.Logger) (*LiveKitParticipant, error) { if authResp == nil { return nil, fmt.Errorf("livekit auth response is nil") @@ -115,6 +129,13 @@ func (p *LiveKitParticipant) SetRemoteVideoHandlers(selectedParticipant string, p.mu.Unlock() } +func (p *LiveKitParticipant) SetRemoteScreenShareHandlers(frameHandler func(frame LiveKitVideoFrame) error, muteHandler func(muted bool)) { + p.mu.Lock() + p.remoteScreenFrame = frameHandler + p.remoteScreenMuteStateChange = muteHandler + p.mu.Unlock() +} + func (p *LiveKitParticipant) requestRemoteVideoKeyframe() bool { p.mu.Lock() if p.disconnected { @@ -123,39 +144,58 @@ func (p *LiveKitParticipant) requestRemoteVideoKeyframe() bool { } pli := p.remoteVideoPLI ssrc := p.remoteVideoSSRC + source := liveKitVideoSourceCamera + if p.remoteScreenActive && p.remoteScreenPLI != nil && p.remoteScreenSSRC != 0 { + pli = p.remoteScreenPLI + ssrc = p.remoteScreenSSRC + source = liveKitVideoSourceScreenShare + } else if p.remoteScreenActive { + source = liveKitVideoSourceScreenShare + pli = nil + ssrc = 0 + } if pli == nil || ssrc == 0 { - p.remoteVideoKeyframePending = true + p.remoteVideoKeyframePending[source] = true p.mu.Unlock() return false } - p.remoteVideoKeyframePending = false + p.remoteVideoKeyframePending[source] = false p.mu.Unlock() - p.sendRemoteVideoPLI(pli, ssrc) + p.sendRemoteVideoPLI(source, pli, ssrc) return true } func (p *LiveKitParticipant) setRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.setRemoteVideoPLIForSource(liveKitVideoSourceCamera, pli, ssrc) +} + +func (p *LiveKitParticipant) setRemoteVideoPLIForSource(source liveKitVideoSource, pli lksdk.PLIWriter, ssrc webrtc.SSRC) { p.mu.Lock() if p.disconnected { p.mu.Unlock() return } - p.remoteVideoPLI = pli - p.remoteVideoSSRC = ssrc - pending := p.remoteVideoKeyframePending && pli != nil && ssrc != 0 + if source == liveKitVideoSourceScreenShare { + p.remoteScreenPLI = pli + p.remoteScreenSSRC = ssrc + } else { + p.remoteVideoPLI = pli + p.remoteVideoSSRC = ssrc + } + pending := p.remoteVideoKeyframePending[source] && pli != nil && ssrc != 0 if pending { - p.remoteVideoKeyframePending = false + p.remoteVideoKeyframePending[source] = false } p.mu.Unlock() if pending { - p.sendRemoteVideoPLI(pli, ssrc) + p.sendRemoteVideoPLI(source, pli, ssrc) } } -func (p *LiveKitParticipant) sendRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { +func (p *LiveKitParticipant) sendRemoteVideoPLI(source liveKitVideoSource, pli lksdk.PLIWriter, ssrc webrtc.SSRC) { p.mu.Lock() if !p.disconnected { - p.remoteVideoKeyframeAwaited = true + p.remoteVideoKeyframeAwaited[source] = true } p.mu.Unlock() pli(ssrc) @@ -204,12 +244,10 @@ func (p *LiveKitParticipant) PublishVideoTrack(name string) error { if name == "" { name = "whatsapp-video" } - pub, err := p.room.LocalParticipant.PublishTrack(track, &lksdk.TrackPublicationOptions{ - Name: name, - Source: livekitproto.TrackSource_CAMERA, - VideoWidth: p.videoCfg.MaxWidth, - VideoHeight: p.videoCfg.MaxHeight, - }) + pub, err := p.room.LocalParticipant.PublishTrack( + track, + videoTrackPublicationOptions(name, livekitproto.TrackSource_CAMERA, p.videoCfg), + ) if err != nil { _ = track.Close() return err @@ -219,6 +257,45 @@ func (p *LiveKitParticipant) PublishVideoTrack(name string) error { return nil } +func (p *LiveKitParticipant) PublishScreenShareTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.screen != nil { + return nil + } + track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-screen" + } + pub, err := p.room.LocalParticipant.PublishTrack( + track, + videoTrackPublicationOptions(name, livekitproto.TrackSource_SCREEN_SHARE, p.videoCfg), + ) + if err != nil { + _ = track.Close() + return err + } + pub.SetMuted(true) + p.screen = track + p.screenPub = pub + return nil +} + +func videoTrackPublicationOptions(name string, source livekitproto.TrackSource, cfg VideoConfig) *lksdk.TrackPublicationOptions { + return &lksdk.TrackPublicationOptions{ + Name: name, + Source: source, + VideoWidth: cfg.MaxWidth, + VideoHeight: cfg.MaxHeight, + } +} + func (p *LiveKitParticipant) SetWhatsAppAudioMuted(muted bool) { p.mu.Lock() pub := p.audioPub @@ -241,6 +318,17 @@ func (p *LiveKitParticipant) SetWhatsAppVideoMuted(muted bool) { p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp video mute state") } +func (p *LiveKitParticipant) SetWhatsAppScreenShareMuted(muted bool) { + p.mu.Lock() + pub := p.screenPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp screen-share mute state") +} + func (p *LiveKitParticipant) SetWhatsAppVideoOrientation(orientation int) { p.mu.Lock() video := p.video @@ -265,6 +353,12 @@ func (p *LiveKitParticipant) WhatsAppVideoSink() *LiveKitH264Writer { return NewLiveKitH264Writer(p.video, videoFrameDuration(p.videoCfg)) } +func (p *LiveKitParticipant) WhatsAppScreenShareSink() *LiveKitH264Writer { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitH264Writer(p.screen, videoFrameDuration(p.videoCfg)) +} + func (p *LiveKitParticipant) MatrixAudioSource() *MeowcallerAudioSource { return p.audioSrc } @@ -291,11 +385,15 @@ func (p *LiveKitParticipant) Close() { audioPub := p.audioPub video := p.video videoPub := p.videoPub + screen := p.screen + screenPub := p.screenPub p.room = nil p.audio = nil p.audioPub = nil p.video = nil p.videoPub = nil + p.screen = nil + p.screenPub = nil p.mu.Unlock() p.closeRemoteTracks() if audioPub != nil { @@ -304,6 +402,9 @@ func (p *LiveKitParticipant) Close() { if videoPub != nil { videoPub.SetMuted(true) } + if screenPub != nil { + screenPub.SetMuted(true) + } if audio != nil { audio.ClearQueue() _ = audio.Close() @@ -311,6 +412,9 @@ func (p *LiveKitParticipant) Close() { if video != nil { _ = video.Close() } + if screen != nil { + _ = screen.Close() + } if room != nil { room.Disconnect() } @@ -374,18 +478,25 @@ func (p *LiveKitParticipant) onVideoTrackSubscribed(ctx context.Context, track * p.handleRemoteVideoMuteState(publication, rp, true) return } - p.setRemoteVideoPLI(rp.WritePLI, track.SSRC()) + source := liveKitVideoSourceRole(publication.Source()) + p.setRemoteVideoPLIForSource(source, rp.WritePLI, track.SSRC()) p.handleRemoteVideoMuteState(publication, rp, publication.IsMuted()) - go p.forwardRemoteH264Track(ctx, track, rp) + go p.forwardRemoteH264Track(ctx, track, rp, source) } func (p *LiveKitParticipant) onTrackUnsubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { if track.Kind() == webrtc.RTPCodecTypeVideo { + source := liveKitVideoSourceRole(publication.Source()) p.mu.Lock() - if p.remoteVideoSSRC == track.SSRC() { + if source == liveKitVideoSourceScreenShare && p.remoteScreenSSRC == track.SSRC() { + p.remoteScreenPLI = nil + p.remoteScreenSSRC = 0 + p.remoteScreenActive = false + p.remoteVideoKeyframePending[liveKitVideoSourceScreenShare] = true + } else if source == liveKitVideoSourceCamera && p.remoteVideoSSRC == track.SSRC() { p.remoteVideoPLI = nil p.remoteVideoSSRC = 0 - p.remoteVideoKeyframePending = true + p.remoteVideoKeyframePending[liveKitVideoSourceCamera] = true } p.mu.Unlock() p.handleRemoteVideoMuteState(publication, rp, true) @@ -442,7 +553,6 @@ func (p *LiveKitParticipant) handleRemoteVideoMuteState(pub lksdk.TrackPublicati identity := participant.Identity() p.mu.Lock() selected := p.selectedRemoteParticipant - handler := p.remoteVideoMuteStateChange p.mu.Unlock() if !remoteParticipantSelected(selected, identity) { p.log.Debug(). @@ -452,6 +562,14 @@ func (p *LiveKitParticipant) handleRemoteVideoMuteState(pub lksdk.TrackPublicati Msg("Ignoring LiveKit video mute state from non-selected participant") return } + source := liveKitVideoSourceRole(pub.Source()) + p.mu.Lock() + handler := p.remoteVideoMuteStateChange + if source == liveKitVideoSourceScreenShare { + p.remoteScreenActive = !muted + handler = p.remoteScreenMuteStateChange + } + p.mu.Unlock() p.log.Debug(). Str("participant", identity). Str("track_id", pub.SID()). @@ -462,7 +580,7 @@ func (p *LiveKitParticipant) handleRemoteVideoMuteState(pub lksdk.TrackPublicati } } -func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track *webrtc.TrackRemote, rp *lksdk.RemoteParticipant) { +func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track *webrtc.TrackRemote, rp *lksdk.RemoteParticipant, source liveKitVideoSource) { builder := samplebuilder.New( liveKitH264MaxLatePackets, &codecs.H264Packet{}, @@ -473,6 +591,7 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * p.log.Info(). Str("participant", string(rp.Identity())). Str("track_id", track.ID()). + Str("source", source.String()). Str("fmtp", track.Codec().SDPFmtpLine). Msg("Started forwarding LiveKit H.264 video to WhatsApp") for { @@ -498,9 +617,9 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * accessUnit, repeatedParameterSets := parameterSets.Normalize(sample.Data) nalTypes, profileLevelID, hasIDR, hasSPS, hasPPS := h264AccessUnitMetadata(accessUnit) p.mu.Lock() - afterPLI := hasIDR && p.remoteVideoKeyframeAwaited + afterPLI := hasIDR && p.remoteVideoKeyframeAwaited[source] if afterPLI { - p.remoteVideoKeyframeAwaited = false + p.remoteVideoKeyframeAwaited[source] = false } p.mu.Unlock() if hasIDR && (!loggedIDR || afterPLI || repeatedParameterSets) { @@ -519,6 +638,9 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * } p.mu.Lock() handler := p.remoteVideoFrame + if source == liveKitVideoSourceScreenShare { + handler = p.remoteScreenFrame + } p.mu.Unlock() if handler == nil { continue @@ -531,12 +653,27 @@ func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track * Err(err). Str("participant", string(rp.Identity())). Str("track_id", track.ID()). + Str("source", source.String()). Msg("Failed to forward LiveKit H.264 frame to WhatsApp") } } } } +func liveKitVideoSourceRole(source livekitproto.TrackSource) liveKitVideoSource { + if source == livekitproto.TrackSource_SCREEN_SHARE { + return liveKitVideoSourceScreenShare + } + return liveKitVideoSourceCamera +} + +func (s liveKitVideoSource) String() string { + if s == liveKitVideoSourceScreenShare { + return "screen_share" + } + return "camera" +} + func (p *LiveKitParticipant) selectedParticipant() string { p.mu.Lock() defer p.mu.Unlock() @@ -553,8 +690,11 @@ func (p *LiveKitParticipant) closeRemoteTracks() { p.remoteAudio = nil p.remoteVideoPLI = nil p.remoteVideoSSRC = 0 - p.remoteVideoKeyframePending = false - p.remoteVideoKeyframeAwaited = false + p.remoteScreenPLI = nil + p.remoteScreenSSRC = 0 + p.remoteScreenActive = false + p.remoteVideoKeyframePending = [2]bool{} + p.remoteVideoKeyframeAwaited = [2]bool{} cancel := p.remoteMediaCancel p.remoteMediaCancel = nil p.mu.Unlock() diff --git a/pkg/connector/voip/livekit_test.go b/pkg/connector/voip/livekit_test.go new file mode 100644 index 0000000..3a425b7 --- /dev/null +++ b/pkg/connector/voip/livekit_test.go @@ -0,0 +1,30 @@ +package voip + +import ( + "testing" + + livekitproto "github.com/livekit/protocol/livekit" +) + +func TestVideoTrackPublicationOptionsPreserveScreenShareSource(t *testing.T) { + cfg := VideoConfig{MaxWidth: 1280, MaxHeight: 720} + opts := videoTrackPublicationOptions("whatsapp-screen", livekitproto.TrackSource_SCREEN_SHARE, cfg) + if opts.Name != "whatsapp-screen" { + t.Fatalf("track name = %q, want whatsapp-screen", opts.Name) + } + if opts.Source != livekitproto.TrackSource_SCREEN_SHARE { + t.Fatalf("track source = %s, want SCREEN_SHARE", opts.Source) + } + if opts.VideoWidth != 1280 || opts.VideoHeight != 720 { + t.Fatalf("track dimensions = %dx%d, want 1280x720", opts.VideoWidth, opts.VideoHeight) + } +} + +func TestLiveKitVideoSourceClassifiesScreenShareIndependently(t *testing.T) { + if liveKitVideoSourceRole(livekitproto.TrackSource_CAMERA) != liveKitVideoSourceCamera { + t.Fatal("camera publication was not classified as camera") + } + if liveKitVideoSourceRole(livekitproto.TrackSource_SCREEN_SHARE) != liveKitVideoSourceScreenShare { + t.Fatal("screen-share publication was not classified as screen share") + } +} diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index e1bbd8d..d9bf0b6 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -217,7 +217,7 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth if videoEnabled { var videoBuffer whatsAppVideoStartupBuffer var videoBufferLock sync.Mutex - participant.SetRemoteVideoHandlers(selectedRemoteParticipantID, func(frame LiveKitVideoFrame) error { + sendMatrixVideoFrame := func(frame LiveKitVideoFrame) error { if call.State() == meowcaller.CallPhaseEnded { return nil } @@ -243,8 +243,13 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth Msg("Flushed buffered LiveKit H.264 video to WhatsApp") } return nil - }, func(muted bool) { - m.handleMatrixVideoMuteState(call, muted) + } + sourceRouter := newMatrixVideoSourceRouter(sendMatrixVideoFrame) + participant.SetRemoteVideoHandlers(selectedRemoteParticipantID, sourceRouter.WriteCamera, func(muted bool) { + m.handleMatrixCameraMuteState(call, sourceRouter, muted) + }) + participant.SetRemoteScreenShareHandlers(sourceRouter.WriteScreen, func(muted bool) { + m.handleMatrixScreenShareMuteState(call, participant, sourceRouter, muted) }) } if err = participant.PublishAudioTrack("whatsapp-audio"); err != nil { @@ -262,7 +267,26 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth m.clearLiveKitConnecting(waCallID) return err } - call.ReceiveVideo(participant.WhatsAppVideoSink()) + if err = participant.PublishScreenShareTrack("whatsapp-screen"); err != nil { + call.Receive(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + videoRouter := newWhatsAppVideoRouter( + participant.WhatsAppVideoSink(), + participant.WhatsAppScreenShareSink(), + participant.SetWhatsAppVideoMuted, + participant.SetWhatsAppScreenShareMuted, + ) + call.ReceiveVideo(nil) + call.OnParticipantVideoFrame(videoRouter.WriteParticipantFrame) + call.OnGroupState(videoRouter.SetGroupState) + call.OnScreenShare(videoRouter.SetScreenShare) + for _, state := range call.ScreenShares() { + videoRouter.SetScreenShare(state) + } } answeredIncoming := call.State() == meowcaller.CallPhaseRinging if call.State() == meowcaller.CallPhaseRinging { @@ -402,6 +426,71 @@ func (m *Manager) handleMatrixVideoMuteState(call *meowcaller.Call, muted bool) Msg("Sent WhatsApp local video state from LiveKit") } +func (m *Manager) handleMatrixCameraMuteState(call *meowcaller.Call, sourceRouter *matrixVideoSourceRouter, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + if sourceRouter != nil && sourceRouter.ScreenSharing() { + m.mu.Lock() + m.matrixVideoMuted[call.ID()] = muted + m.mu.Unlock() + return + } + m.handleMatrixVideoMuteState(call, muted) +} + +func (m *Manager) handleMatrixScreenShareMuteState( + call *meowcaller.Call, + participant *LiveKitParticipant, + sourceRouter *matrixVideoSourceRouter, + muted bool, +) { + if m == nil || call == nil || sourceRouter == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + active := !muted + if sourceRouter.ScreenSharing() == active { + return + } + sourceRouter.SetScreenSharing(active) + + var err error + startedVideo := false + if active { + if !call.IsSendingVideo() { + err = call.StartVideo() + startedVideo = err == nil + } + if err == nil { + err = call.StartScreenShare(nil) + if err != nil && startedVideo { + _ = call.StopVideo() + } + } + } else { + err = call.StopScreenShare() + if err == nil && m.currentMatrixVideoMuted(call.ID()) && call.IsSendingVideo() { + err = call.StopVideo() + } + } + if err != nil { + sourceRouter.SetScreenSharing(!active) + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Bool("active", active). + Msg("Failed to update WhatsApp screen-share state from LiveKit") + return + } + if participant != nil { + participant.requestRemoteVideoKeyframe() + } + m.log.Info(). + Str("call_id", call.ID()). + Bool("active", active). + Msg("Updated WhatsApp screen-share state from LiveKit") +} + func (m *Manager) handleWhatsAppAudioMuteState(callID string, muted bool) { if m == nil || callID == "" { return @@ -476,6 +565,16 @@ func (m *Manager) currentMatrixAudioMuted(callID string) bool { return muted } +func (m *Manager) currentMatrixVideoMuted(callID string) bool { + if m == nil { + return false + } + m.mu.Lock() + muted := m.matrixVideoMuted[callID] + m.mu.Unlock() + return muted +} + func localMuteStateFor(muted bool) string { if muted { return localMuteState diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go index 504716d..f229595 100644 --- a/pkg/connector/voip/video_test.go +++ b/pkg/connector/voip/video_test.go @@ -111,6 +111,31 @@ func TestLiveKitParticipantRequestsRemoteVideoKeyframe(t *testing.T) { } } +func TestLiveKitParticipantKeepsScreenShareKeyframeRequestSourceSpecific(t *testing.T) { + const ( + cameraSSRC = webrtc.SSRC(0x11111111) + screenSSRC = webrtc.SSRC(0x22222222) + ) + var gotSSRC webrtc.SSRC + participant := &LiveKitParticipant{remoteScreenActive: true} + + if participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned true before screen-share subscription") + } + participant.setRemoteVideoPLIForSource(liveKitVideoSourceCamera, func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, cameraSSRC) + if gotSSRC != 0 { + t.Fatalf("camera subscription consumed pending screen-share PLI with SSRC %#x", gotSSRC) + } + participant.setRemoteVideoPLIForSource(liveKitVideoSourceScreenShare, func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, screenSSRC) + if gotSSRC != screenSSRC { + t.Fatalf("screen-share subscription PLI SSRC = %#x, want %#x", gotSSRC, screenSSRC) + } +} + func TestManagerDefersVideoKeyframeOnlyForTrackedCall(t *testing.T) { manager := &Manager{ calls: make(map[string]*meowcaller.Call), From 86573da9eb90846ac4c99a6d6a674490ea0d453b Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 04:47:36 +0200 Subject: [PATCH 35/44] matrixrtc: bridge call reactions and hand state --- pkg/connector/client.go | 10 + pkg/connector/matrixrtc.go | 33 ++- pkg/connector/matrixrtc_outgoing.go | 56 ++-- pkg/connector/matrixrtc_reactions.go | 253 ++++++++++++++++++ pkg/connector/matrixrtc_test.go | 43 +++ pkg/connector/voip/manager.go | 66 +++++ pkg/connector/voip/matrixrtc.go | 81 ++++-- pkg/connector/voip/matrixrtc_test.go | 46 +++- pkg/connector/voip/reactions.go | 54 ++++ pkg/connector/wadb/call.go | 78 ++++-- pkg/connector/wadb/call_test.go | 31 +++ .../wadb/upgrades/00-latest-schema.sql | 6 +- .../wadb/upgrades/11-matrixrtc-reactions.sql | 5 + pkg/connector/wadb/upgrades/upgrades_test.go | 71 +++++ 14 files changed, 762 insertions(+), 71 deletions(-) create mode 100644 pkg/connector/matrixrtc_reactions.go create mode 100644 pkg/connector/voip/reactions.go create mode 100644 pkg/connector/wadb/call_test.go create mode 100644 pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql create mode 100644 pkg/connector/wadb/upgrades/upgrades_test.go diff --git a/pkg/connector/client.go b/pkg/connector/client.go index f35aa60..69c2f23 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -24,6 +24,7 @@ import ( "sync/atomic" "time" + "github.com/purpshell/meowcaller" "github.com/rs/zerolog" "go.mau.fi/util/exsync" "go.mau.fi/whatsmeow" @@ -87,6 +88,12 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. w.VOIP = voip.NewManager(w.Client, makeVOIPConfig(wa.Config.VOIP), w.UserLogin.Log.With().Str("component", "voip").Logger()) w.VOIP.SetIncomingCallHandler(w.handleIncomingVOIPCall) w.VOIP.SetCallEndHandler(w.handleVOIPCallEnded) + w.VOIP.SetCallReactionHandler(func(callID string, reaction meowcaller.CallReaction) { + go w.handleWhatsAppCallReaction(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, reaction) + }) + w.VOIP.SetHandRaiseHandler(func(callID string, state meowcaller.HandRaiseState) { + go w.handleWhatsAppHandRaise(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) + }) w.Client.SetForceActiveDeliveryReceipts(wa.Config.ForceActiveDeliveryReceipts) w.Client.InitialAutoReconnect = wa.Config.InitialAutoReconnect w.Client.UseRetryMessageStore = wa.Config.UseWhatsAppRetryStore @@ -118,6 +125,9 @@ type WhatsAppClient struct { nextResync time.Time directMediaRetries map[networkid.MessageID]*directMediaRetry directMediaLock sync.Mutex + voipHandBridgeLock sync.Mutex + voipHandRaiseLock sync.Mutex + voipHandRaises map[string]map[types.JID]bool mediaRetryLock *semaphore.Weighted offlineSyncWaiter atomic.Pointer[chan error] isNewLogin bool diff --git a/pkg/connector/matrixrtc.go b/pkg/connector/matrixrtc.go index eb2081b..2d86f5a 100644 --- a/pkg/connector/matrixrtc.go +++ b/pkg/connector/matrixrtc.go @@ -12,6 +12,7 @@ import ( "maunium.net/go/mautrix/id" "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" ) const ( @@ -102,7 +103,12 @@ func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *even if err != nil { log.Err(err).Msg("Failed to look up active MatrixRTC calls for room") return - } else if len(activeCalls) == 0 { + } + if isMatrixRTCCallControlEvent(parsed) { + wa.handleMatrixRTCCallControlEvent(ctx, parsed, activeCalls, log) + return + } + if len(activeCalls) == 0 { if !shouldStartOutboundMatrixRTCCall(evt, parsed, wa.Config.VOIP.MatrixRTC.MembershipEventCompat) { log.Debug().Msg("Ignoring MatrixRTC event without active bridged calls in the room") return @@ -142,6 +148,19 @@ func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *even callLog.Debug().Msg("WhatsApp login has no VOIP manager for MatrixRTC event") continue } + if matrixRTCMembershipEventMatchesCall(parsed, activeCall) && + activeCall.SelectedMembershipEventID != parsed.EventID { + if activeCall.SelectedHandRaiseEventID != "" { + if err = client.VOIP.SetHandRaised(activeCall.WACallID, false); err != nil { + callLog.Warn().Err(err).Msg("Failed to lower WhatsApp hand after MatrixRTC membership replacement") + } + activeCall.SelectedHandRaiseEventID = "" + } + activeCall.SelectedMembershipEventID = parsed.EventID + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + callLog.Err(err).Msg("Failed to persist replacement MatrixRTC membership event") + } + } handled++ endedCalls := client.VOIP.HandleMatrixRTCCallEvent(ctx, parsed, activeCall.WACallID) if shouldEndMatrixRTCCallFromMembership(parsed, activeCall.SelectedPublisherID) { @@ -176,6 +195,18 @@ func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *even Msg("Handled MatrixRTC event for active bridged calls") } +func matrixRTCMembershipEventMatchesCall(evt voip.MatrixRTCEvent, call *wadb.MatrixRTCCall) bool { + if call == nil || evt.EventID == "" || !voip.MatrixRTCEventHasJoinContent(evt) { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + return matrixRTCEventMatchesParticipant(evt, call.SelectedPublisherID) + default: + return false + } +} + func shouldActivateMatrixRTCCall(evt voip.MatrixRTCEvent, callState string) bool { if callState != "ringing" { return false diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index ddd39aa..eb513ab 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -42,6 +42,7 @@ func (wa *WhatsAppClient) handleIncomingVOIPCall(call *meowcaller.Call) { } func (wa *WhatsAppClient) handleVOIPCallEnded(callID, reason string) { + wa.clearWhatsAppRemoteHandRaises(callID) ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) log := wa.UserLogin.Log.With().Str("call_id", callID).Str("reason", reason).Logger() call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) @@ -142,6 +143,10 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal _ = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, wa.UserLogin.ID, call.ID(), "ended", "matrixrtc_announce_failed", err.Error(), time.Now()) return err } + record.BridgeMembershipEventID = session.MembershipEventID + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + return err + } log.Info(). Stringer("room_id", portal.MXID). Stringer("participant_mxid", intent.GetMXID()). @@ -249,22 +254,23 @@ func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal now := time.Now() deviceID := voip.MatrixRTCDeviceID(string(wa.UserLogin.ID), call.ID()) record := &wadb.MatrixRTCCall{ - UserLoginID: wa.UserLogin.ID, - WACallID: call.ID(), - RoomID: trigger.RoomID, - PortalKey: portal.PortalKey, - PeerJID: peer, - Direction: "outgoing", - MediaKind: mediaKind, - FocusType: focus.Type, - LiveKitServiceURL: focus.LiveKitServiceURL, - LiveKitRoom: trigger.RoomID.String(), - MatrixParticipantMXID: intent.GetMXID(), - MatrixSessionID: deviceID, - SelectedPublisherID: matrixRTCTriggerParticipantID(trigger), - AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, - State: "joining_livekit", - CreatedTS: now, + UserLoginID: wa.UserLogin.ID, + WACallID: call.ID(), + RoomID: trigger.RoomID, + PortalKey: portal.PortalKey, + PeerJID: peer, + Direction: "outgoing", + MediaKind: mediaKind, + FocusType: focus.Type, + LiveKitServiceURL: focus.LiveKitServiceURL, + LiveKitRoom: trigger.RoomID.String(), + MatrixParticipantMXID: intent.GetMXID(), + MatrixSessionID: deviceID, + SelectedPublisherID: matrixRTCTriggerParticipantID(trigger), + SelectedMembershipEventID: trigger.EventID, + AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, + State: "joining_livekit", + CreatedTS: now, } if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { _ = call.Hangup() @@ -283,6 +289,7 @@ func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal if err = wa.sendMatrixRTCMembership(ctx, intent, trigger.RoomID, session); err != nil { return wa.failMatrixRTCActivation(ctx, record, "matrixrtc_membership_failed", err) } + record.BridgeMembershipEventID = session.MembershipEventID if err = wa.connectOutboundMatrixRTCCall(ctx, record, trigger); err != nil { wa.UserLogin.Log.Warn(). Err(err). @@ -328,21 +335,28 @@ func (wa *WhatsAppClient) sendMatrixRTCMembership(ctx context.Context, intent br modernMessageSent := false if matrixRTCCompatAllowsModern(membershipMode) { content := voip.BuildRTCMembershipContent(*session) - if _, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCMembershipEventType(event.MessageEventType), content, matrixRTCStickyDuration); err != nil { + resp, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCMembershipEventType(event.MessageEventType), content, matrixRTCStickyDuration) + if err != nil { return err } + if resp != nil { + session.MembershipEventID = resp.EventID + } modernMessageSent = true stateKey := voip.MatrixRTCStateKey(session.UserID, session.DeviceID) - if _, err := intent.SendState(ctx, roomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: content}, now); err != nil { + stateResp, err := intent.SendState(ctx, roomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: content}, now) + if err != nil { wa.UserLogin.Log.Warn(). Err(err). Stringer("room_id", roomID). Str("state_key", stateKey). Msg("Failed to send MatrixRTC membership state event after sticky message membership") + } else if session.MembershipEventID == "" && stateResp != nil { + session.MembershipEventID = stateResp.EventID } } if matrixRTCCompatAllowsLegacy(membershipMode) { - _, err := intent.SendState(ctx, roomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: voip.BuildLegacyCallMemberContent(*session)}, now) + resp, err := intent.SendState(ctx, roomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: voip.BuildLegacyCallMemberContent(*session)}, now) if err != nil { if modernMessageSent { wa.UserLogin.Log.Warn(). @@ -353,6 +367,9 @@ func (wa *WhatsAppClient) sendMatrixRTCMembership(ctx context.Context, intent br } return err } + if session.MembershipEventID == "" && resp != nil { + session.MembershipEventID = resp.EventID + } } return nil } @@ -435,6 +452,7 @@ func (wa *WhatsAppClient) activateMatrixRTCCall(ctx context.Context, call *wadb. call.State = "joining_livekit" call.LastError = "" call.SelectedPublisherID = matrixRTCTriggerParticipantID(trigger) + call.SelectedMembershipEventID = trigger.EventID if err := wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { return err } diff --git a/pkg/connector/matrixrtc_reactions.go b/pkg/connector/matrixrtc_reactions.go new file mode 100644 index 0000000..f52c1fe --- /dev/null +++ b/pkg/connector/matrixrtc_reactions.go @@ -0,0 +1,253 @@ +package connector + +import ( + "context" + "strings" + + "github.com/purpshell/meowcaller" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/event" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func isMatrixRTCCallControlEvent(evt voip.MatrixRTCEvent) bool { + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction, voip.MatrixRTCEventKindHandRaise, voip.MatrixRTCEventKindRedaction: + return true + default: + return false + } +} + +func matrixRTCControlEventMatchesCall(evt voip.MatrixRTCEvent, call *wadb.MatrixRTCCall) bool { + if call == nil || !matrixRTCEventSenderMatchesPublisher(evt.Sender.String(), call.SelectedPublisherID) { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction, voip.MatrixRTCEventKindHandRaise: + return evt.RelatesToEventID != "" && evt.RelatesToEventID == call.SelectedMembershipEventID + case voip.MatrixRTCEventKindRedaction: + return evt.Redacts != "" && evt.Redacts == call.SelectedHandRaiseEventID + default: + return false + } +} + +func matrixRTCEventSenderMatchesPublisher(sender, publisherID string) bool { + if sender == "" || publisherID == "" { + return false + } + return publisherID == sender || strings.HasPrefix(publisherID, sender+":") +} + +func (wa *WhatsAppConnector) handleMatrixRTCCallControlEvent( + ctx context.Context, + evt voip.MatrixRTCEvent, + activeCalls []*wadb.MatrixRTCCall, + log zerolog.Logger, +) { + for _, activeCall := range activeCalls { + if !matrixRTCControlEventMatchesCall(evt, activeCall) { + continue + } + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, activeCall.UserLoginID) + if err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to resolve login for MatrixRTC call control") + continue + } + if login == nil { + continue + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + continue + } + + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction: + if evt.RelationType != event.RelReference { + continue + } + emoji, supported := voip.NormalizeWhatsAppCallReaction(evt.ReactionEmoji) + if !supported { + log.Debug().Str("emoji", evt.ReactionEmoji).Msg("Ignoring unsupported MatrixRTC call reaction") + continue + } + if err = client.VOIP.SendReaction(activeCall.WACallID, emoji); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Str("emoji", emoji).Msg("Failed to send MatrixRTC reaction to WhatsApp") + } + case voip.MatrixRTCEventKindHandRaise: + if evt.RelationType != event.RelAnnotation || evt.RelationKey != "🖐️" || evt.EventID == "" { + continue + } + if activeCall.SelectedHandRaiseEventID != "" { + continue + } + if err = client.VOIP.SetHandRaised(activeCall.WACallID, true); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to raise hand in WhatsApp call") + continue + } + activeCall.SelectedHandRaiseEventID = evt.EventID + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to persist MatrixRTC hand raise") + _ = client.VOIP.SetHandRaised(activeCall.WACallID, false) + } + case voip.MatrixRTCEventKindRedaction: + if err = client.VOIP.SetHandRaised(activeCall.WACallID, false); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to lower hand in WhatsApp call") + continue + } + activeCall.SelectedHandRaiseEventID = "" + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to persist MatrixRTC hand lowering") + _ = client.VOIP.SetHandRaised(activeCall.WACallID, true) + } + } + } +} + +func (wa *WhatsAppClient) handleWhatsAppCallReaction(ctx context.Context, callID string, reaction meowcaller.CallReaction) { + if reaction.Removed { + return + } + emoji, supported := voip.NormalizeWhatsAppCallReaction(reaction.Emoji) + if !supported { + return + } + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil || call == nil || !call.EndedTS.IsZero() || call.BridgeMembershipEventID == "" { + if err != nil { + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to load MatrixRTC call for WhatsApp reaction") + } + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + _, err = intent.SendMessage(ctx, call.RoomID, voip.ElementCallReactionEventType(), &event.Content{ + Raw: voip.BuildElementCallReactionContent(call.BridgeMembershipEventID, emoji), + }, nil) + if err != nil { + wa.UserLogin.Log.Warn().Err(err).Str("call_id", callID).Str("emoji", emoji).Msg("Failed to bridge WhatsApp call reaction to MatrixRTC") + } +} + +func (wa *WhatsAppClient) handleWhatsAppHandRaise(ctx context.Context, callID string, state meowcaller.HandRaiseState) { + if wa.isOwnWhatsAppCallParticipant(state.Participant) { + return + } + wa.voipHandBridgeLock.Lock() + defer wa.voipHandBridgeLock.Unlock() + raised, changed := wa.updateWhatsAppRemoteHandRaise(callID, state) + if !changed { + return + } + rollback := func() { + state.Raised = !state.Raised + wa.updateWhatsAppRemoteHandRaise(callID, state) + } + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil || call == nil || !call.EndedTS.IsZero() || call.BridgeMembershipEventID == "" { + rollback() + if err != nil { + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to load MatrixRTC call for WhatsApp hand state") + } + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + if raised { + if call.BridgeHandRaiseEventID != "" { + return + } + resp, sendErr := intent.SendMessage(ctx, call.RoomID, event.EventReaction, &event.Content{ + Raw: voip.BuildElementCallHandRaiseContent(call.BridgeMembershipEventID), + }, nil) + if sendErr != nil { + rollback() + wa.UserLogin.Log.Warn().Err(sendErr).Str("call_id", callID).Msg("Failed to bridge WhatsApp hand raise to MatrixRTC") + return + } + if resp != nil { + call.BridgeHandRaiseEventID = resp.EventID + } + if call.BridgeHandRaiseEventID == "" { + rollback() + return + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + rollback() + _, _ = intent.SendMessage(ctx, call.RoomID, event.EventRedaction, &event.Content{ + Parsed: &event.RedactionEventContent{Redacts: call.BridgeHandRaiseEventID}, + }, nil) + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to persist bridged WhatsApp hand raise") + } + return + } else { + if call.BridgeHandRaiseEventID == "" { + return + } + handRaiseEventID := call.BridgeHandRaiseEventID + call.BridgeHandRaiseEventID = "" + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + rollback() + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to persist bridged WhatsApp hand lowering") + return + } + _, sendErr := intent.SendMessage(ctx, call.RoomID, event.EventRedaction, &event.Content{ + Parsed: &event.RedactionEventContent{Redacts: handRaiseEventID}, + }, nil) + if sendErr != nil { + rollback() + call.BridgeHandRaiseEventID = handRaiseEventID + _ = wa.Main.DB.MatrixRTCCall.Put(ctx, call) + wa.UserLogin.Log.Warn().Err(sendErr).Str("call_id", callID).Msg("Failed to bridge WhatsApp hand lowering to MatrixRTC") + return + } + } +} + +func (wa *WhatsAppClient) updateWhatsAppRemoteHandRaise(callID string, state meowcaller.HandRaiseState) (raised, changed bool) { + if callID == "" || state.Participant.IsEmpty() { + return false, false + } + participant := state.Participant.ToNonAD() + wa.voipHandRaiseLock.Lock() + defer wa.voipHandRaiseLock.Unlock() + if wa.voipHandRaises == nil { + wa.voipHandRaises = make(map[string]map[types.JID]bool) + } + hands := wa.voipHandRaises[callID] + if hands == nil { + hands = make(map[types.JID]bool) + wa.voipHandRaises[callID] = hands + } + wasRaised := len(hands) > 0 + if state.Raised { + hands[participant] = true + } else { + delete(hands, participant) + } + raised = len(hands) > 0 + if !raised { + delete(wa.voipHandRaises, callID) + } + return raised, wasRaised != raised +} + +func (wa *WhatsAppClient) clearWhatsAppRemoteHandRaises(callID string) { + wa.voipHandBridgeLock.Lock() + defer wa.voipHandBridgeLock.Unlock() + wa.voipHandRaiseLock.Lock() + delete(wa.voipHandRaises, callID) + wa.voipHandRaiseLock.Unlock() +} + +func (wa *WhatsAppClient) isOwnWhatsAppCallParticipant(participant types.JID) bool { + if participant.IsEmpty() || wa.GetStore() == nil { + return false + } + participant = participant.ToNonAD() + return participant == wa.GetStore().GetLID().ToNonAD() || + participant == wa.GetStore().GetJID().ToNonAD() +} diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index c9a9405..baa450b 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/purpshell/meowcaller" "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -187,6 +188,48 @@ func TestMatrixRTCPortalSupportsWhatsAppGroupCalls(t *testing.T) { } } +func TestMatrixRTCControlEventMatchesSelectedMembership(t *testing.T) { + call := &wadb.MatrixRTCCall{ + SelectedPublisherID: "@alice:example.com:DEVICE", + SelectedMembershipEventID: "$membership", + } + reaction := voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindCallReaction, + Sender: "@alice:example.com", + RelatesToEventID: "$membership", + } + if !matrixRTCControlEventMatchesCall(reaction, call) { + t.Fatal("reaction did not match its selected MatrixRTC membership") + } + reaction.Sender = "@mallory:example.com" + if matrixRTCControlEventMatchesCall(reaction, call) { + t.Fatal("reaction from another Matrix user matched the selected membership") + } +} + +func TestWhatsAppRemoteHandRaisesAreAggregated(t *testing.T) { + wa := &WhatsAppClient{} + alice := types.NewJID("111", types.DefaultUserServer) + bob := types.NewJID("222", types.DefaultUserServer) + + raised, changed := wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: alice, Raised: true}) + if !raised || !changed { + t.Fatalf("first raise = %t, %t, want true, true", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: bob, Raised: true}) + if !raised || changed { + t.Fatalf("second raise = %t, %t, want true, false", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: alice, Raised: false}) + if !raised || changed { + t.Fatalf("first lower = %t, %t, want true, false", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: bob, Raised: false}) + if raised || !changed { + t.Fatalf("last lower = %t, %t, want false, true", raised, changed) + } +} + func TestReserveMatrixRTCOutboundStartSuppressesDuplicates(t *testing.T) { wa := &WhatsAppConnector{} if !wa.reserveMatrixRTCOutboundStart("!room:example.com") { diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index d9bf0b6..7f3cfb2 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -53,6 +53,8 @@ type Manager struct { videoKeyframePending map[string]bool incomingCallNotify func(*meowcaller.Call) callEndNotify func(callID, reason string) + callReactionNotify func(callID string, reaction meowcaller.CallReaction) + handRaiseNotify func(callID string, state meowcaller.HandRaiseState) } func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Manager { @@ -122,6 +124,54 @@ func (m *Manager) SetCallEndHandler(handler func(callID, reason string)) { m.mu.Unlock() } +func (m *Manager) SetCallReactionHandler(handler func(callID string, reaction meowcaller.CallReaction)) { + if m == nil { + return + } + m.mu.Lock() + m.callReactionNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetHandRaiseHandler(handler func(callID string, state meowcaller.HandRaiseState)) { + if m == nil { + return + } + m.mu.Lock() + m.handRaiseNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SendReaction(callID, emoji string) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return ErrCallNotFound + } + normalized, ok := NormalizeWhatsAppCallReaction(emoji) + if !ok { + return fmt.Errorf("unsupported WhatsApp call reaction %q", emoji) + } + return call.SendReaction(normalized) +} + +func (m *Manager) SetHandRaised(callID string, raised bool) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return ErrCallNotFound + } + return call.SetHandRaised(raised) +} + func (m *Manager) Dial(ctx context.Context, target string, video ...bool) (*meowcaller.Call, error) { if !m.Enabled() { return nil, ErrNotEnabled @@ -746,6 +796,22 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { call.OnVideoState(func(state meowcaller.VideoState) { m.handleWhatsAppVideoState(call.ID(), state) }) + call.OnReaction(func(reaction meowcaller.CallReaction) { + m.mu.Lock() + handler := m.callReactionNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), reaction) + } + }) + call.OnHandRaise(func(state meowcaller.HandRaiseState) { + m.mu.Lock() + handler := m.handRaiseNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), state) + } + }) } func (m *Manager) requestLiveKitVideoKeyframe(callID string) { diff --git a/pkg/connector/voip/matrixrtc.go b/pkg/connector/voip/matrixrtc.go index 1e0e7f9..1f6b154 100644 --- a/pkg/connector/voip/matrixrtc.go +++ b/pkg/connector/voip/matrixrtc.go @@ -11,12 +11,13 @@ import ( ) const ( - EventTypeGroupCall = "org.matrix.msc3401.call" - EventTypeGroupCallMember = "org.matrix.msc3401.call.member" - EventTypeRTCMembership = "org.matrix.msc4143.rtc.member" - EventTypeRTCNotification = "org.matrix.msc4075.rtc.notification" - EventTypeCallNotify = "org.matrix.msc4075.call.notify" - EventTypeRTCDecline = "org.matrix.msc4310.rtc.decline" + EventTypeGroupCall = "org.matrix.msc3401.call" + EventTypeGroupCallMember = "org.matrix.msc3401.call.member" + EventTypeRTCMembership = "org.matrix.msc4143.rtc.member" + EventTypeRTCNotification = "org.matrix.msc4075.rtc.notification" + EventTypeCallNotify = "org.matrix.msc4075.call.notify" + EventTypeRTCDecline = "org.matrix.msc4310.rtc.decline" + EventTypeElementCallReaction = "io.element.call.reaction" MatrixRTCApplicationCall = "m.call" MatrixRTCDefaultSlotID = "m.call#ROOM" @@ -31,6 +32,9 @@ var supportedMatrixRTCEventTypes = []event.Type{ {Type: EventTypeRTCNotification, Class: event.MessageEventType}, {Type: EventTypeCallNotify, Class: event.MessageEventType}, {Type: EventTypeRTCDecline, Class: event.MessageEventType}, + ElementCallReactionEventType(), + event.EventReaction, + event.EventRedaction, } type MatrixRTCEventKind string @@ -43,21 +47,31 @@ const ( MatrixRTCEventKindRTCNotification MatrixRTCEventKind = "rtc_notification" MatrixRTCEventKindLegacyCallNotify MatrixRTCEventKind = "legacy_call_notify" MatrixRTCEventKindRTCDecline MatrixRTCEventKind = "rtc_decline" + MatrixRTCEventKindCallReaction MatrixRTCEventKind = "call_reaction" + MatrixRTCEventKindHandRaise MatrixRTCEventKind = "hand_raise" + MatrixRTCEventKindRedaction MatrixRTCEventKind = "redaction" ) type MatrixRTCEvent struct { - Type event.Type - Kind MatrixRTCEventKind - RoomID id.RoomID - Sender id.UserID - StateKey string - CallID string - DeviceID string - SessionID string - Intent string - LifetimeMS int - FociPreferred []Focus - Raw map[string]any + Type event.Type + Kind MatrixRTCEventKind + RoomID id.RoomID + Sender id.UserID + StateKey string + CallID string + DeviceID string + SessionID string + Intent string + LifetimeMS int + FociPreferred []Focus + Raw map[string]any + EventID id.EventID + RelatesToEventID id.EventID + RelationType event.RelationType + RelationKey string + ReactionEmoji string + ReactionName string + Redacts id.EventID } type MatrixRTCSession struct { @@ -71,6 +85,7 @@ type MatrixRTCSession struct { Expires time.Duration StickyKey string NotificationEventID id.EventID + MembershipEventID id.EventID } func SupportedMatrixRTCEventTypes() []event.Type { @@ -91,6 +106,12 @@ func ClassifyMatrixRTCEventType(evtType event.Type) MatrixRTCEventKind { return MatrixRTCEventKindLegacyCallNotify case EventTypeRTCDecline: return MatrixRTCEventKindRTCDecline + case EventTypeElementCallReaction: + return MatrixRTCEventKindCallReaction + case event.EventReaction.Type: + return MatrixRTCEventKindHandRaise + case event.EventRedaction.Type: + return MatrixRTCEventKindRedaction default: return MatrixRTCEventKindUnknown } @@ -106,16 +127,20 @@ func ParseMatrixRTCEvent(evt *event.Event) (MatrixRTCEvent, bool) { } raw := rawMatrixRTCContent(evt) parsed := MatrixRTCEvent{ - Type: evt.Type, - Kind: kind, - RoomID: evt.RoomID, - Sender: evt.Sender, - Raw: raw, + Type: evt.Type, + Kind: kind, + RoomID: evt.RoomID, + Sender: evt.Sender, + EventID: evt.ID, + Raw: raw, } if evt.StateKey != nil { parsed.StateKey = *evt.StateKey } fillMatrixRTCFields(&parsed, raw) + if evt.Redacts != "" { + parsed.Redacts = evt.Redacts + } return parsed, true } @@ -147,6 +172,16 @@ func fillMatrixRTCFields(parsed *MatrixRTCEvent, raw map[string]any) { parsed.SessionID = firstString(raw, "session_id", "m.session_id", "sessionId", "sessionID") parsed.Intent = firstString(raw, "intent", "m.call.intent", "call_intent") parsed.LifetimeMS = firstInt(raw, "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms") + if relatesTo, ok := raw["m.relates_to"].(map[string]any); ok { + parsed.RelatesToEventID = id.EventID(firstString(relatesTo, "event_id")) + parsed.RelationType = event.RelationType(firstString(relatesTo, "rel_type")) + parsed.RelationKey = firstString(relatesTo, "key") + } + parsed.ReactionEmoji = firstString(raw, "emoji") + parsed.ReactionName = firstString(raw, "name") + if parsed.Redacts == "" { + parsed.Redacts = id.EventID(firstString(raw, "redacts")) + } forEachObject(raw["application"], func(application map[string]any) { if parsed.Intent == "" { parsed.Intent = firstString(application, "intent", "m.call.intent", "call_intent") diff --git a/pkg/connector/voip/matrixrtc_test.go b/pkg/connector/voip/matrixrtc_test.go index 42e22c7..1c605b0 100644 --- a/pkg/connector/voip/matrixrtc_test.go +++ b/pkg/connector/voip/matrixrtc_test.go @@ -10,8 +10,8 @@ import ( func TestSupportedMatrixRTCEventTypesHaveExplicitClasses(t *testing.T) { types := SupportedMatrixRTCEventTypes() - if len(types) != 7 { - t.Fatalf("SupportedMatrixRTCEventTypes returned %d types, want 7", len(types)) + if len(types) != 10 { + t.Fatalf("SupportedMatrixRTCEventTypes returned %d types, want 10", len(types)) } for _, evtType := range types { switch evtType.Type { @@ -23,7 +23,8 @@ func TestSupportedMatrixRTCEventTypesHaveExplicitClasses(t *testing.T) { if evtType.Class != event.StateEventType && evtType.Class != event.MessageEventType { t.Fatalf("%s class = %v, want state or message", evtType.Type, evtType.Class) } - case EventTypeRTCNotification, EventTypeCallNotify, EventTypeRTCDecline: + case EventTypeRTCNotification, EventTypeCallNotify, EventTypeRTCDecline, + EventTypeElementCallReaction, event.EventReaction.Type, event.EventRedaction.Type: if evtType.Class != event.MessageEventType { t.Fatalf("%s class = %v, want message", evtType.Type, evtType.Class) } @@ -33,6 +34,45 @@ func TestSupportedMatrixRTCEventTypesHaveExplicitClasses(t *testing.T) { } } +func TestParseElementCallReactionEvent(t *testing.T) { + evt := &event.Event{ + ID: id.EventID("$reaction"), + Type: ElementCallReactionEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + Content: event.Content{Raw: map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": "m.reference", + "event_id": "$membership", + }, + "emoji": "❤️", + "name": "generic", + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatal("ParseMatrixRTCEvent did not recognize Element Call reaction") + } + if parsed.Kind != MatrixRTCEventKindCallReaction || + parsed.EventID != "$reaction" || + parsed.RelatesToEventID != "$membership" || + parsed.ReactionEmoji != "❤️" { + t.Fatalf("unexpected parsed reaction: %+v", parsed) + } +} + +func TestSupportedWhatsAppCallReactions(t *testing.T) { + for _, emoji := range []string{"👍", "❤️", "😂", "😮", "😢", "🙏"} { + normalized, ok := NormalizeWhatsAppCallReaction(emoji) + if !ok || normalized != emoji { + t.Fatalf("NormalizeWhatsAppCallReaction(%q) = %q, %t", emoji, normalized, ok) + } + } + if normalized, ok := NormalizeWhatsAppCallReaction("🎉"); ok || normalized != "" { + t.Fatalf("unsupported reaction normalized to %q, %t", normalized, ok) + } +} + func TestParseMatrixRTCDeclineEvent(t *testing.T) { evt := &event.Event{ Type: event.Type{Type: EventTypeRTCDecline, Class: event.MessageEventType}, diff --git a/pkg/connector/voip/reactions.go b/pkg/connector/voip/reactions.go new file mode 100644 index 0000000..8e7afd7 --- /dev/null +++ b/pkg/connector/voip/reactions.go @@ -0,0 +1,54 @@ +package voip + +import ( + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +var supportedWhatsAppCallReactions = map[string]string{ + "👍": "thumbsup", + "❤️": "generic", + "😂": "generic", + "😮": "generic", + "😢": "generic", + "🙏": "generic", +} + +func ElementCallReactionEventType() event.Type { + return event.Type{Type: EventTypeElementCallReaction, Class: event.MessageEventType} +} + +func NormalizeWhatsAppCallReaction(emoji string) (string, bool) { + if _, ok := supportedWhatsAppCallReactions[emoji]; ok { + return emoji, true + } + return "", false +} + +func ElementCallReactionName(emoji string) string { + if name, ok := supportedWhatsAppCallReactions[emoji]; ok { + return name + } + return "generic" +} + +func BuildElementCallReactionContent(membershipEventID id.EventID, emoji string) map[string]any { + return map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": string(event.RelReference), + "event_id": membershipEventID, + }, + "emoji": emoji, + "name": ElementCallReactionName(emoji), + } +} + +func BuildElementCallHandRaiseContent(membershipEventID id.EventID) map[string]any { + return map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": string(event.RelAnnotation), + "event_id": membershipEventID, + "key": "🖐️", + }, + } +} diff --git a/pkg/connector/wadb/call.go b/pkg/connector/wadb/call.go index 804abb5..6c25f3a 100644 --- a/pkg/connector/wadb/call.go +++ b/pkg/connector/wadb/call.go @@ -17,28 +17,32 @@ type MatrixRTCCallQuery struct { } type MatrixRTCCall struct { - BridgeID networkid.BridgeID - UserLoginID networkid.UserLoginID - WACallID string - RoomID id.RoomID - PortalKey networkid.PortalKey - PeerJID types.JID - Direction string - MediaKind string - FocusType string - LiveKitServiceURL string - LiveKitRoom string - MatrixParticipantMXID id.UserID - MatrixSessionID string - SelectedPublisherID string - AudioPolicy string - State string - CreatedTS time.Time - JoinedTS time.Time - AnsweredTS time.Time - EndedTS time.Time - EndReason string - LastError string + BridgeID networkid.BridgeID + UserLoginID networkid.UserLoginID + WACallID string + RoomID id.RoomID + PortalKey networkid.PortalKey + PeerJID types.JID + Direction string + MediaKind string + FocusType string + LiveKitServiceURL string + LiveKitRoom string + MatrixParticipantMXID id.UserID + MatrixSessionID string + SelectedPublisherID string + BridgeMembershipEventID id.EventID + SelectedMembershipEventID id.EventID + BridgeHandRaiseEventID id.EventID + SelectedHandRaiseEventID id.EventID + AudioPolicy string + State string + CreatedTS time.Time + JoinedTS time.Time + AnsweredTS time.Time + EndedTS time.Time + EndReason string + LastError string } const ( @@ -47,10 +51,12 @@ const ( bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, direction, media_kind, focus_type, livekit_service_url, livekit_room, matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, end_reason, last_error ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27) ON CONFLICT (bridge_id, user_login_id, wa_call_id) DO UPDATE SET room_id=excluded.room_id, portal_id=excluded.portal_id, @@ -64,6 +70,10 @@ const ( matrix_participant_mxid=excluded.matrix_participant_mxid, matrix_session_id=excluded.matrix_session_id, selected_publisher_id=excluded.selected_publisher_id, + bridge_membership_event_id=excluded.bridge_membership_event_id, + selected_membership_event_id=excluded.selected_membership_event_id, + bridge_hand_raise_event_id=excluded.bridge_hand_raise_event_id, + selected_hand_raise_event_id=excluded.selected_hand_raise_event_id, audio_policy=excluded.audio_policy, state=excluded.state, joined_ts=excluded.joined_ts, @@ -77,6 +87,8 @@ const ( bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, direction, media_kind, focus_type, livekit_service_url, livekit_room, matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, end_reason, last_error FROM whatsapp_matrixrtc_call @@ -87,6 +99,8 @@ const ( bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, direction, media_kind, focus_type, livekit_service_url, livekit_room, matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, end_reason, last_error FROM whatsapp_matrixrtc_call @@ -97,6 +111,8 @@ const ( bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, direction, media_kind, focus_type, livekit_service_url, livekit_room, matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, end_reason, last_error FROM whatsapp_matrixrtc_call @@ -139,7 +155,9 @@ func (cq *MatrixRTCCallQuery) Delete(ctx context.Context, loginID networkid.User } func (call *MatrixRTCCall) Scan(row dbutil.Scannable) (*MatrixRTCCall, error) { - var liveKitRoom, participantMXID, matrixSessionID, selectedPublisherID, endReason, lastError sql.NullString + var liveKitRoom, participantMXID, matrixSessionID, selectedPublisherID sql.NullString + var bridgeMembershipEventID, selectedMembershipEventID, bridgeHandRaiseEventID, selectedHandRaiseEventID sql.NullString + var endReason, lastError sql.NullString var joinedTS, answeredTS, endedTS sql.NullInt64 var createdTS int64 err := row.Scan( @@ -158,6 +176,10 @@ func (call *MatrixRTCCall) Scan(row dbutil.Scannable) (*MatrixRTCCall, error) { &participantMXID, &matrixSessionID, &selectedPublisherID, + &bridgeMembershipEventID, + &selectedMembershipEventID, + &bridgeHandRaiseEventID, + &selectedHandRaiseEventID, &call.AudioPolicy, &call.State, &createdTS, @@ -178,6 +200,10 @@ func (call *MatrixRTCCall) Scan(row dbutil.Scannable) (*MatrixRTCCall, error) { call.MatrixParticipantMXID = id.UserID(participantMXID.String) call.MatrixSessionID = matrixSessionID.String call.SelectedPublisherID = selectedPublisherID.String + call.BridgeMembershipEventID = id.EventID(bridgeMembershipEventID.String) + call.SelectedMembershipEventID = id.EventID(selectedMembershipEventID.String) + call.BridgeHandRaiseEventID = id.EventID(bridgeHandRaiseEventID.String) + call.SelectedHandRaiseEventID = id.EventID(selectedHandRaiseEventID.String) call.EndReason = endReason.String call.LastError = lastError.String return call, nil @@ -200,6 +226,10 @@ func (call *MatrixRTCCall) sqlVariables() []any { nullString(string(call.MatrixParticipantMXID)), nullString(call.MatrixSessionID), nullString(call.SelectedPublisherID), + nullString(string(call.BridgeMembershipEventID)), + nullString(string(call.SelectedMembershipEventID)), + nullString(string(call.BridgeHandRaiseEventID)), + nullString(string(call.SelectedHandRaiseEventID)), call.AudioPolicy, call.State, nullableUnix(call.CreatedTS), diff --git a/pkg/connector/wadb/call_test.go b/pkg/connector/wadb/call_test.go new file mode 100644 index 0000000..1254d04 --- /dev/null +++ b/pkg/connector/wadb/call_test.go @@ -0,0 +1,31 @@ +package wadb + +import ( + "testing" + + "maunium.net/go/mautrix/id" +) + +func TestMatrixRTCCallSQLVariablesIncludeReactionEventIDs(t *testing.T) { + call := &MatrixRTCCall{ + BridgeMembershipEventID: id.EventID("$bridge-member"), + SelectedMembershipEventID: id.EventID("$selected-member"), + BridgeHandRaiseEventID: id.EventID("$bridge-hand"), + SelectedHandRaiseEventID: id.EventID("$selected-hand"), + } + variables := call.sqlVariables() + if len(variables) != 27 { + t.Fatalf("MatrixRTCCall.sqlVariables returned %d values, want 27", len(variables)) + } + for index, want := range map[int]string{ + 15: "$bridge-member", + 16: "$selected-member", + 17: "$bridge-hand", + 18: "$selected-hand", + } { + value, ok := variables[index].(*string) + if !ok || value == nil || *value != want { + t.Fatalf("SQL variable %d = %#v, want %q", index, variables[index], want) + } + } +} diff --git a/pkg/connector/wadb/upgrades/00-latest-schema.sql b/pkg/connector/wadb/upgrades/00-latest-schema.sql index ba6ada4..648091c 100644 --- a/pkg/connector/wadb/upgrades/00-latest-schema.sql +++ b/pkg/connector/wadb/upgrades/00-latest-schema.sql @@ -1,4 +1,4 @@ --- v0 -> v10 (compatible with v3+): Latest revision +-- v0 -> v11 (compatible with v3+): Latest revision CREATE TABLE whatsapp_poll_option_id ( bridge_id TEXT NOT NULL, @@ -115,6 +115,10 @@ CREATE TABLE whatsapp_matrixrtc_call ( matrix_participant_mxid TEXT, matrix_session_id TEXT, selected_publisher_id TEXT, + bridge_membership_event_id TEXT, + selected_membership_event_id TEXT, + bridge_hand_raise_event_id TEXT, + selected_hand_raise_event_id TEXT, audio_policy TEXT NOT NULL, state TEXT NOT NULL, created_ts BIGINT NOT NULL, diff --git a/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql b/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql new file mode 100644 index 0000000..e91fa8c --- /dev/null +++ b/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql @@ -0,0 +1,5 @@ +-- v11 (compatible with v3+): Persist MatrixRTC reaction relation targets +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN bridge_membership_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN selected_membership_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN bridge_hand_raise_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN selected_hand_raise_event_id TEXT; diff --git a/pkg/connector/wadb/upgrades/upgrades_test.go b/pkg/connector/wadb/upgrades/upgrades_test.go new file mode 100644 index 0000000..1f04c00 --- /dev/null +++ b/pkg/connector/wadb/upgrades/upgrades_test.go @@ -0,0 +1,71 @@ +package upgrades + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +func TestMatrixRTCReactionUpgradeSQLite(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, statement := range []string{ + `CREATE TABLE user_login ( + bridge_id TEXT NOT NULL, + id TEXT NOT NULL, + PRIMARY KEY (bridge_id, id) + )`, + `CREATE TABLE portal ( + bridge_id TEXT NOT NULL, + id TEXT NOT NULL, + receiver TEXT NOT NULL, + PRIMARY KEY (bridge_id, id, receiver) + )`, + } { + if _, err = db.ExecContext(context.Background(), statement); err != nil { + t.Fatal(err) + } + } + + for _, name := range []string{"10-matrixrtc-call.sql", "11-matrixrtc-reactions.sql"} { + script, readErr := rawUpgrades.ReadFile(name) + if readErr != nil { + t.Fatal(readErr) + } + if _, err = db.ExecContext(context.Background(), string(script)); err != nil { + t.Fatalf("%s failed: %v", name, err) + } + } + + rows, err := db.QueryContext(context.Background(), "PRAGMA table_info(whatsapp_matrixrtc_call)") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var cid, notNull, primaryKey int + var name, dataType string + var defaultValue any + if err = rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &primaryKey); err != nil { + t.Fatal(err) + } + columns[name] = true + } + for _, name := range []string{ + "bridge_membership_event_id", + "selected_membership_event_id", + "bridge_hand_raise_event_id", + "selected_hand_raise_event_id", + } { + if !columns[name] { + t.Fatalf("upgraded MatrixRTC call table is missing %s", name) + } + } +} From 84b08c25192148489dee76dac15b6d0f74dfdae2 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 04:55:21 +0200 Subject: [PATCH 36/44] commands: add group call participant controls --- pkg/connector/commands_voip.go | 175 ++++++++++++++++++++++++++++ pkg/connector/commands_voip_test.go | 75 ++++++++++++ pkg/connector/connector.go | 1 + pkg/connector/voip/manager.go | 38 ++++++ 4 files changed, 289 insertions(+) create mode 100644 pkg/connector/commands_voip.go create mode 100644 pkg/connector/commands_voip_test.go diff --git a/pkg/connector/commands_voip.go b/pkg/connector/commands_voip.go new file mode 100644 index 0000000..d2ceb67 --- /dev/null +++ b/pkg/connector/commands_voip.go @@ -0,0 +1,175 @@ +package connector + +import ( + "errors" + "fmt" + "slices" + "strings" + + "github.com/purpshell/meowcaller" + "maunium.net/go/mautrix/bridgev2/commands" + "maunium.net/go/mautrix/bridgev2/networkid" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +var HelpSectionCalls = commands.HelpSection{Name: "Calls", Order: 27} + +var cmdCallParticipants = &commands.FullHandler{ + Func: fnCallParticipants, + Name: "call-participants", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "List the WhatsApp participants in the active call.", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallAdd = &commands.FullHandler{ + Func: fnCallAdd, + Name: "call-add", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Invite a WhatsApp user to the active call.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallRing = &commands.FullHandler{ + Func: fnCallRing, + Name: "call-ring", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Ring a non-connected WhatsApp participant already in the active call.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +func fnCallParticipants(ce *commands.Event) { + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + state, ok, err := client.VOIP.GroupState(call.WACallID) + if err != nil { + ce.Reply("Failed to read the active call roster: %v", err) + return + } + if !ok { + ce.Reply("WhatsApp has not advertised a group roster for this call yet.") + return + } + ce.Reply(formatGroupCallRoster(state)) +} + +func fnCallAdd(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.AddParticipant(ce.Ctx, call.WACallID, target); err != nil { + ce.Reply("Failed to invite the participant: %v", err) + return + } + ce.Reply("Invited `%s` to the active WhatsApp call.", target) +} + +func fnCallRing(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.RingParticipant(ce.Ctx, call.WACallID, target); err != nil { + ce.Reply("Failed to ring the participant: %v", err) + return + } + ce.Reply("Rang `%s` in the active WhatsApp call.", target) +} + +func callTargetArg(ce *commands.Event) (string, bool) { + if len(ce.Args) != 1 { + ce.Reply("Usage: `$cmdprefix %s `", ce.Command) + return "", false + } + return strings.TrimSpace(ce.Args[0]), true +} + +func activePortalCall(ce *commands.Event) (*WhatsAppClient, *wadb.MatrixRTCCall, error) { + if ce.Portal == nil { + return nil, nil, errors.New("this command can only be used in a portal room") + } + login := ce.Bridge.GetCachedUserLoginByID(ce.Portal.Receiver) + if login == nil { + return nil, nil, errors.New("the WhatsApp login for this portal is not available") + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || !client.IsLoggedIn() { + return nil, nil, errors.New("the WhatsApp login for this portal is not connected") + } + calls, err := client.Main.DB.MatrixRTCCall.GetActiveInRoom(ce.Ctx, ce.Portal.MXID) + if err != nil { + return nil, nil, fmt.Errorf("query active calls: %w", err) + } + call, err := selectActiveCallForLogin(calls, login.ID) + if err != nil { + return nil, nil, err + } + return client, call, nil +} + +func selectActiveCallForLogin(calls []*wadb.MatrixRTCCall, loginID networkid.UserLoginID) (*wadb.MatrixRTCCall, error) { + var selected *wadb.MatrixRTCCall + for _, call := range calls { + if call == nil || call.UserLoginID != loginID { + continue + } + if selected != nil { + return nil, errors.New("multiple active calls are tracked in this room") + } + selected = call + } + if selected == nil { + return nil, errors.New("there is no active call in this room") + } + return selected, nil +} + +func formatGroupCallRoster(state meowcaller.GroupCallState) string { + participants := slices.Clone(state.Participants) + slices.SortFunc(participants, func(a, b meowcaller.GroupCallParticipant) int { + return strings.Compare(a.JID.String(), b.JID.String()) + }) + lines := make([]string, 0, len(participants)+1) + lines = append(lines, fmt.Sprintf("**WhatsApp call participants (transaction %d):**", state.TransactionID)) + for _, participant := range participants { + identity := participant.JID + if !participant.PN.IsEmpty() { + identity = participant.PN + } + detail := fmt.Sprintf("%s; %d device(s)", participant.State, len(participant.Devices)) + if participant.HandRaised { + detail += "; hand raised" + } + lines = append(lines, fmt.Sprintf("- `%s`: %s", identity, detail)) + } + if state.RekeyRequested { + lines = append(lines, "- WhatsApp requested a group media rekey.") + } + return strings.Join(lines, "\n") +} diff --git a/pkg/connector/commands_voip_test.go b/pkg/connector/commands_voip_test.go new file mode 100644 index 0000000..7b70848 --- /dev/null +++ b/pkg/connector/commands_voip_test.go @@ -0,0 +1,75 @@ +package connector + +import ( + "strings" + "testing" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/bridgev2/networkid" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func TestSelectActiveCallForLogin(t *testing.T) { + alice := networkid.UserLoginID("alice") + bob := networkid.UserLoginID("bob") + calls := []*wadb.MatrixRTCCall{ + {UserLoginID: bob, WACallID: "bob-call"}, + {UserLoginID: alice, WACallID: "alice-call"}, + } + call, err := selectActiveCallForLogin(calls, alice) + if err != nil { + t.Fatalf("selectActiveCallForLogin returned error: %v", err) + } + if call.WACallID != "alice-call" { + t.Fatalf("selected call = %q, want alice-call", call.WACallID) + } +} + +func TestSelectActiveCallForLoginRejectsMissingAndAmbiguousCalls(t *testing.T) { + loginID := networkid.UserLoginID("alice") + if _, err := selectActiveCallForLogin(nil, loginID); err == nil { + t.Fatal("selectActiveCallForLogin accepted an empty call list") + } + calls := []*wadb.MatrixRTCCall{ + {UserLoginID: loginID, WACallID: "first"}, + {UserLoginID: loginID, WACallID: "second"}, + } + if _, err := selectActiveCallForLogin(calls, loginID); err == nil { + t.Fatal("selectActiveCallForLogin accepted multiple calls for one login") + } +} + +func TestFormatGroupCallRoster(t *testing.T) { + state := meowcaller.GroupCallState{ + TransactionID: 42, + RekeyRequested: true, + Participants: []meowcaller.GroupCallParticipant{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + State: "connected", + Devices: []meowcaller.GroupCallDevice{ + {JID: types.NewJID("222", types.HiddenUserServer)}, + }, + HandRaised: true, + }, + { + JID: types.NewJID("111", types.HiddenUserServer), + State: "ringing", + }, + }, + } + got := formatGroupCallRoster(state) + for _, want := range []string{ + "transaction 42", + "`111@lid`: ringing; 0 device(s)", + "`15550000002@s.whatsapp.net`: connected; 1 device(s); hand raised", + "requested a group media rekey", + } { + if !strings.Contains(got, want) { + t.Errorf("formatted roster missing %q:\n%s", want, got) + } + } +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 1ca1340..a9bb517 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -116,6 +116,7 @@ func (wa *WhatsAppConnector) Init(bridge *bridgev2.Bridge) { wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) wa.Bridge.Commands.(*commands.Processor).AddHandlers( cmdAccept, cmdSync, cmdInviteLink, cmdResolveLink, cmdJoin, + cmdCallParticipants, cmdCallAdd, cmdCallRing, ) wa.mediaEditCache = make(MediaEditCache) wa.initMatrixRTCEventHooks() diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 7f3cfb2..86036a7 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -172,6 +172,44 @@ func (m *Manager) SetHandRaised(callID string, raised bool) error { return call.SetHandRaised(raised) } +func (m *Manager) GroupState(callID string) (meowcaller.GroupCallState, bool, error) { + call, err := m.activeCall(callID) + if err != nil { + return meowcaller.GroupCallState{}, false, err + } + state, ok := call.GroupState() + return state, ok, nil +} + +func (m *Manager) AddParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.AddParticipant(ctx, target) +} + +func (m *Manager) RingParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.RingParticipant(ctx, target) +} + +func (m *Manager) activeCall(callID string) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return nil, ErrCallNotFound + } + return call, nil +} + func (m *Manager) Dial(ctx context.Context, target string, video ...bool) (*meowcaller.Call, error) { if !m.Enabled() { return nil, ErrNotEnabled From aae11aba360f5ba31b4fcc59ae64c2a2d9920b7a Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 04:59:42 +0200 Subject: [PATCH 37/44] matrixrtc: bridge WhatsApp call links --- pkg/connector/commands_voip.go | 334 +++++++++++++++++++++++++++- pkg/connector/commands_voip_test.go | 69 ++++++ pkg/connector/connector.go | 2 + pkg/connector/matrixrtc_outgoing.go | 57 ++++- pkg/connector/voip/manager.go | 62 ++++++ 5 files changed, 513 insertions(+), 11 deletions(-) diff --git a/pkg/connector/commands_voip.go b/pkg/connector/commands_voip.go index d2ceb67..685a804 100644 --- a/pkg/connector/commands_voip.go +++ b/pkg/connector/commands_voip.go @@ -50,6 +50,87 @@ var cmdCallRing = &commands.FullHandler{ RequiresPortal: true, } +var cmdCallLinkCreate = &commands.FullHandler{ + Func: fnCallLinkCreate, + Name: "call-link-create", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Create a reusable WhatsApp call link.", + Args: "[audio|video]", + }, + RequiresLogin: true, +} + +var cmdCallLinkPreview = &commands.FullHandler{ + Func: fnCallLinkPreview, + Name: "call-link-preview", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Preview a WhatsApp call link without joining it.", + Args: " [audio|video]", + }, + RequiresLogin: true, +} + +var cmdCallLinkJoin = &commands.FullHandler{ + Func: fnCallLinkJoin, + Name: "call-link-join", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Join a WhatsApp call link and ring it into the current Matrix room.", + Args: " [audio|video]", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallWaiting = &commands.FullHandler{ + Func: fnCallWaiting, + Name: "call-waiting", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Show the waiting room for the active WhatsApp call link.", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallApproval = &commands.FullHandler{ + Func: fnCallApproval, + Name: "call-approval", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Enable or disable approval for the active WhatsApp call link.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallAdmit = &commands.FullHandler{ + Func: fnCallAdmit, + Name: "call-admit", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Admit a user from the active WhatsApp call link waiting room.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallDeny = &commands.FullHandler{ + Func: fnCallDeny, + Name: "call-deny", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Deny a user from the active WhatsApp call link waiting room.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + func fnCallParticipants(ce *commands.Event) { client, call, err := activePortalCall(ce) if err != nil { @@ -102,6 +183,148 @@ func fnCallRing(ce *commands.Event) { ce.Reply("Rang `%s` in the active WhatsApp call.", target) } +func fnCallLinkCreate(ce *commands.Event) { + video, err := callMediaArg(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-create [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + link, err := client.VOIP.CreateCallLink(ce.Ctx, video) + if err != nil { + ce.Reply("Failed to create the WhatsApp call link: %v", err) + return + } + ce.Reply("Created a WhatsApp %s call link:\n\n%s", callMediaName(video), link.URL) +} + +func fnCallLinkPreview(ce *commands.Event) { + token, video, err := callLinkArgs(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-preview [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + preview, err := client.VOIP.PreviewCallLink(ce.Ctx, token, video) + if err != nil { + ce.Reply("Failed to preview the WhatsApp call link: %v", err) + return + } + creator := preview.Creator + if !preview.CreatorPhoneNumber.IsEmpty() { + creator = preview.CreatorPhoneNumber + } + ce.Reply( + "**WhatsApp %s call link**\n\nCreator: `%s`\n\nApproval required: **%t**\n\nYou are an admin: **%t**", + callMediaName(preview.Video), creator, preview.ApprovalRequired, preview.IsAdmin, + ) +} + +func fnCallLinkJoin(ce *commands.Event) { + token, video, err := callLinkArgs(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-join [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + call, err := client.joinMatrixRTCCallLink(ce.Ctx, ce.Portal, token, video) + if err != nil { + ce.Reply("Failed to join the WhatsApp call link: %v", err) + return + } + if state, ok, _ := client.VOIP.WaitingRoomState(call.ID()); ok && state.InWaitingRoom { + ce.Reply("Joined the WhatsApp call link waiting room. Element will ring in this room while approval is pending.") + } else { + ce.Reply("Joined the WhatsApp call link. Element will ring in this room.") + } +} + +func fnCallWaiting(ce *commands.Event) { + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + state, ok, err := client.VOIP.WaitingRoomState(call.WACallID) + if err != nil { + ce.Reply("Failed to read the waiting room: %v", err) + return + } + if !ok { + ce.Reply("The active call has no WhatsApp call-link waiting-room state.") + return + } + ce.Reply(formatWaitingRoomState(state)) +} + +func fnCallApproval(ce *commands.Event) { + if len(ce.Args) != 1 { + ce.Reply("Usage: `$cmdprefix call-approval `") + return + } + enabled, err := parseCallApproval(ce.Args[0]) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-approval `") + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.SetApprovalRequired(ce.Ctx, call.WACallID, enabled); err != nil { + ce.Reply("Failed to change call-link approval: %v", err) + return + } + ce.Reply("WhatsApp call-link approval is now **%s**.", map[bool]string{true: "enabled", false: "disabled"}[enabled]) +} + +func fnCallAdmit(ce *commands.Event) { + fnCallWaitingParticipant(ce, true) +} + +func fnCallDeny(ce *commands.Event) { + fnCallWaitingParticipant(ce, false) +} + +func fnCallWaitingParticipant(ce *commands.Event, admit bool) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if admit { + err = client.VOIP.AdmitParticipant(ce.Ctx, call.WACallID, target) + } else { + err = client.VOIP.DenyParticipant(ce.Ctx, call.WACallID, target) + } + if err != nil { + ce.Reply("Failed to update the waiting-room participant: %v", err) + return + } + action := "Admitted" + if !admit { + action = "Denied" + } + ce.Reply("%s `%s` in the WhatsApp call-link waiting room.", action, target) +} + func callTargetArg(ce *commands.Event) (string, bool) { if len(ce.Args) != 1 { ce.Reply("Usage: `$cmdprefix %s `", ce.Command) @@ -114,25 +337,42 @@ func activePortalCall(ce *commands.Event) (*WhatsAppClient, *wadb.MatrixRTCCall, if ce.Portal == nil { return nil, nil, errors.New("this command can only be used in a portal room") } - login := ce.Bridge.GetCachedUserLoginByID(ce.Portal.Receiver) - if login == nil { - return nil, nil, errors.New("the WhatsApp login for this portal is not available") - } - client, ok := login.Client.(*WhatsAppClient) - if !ok || client == nil || !client.IsLoggedIn() { - return nil, nil, errors.New("the WhatsApp login for this portal is not connected") + client, err := commandWhatsAppClient(ce) + if err != nil { + return nil, nil, err } calls, err := client.Main.DB.MatrixRTCCall.GetActiveInRoom(ce.Ctx, ce.Portal.MXID) if err != nil { return nil, nil, fmt.Errorf("query active calls: %w", err) } - call, err := selectActiveCallForLogin(calls, login.ID) + call, err := selectActiveCallForLogin(calls, client.UserLogin.ID) if err != nil { return nil, nil, err } return client, call, nil } +func commandWhatsAppClient(ce *commands.Event) (*WhatsAppClient, error) { + var loginID networkid.UserLoginID + if ce.Portal != nil { + loginID = ce.Portal.Receiver + } else if login := ce.User.GetDefaultLogin(); login != nil { + loginID = login.ID + } + login := ce.Bridge.GetCachedUserLoginByID(loginID) + if login == nil { + return nil, errors.New("the WhatsApp login is not available") + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || !client.IsLoggedIn() { + return nil, errors.New("the WhatsApp login is not connected") + } + if client.VOIP == nil || !client.VOIP.Enabled() { + return nil, errors.New("WhatsApp calling is not enabled") + } + return client, nil +} + func selectActiveCallForLogin(calls []*wadb.MatrixRTCCall, loginID networkid.UserLoginID) (*wadb.MatrixRTCCall, error) { var selected *wadb.MatrixRTCCall for _, call := range calls { @@ -173,3 +413,81 @@ func formatGroupCallRoster(state meowcaller.GroupCallState) string { } return strings.Join(lines, "\n") } + +func formatWaitingRoomState(state meowcaller.WaitingRoomState) string { + lines := []string{ + fmt.Sprintf( + "**WhatsApp call-link waiting room (transaction %d):** approval **%s**, admin **%t**, waiting **%t**", + state.TransactionID, + map[bool]string{true: "enabled", false: "disabled"}[state.Enabled], + state.IsAdmin, + state.InWaitingRoom, + ), + } + users := slices.Clone(state.Users) + slices.SortFunc(users, func(a, b meowcaller.WaitingRoomUser) int { + return strings.Compare(a.JID.String(), b.JID.String()) + }) + for _, user := range users { + identity := user.JID + if !user.PN.IsEmpty() { + identity = user.PN + } + lines = append(lines, fmt.Sprintf("- `%s`: %s", identity, user.State)) + } + if len(users) == 0 { + lines = append(lines, "- No users are waiting.") + } + return strings.Join(lines, "\n") +} + +func callLinkArgs(args []string) (token string, video bool, err error) { + if len(args) < 1 || len(args) > 2 { + return "", false, errors.New("invalid call-link arguments") + } + token = strings.TrimSpace(args[0]) + if token == "" { + return "", false, errors.New("call-link token is empty") + } + if len(args) == 2 { + video, err = callMediaArg(args[1:]) + return + } + video = strings.HasPrefix(strings.ToLower(token), "https://call.whatsapp.com/video/") + return +} + +func callMediaArg(args []string) (bool, error) { + if len(args) == 0 { + return false, nil + } + if len(args) != 1 { + return false, errors.New("invalid call media") + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "audio": + return false, nil + case "video": + return true, nil + default: + return false, errors.New("invalid call media") + } +} + +func callMediaName(video bool) string { + if video { + return "video" + } + return "audio" +} + +func parseCallApproval(raw string) (bool, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "on", "true", "enable", "enabled": + return true, nil + case "off", "false", "disable", "disabled": + return false, nil + default: + return false, errors.New("invalid approval state") + } +} diff --git a/pkg/connector/commands_voip_test.go b/pkg/connector/commands_voip_test.go index 7b70848..75007fe 100644 --- a/pkg/connector/commands_voip_test.go +++ b/pkg/connector/commands_voip_test.go @@ -73,3 +73,72 @@ func TestFormatGroupCallRoster(t *testing.T) { } } } + +func TestCallLinkArgsInfersMediaFromURL(t *testing.T) { + token, video, err := callLinkArgs([]string{"https://call.whatsapp.com/video/TOKEN"}) + if err != nil { + t.Fatalf("callLinkArgs returned error: %v", err) + } + if token != "https://call.whatsapp.com/video/TOKEN" || !video { + t.Fatalf("callLinkArgs = (%q, %t), want video URL and true", token, video) + } + + token, video, err = callLinkArgs([]string{"TOKEN", "video"}) + if err != nil { + t.Fatalf("callLinkArgs with explicit media returned error: %v", err) + } + if token != "TOKEN" || !video { + t.Fatalf("callLinkArgs = (%q, %t), want TOKEN and true", token, video) + } +} + +func TestCallMediaArgRejectsUnknownMedia(t *testing.T) { + if _, err := callMediaArg([]string{"screen"}); err == nil { + t.Fatal("callMediaArg accepted an unknown media kind") + } + if video, err := callMediaArg(nil); err != nil || video { + t.Fatalf("callMediaArg default = (%t, %v), want audio and nil", video, err) + } +} + +func TestParseCallApproval(t *testing.T) { + for _, raw := range []string{"on", "true", "enabled"} { + if enabled, err := parseCallApproval(raw); err != nil || !enabled { + t.Errorf("parseCallApproval(%q) = (%t, %v), want true and nil", raw, enabled, err) + } + } + for _, raw := range []string{"off", "false", "disabled"} { + if enabled, err := parseCallApproval(raw); err != nil || enabled { + t.Errorf("parseCallApproval(%q) = (%t, %v), want false and nil", raw, enabled, err) + } + } + if _, err := parseCallApproval("maybe"); err == nil { + t.Fatal("parseCallApproval accepted an invalid value") + } +} + +func TestFormatWaitingRoomState(t *testing.T) { + state := meowcaller.WaitingRoomState{ + Enabled: true, + IsAdmin: true, + InWaitingRoom: false, + TransactionID: 7, + Users: []meowcaller.WaitingRoomUser{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + State: "pending", + }, + }, + } + got := formatWaitingRoomState(state) + for _, want := range []string{ + "transaction 7", + "approval **enabled**", + "`15550000002@s.whatsapp.net`: pending", + } { + if !strings.Contains(got, want) { + t.Errorf("formatted waiting room missing %q:\n%s", want, got) + } + } +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index a9bb517..ac64f71 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -117,6 +117,8 @@ func (wa *WhatsAppConnector) Init(bridge *bridgev2.Bridge) { wa.Bridge.Commands.(*commands.Processor).AddHandlers( cmdAccept, cmdSync, cmdInviteLink, cmdResolveLink, cmdJoin, cmdCallParticipants, cmdCallAdd, cmdCallRing, + cmdCallLinkCreate, cmdCallLinkPreview, cmdCallLinkJoin, + cmdCallWaiting, cmdCallApproval, cmdCallAdmit, cmdCallDeny, ) wa.mediaEditCache = make(MediaEditCache) wa.initMatrixRTCEventHooks() diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index eb513ab..cb23aad 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -98,6 +98,55 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal return call.Reject() } } + return wa.announceMatrixRTCCallInPortal(ctx, call, portal, peer, "incoming") +} + +func (wa *WhatsAppClient) joinMatrixRTCCallLink( + ctx context.Context, + portal *bridgev2.Portal, + tokenOrURL string, + video bool, +) (*meowcaller.Call, error) { + if portal == nil || portal.MXID == "" { + return nil, fmt.Errorf("call links must be joined from an existing portal room") + } + if video && !wa.Main.Config.VOIP.Video.Enabled { + return nil, fmt.Errorf("WhatsApp call-link video requires voip.video.enabled") + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return nil, err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + return nil, fmt.Errorf("active MatrixRTC call limit reached for login %s", wa.UserLogin.ID) + } + } + call, err := wa.VOIP.JoinCallLink(ctx, tokenOrURL, video) + if err != nil { + return nil, err + } + peer := wa.matrixRTCAnnouncementPeer(ctx, call.Peer()) + if err = wa.announceMatrixRTCCallInPortal(ctx, call, portal, peer, "call_link"); err != nil { + _ = call.Hangup() + return nil, err + } + return call, nil +} + +func (wa *WhatsAppClient) announceMatrixRTCCallInPortal( + ctx context.Context, + call *meowcaller.Call, + portal *bridgev2.Portal, + peer types.JID, + direction string, +) error { + if call == nil { + return fmt.Errorf("WhatsApp call is nil") + } + if portal == nil || portal.MXID == "" { + return fmt.Errorf("Matrix portal room is not available") + } focus, err := voip.DiscoverLiveKitFocus(ctx, nil, wa.Main.Bridge.Matrix.ServerName(), wa.Main.Config.VOIP.MatrixRTC.LiveKitServiceURL) if err != nil { return err @@ -125,7 +174,7 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal RoomID: portal.MXID, PortalKey: portal.PortalKey, PeerJID: peer, - Direction: "incoming", + Direction: direction, MediaKind: session.Intent, FocusType: focus.Type, LiveKitServiceURL: focus.LiveKitServiceURL, @@ -147,11 +196,13 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { return err } - log.Info(). + zerolog.Ctx(ctx).Info(). + Str("call_id", call.ID()). + Str("direction", direction). Stringer("room_id", portal.MXID). Stringer("participant_mxid", intent.GetMXID()). Str("device_id", deviceID). - Msg("Announced incoming WhatsApp call over MatrixRTC") + Msg("Announced WhatsApp call over MatrixRTC") return nil } diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 86036a7..3315e8f 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -197,6 +197,68 @@ func (m *Manager) RingParticipant(ctx context.Context, callID, target string) er return call.RingParticipant(ctx, target) } +func (m *Manager) CreateCallLink(ctx context.Context, video bool) (meowcaller.CallLink, error) { + if !m.Enabled() { + return meowcaller.CallLink{}, ErrNotEnabled + } + return m.client.CreateCallLink(ctx, meowcaller.CallLinkOptions{Video: video}) +} + +func (m *Manager) PreviewCallLink(ctx context.Context, tokenOrURL string, video bool) (meowcaller.CallLinkPreview, error) { + if !m.Enabled() { + return meowcaller.CallLinkPreview{}, ErrNotEnabled + } + return m.client.PreviewCallLink(ctx, tokenOrURL, meowcaller.CallLinkOptions{Video: video}) +} + +func (m *Manager) JoinCallLink(ctx context.Context, tokenOrURL string, video bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + call, err := m.client.JoinCallLink(ctx, tokenOrURL, meowcaller.CallLinkOptions{Video: video}) + if err != nil { + return nil, err + } + if call == nil { + return nil, fmt.Errorf("meowcaller returned no call for call-link join") + } + m.trackCall(call, call.Peer()) + return call, nil +} + +func (m *Manager) WaitingRoomState(callID string) (meowcaller.WaitingRoomState, bool, error) { + call, err := m.activeCall(callID) + if err != nil { + return meowcaller.WaitingRoomState{}, false, err + } + state, ok := call.WaitingRoomState() + return state, ok, nil +} + +func (m *Manager) SetApprovalRequired(ctx context.Context, callID string, enabled bool) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.SetApprovalRequired(ctx, enabled) +} + +func (m *Manager) AdmitParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.AdmitParticipant(ctx, target) +} + +func (m *Manager) DenyParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.DenyParticipant(ctx, target) +} + func (m *Manager) activeCall(callID string) (*meowcaller.Call, error) { if !m.Enabled() { return nil, ErrNotEnabled From 91eff850126ea36487ba142419d70905d6501b9e Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:03:26 +0200 Subject: [PATCH 38/44] voip: select group camera participants --- pkg/connector/commands_voip.go | 29 ++++++++++++ pkg/connector/connector.go | 2 +- pkg/connector/example-config.yaml | 4 +- pkg/connector/voip/group_media.go | 64 +++++++++++++++++++++++--- pkg/connector/voip/group_media_test.go | 52 +++++++++++++++++++++ pkg/connector/voip/manager.go | 43 ++++++++++++++++- 6 files changed, 185 insertions(+), 9 deletions(-) diff --git a/pkg/connector/commands_voip.go b/pkg/connector/commands_voip.go index 685a804..4d7ee92 100644 --- a/pkg/connector/commands_voip.go +++ b/pkg/connector/commands_voip.go @@ -50,6 +50,18 @@ var cmdCallRing = &commands.FullHandler{ RequiresPortal: true, } +var cmdCallVideoSelect = &commands.FullHandler{ + Func: fnCallVideoSelect, + Name: "call-video-select", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Select which WhatsApp group participant is shown on the Matrix camera track.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + var cmdCallLinkCreate = &commands.FullHandler{ Func: fnCallLinkCreate, Name: "call-link-create", @@ -183,6 +195,23 @@ func fnCallRing(ce *commands.Event) { ce.Reply("Rang `%s` in the active WhatsApp call.", target) } +func fnCallVideoSelect(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.SelectVideoParticipant(call.WACallID, target); err != nil { + ce.Reply("Failed to select the WhatsApp video participant: %v", err) + return + } + ce.Reply("Selected `%s` for the WhatsApp camera track.", target) +} + func fnCallLinkCreate(ce *commands.Event) { video, err := callMediaArg(ce.Args) if err != nil { diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index ac64f71..7213cdd 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -116,7 +116,7 @@ func (wa *WhatsAppConnector) Init(bridge *bridgev2.Bridge) { wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) wa.Bridge.Commands.(*commands.Processor).AddHandlers( cmdAccept, cmdSync, cmdInviteLink, cmdResolveLink, cmdJoin, - cmdCallParticipants, cmdCallAdd, cmdCallRing, + cmdCallParticipants, cmdCallAdd, cmdCallRing, cmdCallVideoSelect, cmdCallLinkCreate, cmdCallLinkPreview, cmdCallLinkJoin, cmdCallWaiting, cmdCallApproval, cmdCallAdmit, cmdCallDeny, ) diff --git a/pkg/connector/example-config.yaml b/pkg/connector/example-config.yaml index db5ef0d..30c545c 100644 --- a/pkg/connector/example-config.yaml +++ b/pkg/connector/example-config.yaml @@ -80,7 +80,7 @@ voip: # ring - ring MatrixRTC/Element Call and answer WhatsApp only after a Matrix user joins # auto_answer - test-only behavior that answers WhatsApp immediately incoming_policy: ring - # WhatsApp/meowcaller currently supports one live 1:1 call leg per login. + # Limit concurrent WhatsApp call legs per login. A direct, group, or call-link call counts as one leg. max_active_calls_per_login: 1 matrixrtc: @@ -118,6 +118,8 @@ voip: video: # Video is passed through as H.264. Other codecs are ignored. enabled: false + # active_speaker or selected_participant. Both keep one group camera stable; + # change the selected participant with !wa call-video-select. selected_source_policy: active_speaker max_width: 1280 max_height: 720 diff --git a/pkg/connector/voip/group_media.go b/pkg/connector/voip/group_media.go index 63d29e3..9f87d58 100644 --- a/pkg/connector/voip/group_media.go +++ b/pkg/connector/voip/group_media.go @@ -1,6 +1,7 @@ package voip import ( + "errors" "sync" "github.com/purpshell/meowcaller" @@ -22,6 +23,7 @@ type whatsAppVideoRouter struct { group bool connected map[string]struct{} + aliases map[string]string screenSharers map[string]struct{} selectedCamera string selectedScreen string @@ -37,6 +39,7 @@ func newWhatsAppVideoRouter( setCameraMute: setCameraMute, setScreenMute: setScreenMute, connected: make(map[string]struct{}), + aliases: make(map[string]string), screenSharers: make(map[string]struct{}), } } @@ -46,20 +49,33 @@ func (r *whatsAppVideoRouter) SetGroupState(state meowcaller.GroupCallState) { return } connected := make(map[string]struct{}) + aliases := make(map[string]string) for _, participant := range state.Participants { if participant.State != "connected" { continue } - addVideoParticipantIdentity(connected, participant.JID) - addVideoParticipantIdentity(connected, participant.PN) + canonical := videoParticipantIdentity(participant.JID) + if canonical == "" { + canonical = videoParticipantIdentity(participant.PN) + } + if canonical == "" && len(participant.Devices) > 0 { + canonical = videoParticipantIdentity(participant.Devices[0].JID) + } + if canonical == "" { + continue + } + connected[canonical] = struct{}{} + addVideoParticipantAlias(aliases, participant.JID, canonical) + addVideoParticipantAlias(aliases, participant.PN, canonical) for _, device := range participant.Devices { - addVideoParticipantIdentity(connected, device.JID) + addVideoParticipantAlias(aliases, device.JID, canonical) } } r.mu.Lock() r.group = true r.connected = connected + r.aliases = aliases cameraRemoved := r.selectedCamera != "" if cameraRemoved { _, cameraRemoved = connected[r.selectedCamera] @@ -89,12 +105,42 @@ func (r *whatsAppVideoRouter) SetGroupState(state meowcaller.GroupCallState) { } } +func (r *whatsAppVideoRouter) SelectCamera(participant types.JID) error { + if r == nil { + return errors.New("WhatsApp video router is not available") + } + identity := videoParticipantIdentity(participant) + r.mu.Lock() + if !r.group { + r.mu.Unlock() + return errors.New("the active call has no WhatsApp group video roster") + } + canonical := r.aliases[identity] + if canonical == "" { + canonical = identity + } + if _, ok := r.connected[canonical]; !ok { + r.mu.Unlock() + return errors.New("the selected WhatsApp participant is not connected") + } + r.selectedCamera = canonical + setCameraMute := r.setCameraMute + r.mu.Unlock() + if setCameraMute != nil { + setCameraMute(true) + } + return nil +} + func (r *whatsAppVideoRouter) SetScreenShare(state meowcaller.ScreenShareState) { if r == nil || state.Participant.IsEmpty() { return } participant := videoParticipantIdentity(state.Participant) r.mu.Lock() + if canonical := r.aliases[participant]; canonical != "" { + participant = canonical + } if state.Active { r.screenSharers[participant] = struct{}{} if r.selectedScreen == "" { @@ -121,6 +167,9 @@ func (r *whatsAppVideoRouter) WriteParticipantFrame(frame meowcaller.Participant } identity := participantVideoFrameIdentity(frame) r.mu.Lock() + if canonical := r.aliases[identity]; canonical != "" { + identity = canonical + } _, sharing := r.screenSharers[identity] if sharing { if r.selectedScreen == "" { @@ -176,13 +225,16 @@ func participantVideoFrameIdentity(frame meowcaller.ParticipantVideoFrame) strin return frame.ParticipantID } -func addVideoParticipantIdentity(target map[string]struct{}, jid types.JID) { - if !jid.IsEmpty() { - target[videoParticipantIdentity(jid)] = struct{}{} +func addVideoParticipantAlias(target map[string]string, jid types.JID, canonical string) { + if identity := videoParticipantIdentity(jid); identity != "" { + target[identity] = canonical } } func videoParticipantIdentity(jid types.JID) string { + if jid.IsEmpty() { + return "" + } return jid.ToNonAD().String() } diff --git a/pkg/connector/voip/group_media_test.go b/pkg/connector/voip/group_media_test.go index 344388c..e29b1ac 100644 --- a/pkg/connector/voip/group_media_test.go +++ b/pkg/connector/voip/group_media_test.go @@ -60,6 +60,58 @@ func TestWhatsAppVideoRouterKeepsOneStableGroupCamera(t *testing.T) { } } +func TestWhatsAppVideoRouterSelectsCameraByPhoneNumberAlias(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + aliceLID := types.NewJID("111", types.HiddenUserServer) + alicePN := types.NewJID("15550000001", types.DefaultUserServer) + bobLID := types.NewJID("222", types.HiddenUserServer) + bobPN := types.NewJID("15550000002", types.DefaultUserServer) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + {JID: aliceLID, PN: alicePN, State: "connected"}, + {JID: bobLID, PN: bobPN, State: "connected"}, + }, + }) + if err := router.SelectCamera(bobPN); err != nil { + t.Fatalf("SelectCamera returned error: %v", err) + } + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + Sender: aliceLID, + AccessUnit: []byte{0x01}, + }) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + Sender: bobLID, + AccessUnit: []byte{0x02}, + }) + + if len(camera.frames) != 1 || camera.frames[0][0] != 0x02 { + t.Fatalf("camera frames = %v, want only the selected participant", camera.frames) + } +} + +func TestWhatsAppVideoRouterRejectsDisconnectedCameraSelection(t *testing.T) { + router := newWhatsAppVideoRouter( + &recordingGroupVideoSink{}, + &recordingGroupVideoSink{}, + nil, + nil, + ) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + { + JID: types.NewJID("111", types.HiddenUserServer), + State: "connected", + }, + }, + }) + if err := router.SelectCamera(types.NewJID("222", types.HiddenUserServer)); err == nil { + t.Fatal("SelectCamera accepted a disconnected participant") + } +} + func TestWhatsAppVideoRouterSeparatesScreenShareFromCamera(t *testing.T) { camera := &recordingGroupVideoSink{} screen := &recordingGroupVideoSink{} diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index 3315e8f..d5b390e 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -3,6 +3,7 @@ package voip import ( "context" "fmt" + "strings" "sync" "time" @@ -51,6 +52,7 @@ type Manager struct { whatsAppMuted map[string]bool whatsAppVideoMuted map[string]bool videoKeyframePending map[string]bool + videoRouters map[string]*whatsAppVideoRouter incomingCallNotify func(*meowcaller.Call) callEndNotify func(callID, reason string) callReactionNotify func(callID string, reaction meowcaller.CallReaction) @@ -71,6 +73,7 @@ func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Man whatsAppMuted: make(map[string]bool), whatsAppVideoMuted: make(map[string]bool), videoKeyframePending: make(map[string]bool), + videoRouters: make(map[string]*whatsAppVideoRouter), } if !cfg.Enabled || waClient == nil { return manager @@ -197,6 +200,23 @@ func (m *Manager) RingParticipant(ctx context.Context, callID, target string) er return call.RingParticipant(ctx, target) } +func (m *Manager) SelectVideoParticipant(callID, target string) error { + if _, err := m.activeCall(callID); err != nil { + return err + } + jid, err := parseVideoParticipantTarget(target) + if err != nil { + return err + } + m.mu.Lock() + router := m.videoRouters[callID] + m.mu.Unlock() + if router == nil { + return fmt.Errorf("WhatsApp group video is not connected to LiveKit") + } + return router.SelectCamera(jid) +} + func (m *Manager) CreateCallLink(ctx context.Context, video bool) (meowcaller.CallLink, error) { if !m.Enabled() { return meowcaller.CallLink{}, ErrNotEnabled @@ -320,6 +340,7 @@ func (m *Manager) AbortAll() { m.whatsAppMuted = make(map[string]bool) m.whatsAppVideoMuted = make(map[string]bool) m.videoKeyframePending = make(map[string]bool) + m.videoRouters = make(map[string]*whatsAppVideoRouter) participants := make([]*LiveKitParticipant, 0, len(m.livekit)) for _, participant := range m.livekit { participants = append(participants, participant) @@ -364,6 +385,7 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth m.handleMatrixAudioMuteState(call, muted) }) videoEnabled := m.cfg.Video.Enabled + var videoRouter *whatsAppVideoRouter if videoEnabled { var videoBuffer whatsAppVideoStartupBuffer var videoBufferLock sync.Mutex @@ -424,7 +446,7 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth m.clearLiveKitConnecting(waCallID) return err } - videoRouter := newWhatsAppVideoRouter( + videoRouter = newWhatsAppVideoRouter( participant.WhatsAppVideoSink(), participant.WhatsAppScreenShareSink(), participant.SetWhatsAppVideoMuted, @@ -466,6 +488,9 @@ func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, auth delete(m.videoKeyframePending, waCallID) delete(m.livekitConnecting, waCallID) m.livekit[waCallID] = participant + if videoRouter != nil { + m.videoRouters[waCallID] = videoRouter + } m.mu.Unlock() if videoEnabled && keyframePending { participant.requestRemoteVideoKeyframe() @@ -866,6 +891,7 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { delete(m.whatsAppMuted, call.ID()) delete(m.whatsAppVideoMuted, call.ID()) delete(m.videoKeyframePending, call.ID()) + delete(m.videoRouters, call.ID()) participant := m.livekit[call.ID()] delete(m.livekit, call.ID()) delete(m.livekitConnecting, call.ID()) @@ -914,6 +940,21 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { }) } +func parseVideoParticipantTarget(target string) (types.JID, error) { + target = strings.TrimSpace(target) + if target == "" { + return types.EmptyJID, fmt.Errorf("WhatsApp video participant is empty") + } + if !strings.ContainsRune(target, '@') { + return types.NewJID(strings.TrimPrefix(target, "+"), types.DefaultUserServer), nil + } + jid, err := types.ParseJID(target) + if err != nil { + return types.EmptyJID, fmt.Errorf("parse WhatsApp video participant: %w", err) + } + return jid.ToNonAD(), nil +} + func (m *Manager) requestLiveKitVideoKeyframe(callID string) { m.mu.Lock() call := m.calls[callID] From 8521def8daa24e1d13f02e66b9462cbe3f47d9cb Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:07:43 +0200 Subject: [PATCH 39/44] matrixrtc: route incoming group calls --- pkg/connector/client.go | 38 +++++++++-------- pkg/connector/handlewhatsapp.go | 4 +- pkg/connector/matrixrtc_outgoing.go | 65 ++++++++++++++++++++++++++++- pkg/connector/matrixrtc_test.go | 55 ++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 19 deletions(-) diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 69c2f23..03afabc 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -58,6 +58,7 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. pushNamesSynced: exsync.NewEvent(), createDedup: exsync.NewSet[types.MessageID](), appStateFullSyncAttempted: make(map[appstate.WAPatchName]time.Time), + incomingCallGroups: make(map[string]incomingCallGroup), } login.Client = w @@ -76,7 +77,7 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. if w.Device != nil { log := w.UserLogin.Log.With().Str("component", "whatsmeow").Logger() w.Client = whatsmeow.NewClient(w.Device, waLog.Zerolog(log)) - w.Client.AddEventHandlerWithSuccessStatus(w.handleWAEvent) + w.Client.AddEventHandler(w.trackIncomingCallEvent) w.Client.SynchronousAck = true w.Client.EnableDecryptedEventBuffer = bridgev2.PortalEventBuffer == 0 w.Client.ManualHistorySyncDownload = true @@ -94,6 +95,7 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. w.VOIP.SetHandRaiseHandler(func(callID string, state meowcaller.HandRaiseState) { go w.handleWhatsAppHandRaise(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) }) + w.Client.AddEventHandlerWithSuccessStatus(w.handleWAEvent) w.Client.SetForceActiveDeliveryReceipts(wa.Config.ForceActiveDeliveryReceipts) w.Client.InitialAutoReconnect = wa.Config.InitialAutoReconnect w.Client.UseRetryMessageStore = wa.Config.UseWhatsAppRetryStore @@ -118,22 +120,24 @@ type WhatsAppClient struct { JID types.JID MC mClient - historySyncWakeup chan struct{} - stopLoops atomic.Pointer[context.CancelFunc] - resyncQueue map[types.JID]resyncQueueItem - resyncQueueLock sync.Mutex - nextResync time.Time - directMediaRetries map[networkid.MessageID]*directMediaRetry - directMediaLock sync.Mutex - voipHandBridgeLock sync.Mutex - voipHandRaiseLock sync.Mutex - voipHandRaises map[string]map[types.JID]bool - mediaRetryLock *semaphore.Weighted - offlineSyncWaiter atomic.Pointer[chan error] - isNewLogin bool - pushNamesSynced *exsync.Event - lastPresence types.Presence - createDedup *exsync.Set[types.MessageID] + historySyncWakeup chan struct{} + stopLoops atomic.Pointer[context.CancelFunc] + resyncQueue map[types.JID]resyncQueueItem + resyncQueueLock sync.Mutex + nextResync time.Time + directMediaRetries map[networkid.MessageID]*directMediaRetry + directMediaLock sync.Mutex + voipHandBridgeLock sync.Mutex + voipHandRaiseLock sync.Mutex + voipHandRaises map[string]map[types.JID]bool + incomingCallGroupLock sync.Mutex + incomingCallGroups map[string]incomingCallGroup + mediaRetryLock *semaphore.Weighted + offlineSyncWaiter atomic.Pointer[chan error] + isNewLogin bool + pushNamesSynced *exsync.Event + lastPresence types.Presence + createDedup *exsync.Set[types.MessageID] appStateRecoveryLock sync.Mutex appStateFullSyncAttempted map[appstate.WAPatchName]time.Time diff --git a/pkg/connector/handlewhatsapp.go b/pkg/connector/handlewhatsapp.go index f12ebad..99845ac 100644 --- a/pkg/connector/handlewhatsapp.go +++ b/pkg/connector/handlewhatsapp.go @@ -93,7 +93,9 @@ func (wa *WhatsAppClient) handleWAEvent(rawEvt any) (success bool) { success = wa.handleWACallStart(ctx, evt.GroupJID, evt.CallCreator, evt.CallCreatorAlt, evt.CallID, "", evt.Timestamp) case *events.CallOfferNotice: success = wa.handleWACallStart(ctx, evt.GroupJID, evt.CallCreator, evt.CallCreatorAlt, evt.CallID, evt.Type, evt.Timestamp) - case *events.CallTerminate, *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: + case *events.CallTerminate: + wa.clearIncomingCallGroup(evt.CallID) + case *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: // ignore case *events.IdentityChange: wa.handleWAIdentityChange(ctx, evt) diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index cb23aad..19f2d2c 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -9,6 +9,7 @@ import ( "github.com/purpshell/meowcaller" "github.com/rs/zerolog" "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix" "maunium.net/go/mautrix/bridgev2" mxbridge "maunium.net/go/mautrix/bridgev2/matrix" @@ -26,6 +27,22 @@ const ( matrixRTCStickyDuration = time.Hour ) +type incomingCallGroup struct { + JID types.JID + ExpiresAt time.Time +} + +func (wa *WhatsAppClient) trackIncomingCallEvent(rawEvt any) { + switch evt := rawEvt.(type) { + case *events.CallOffer: + wa.trackIncomingCallGroup(evt.CallID, evt.GroupJID) + case *events.CallOfferNotice: + wa.trackIncomingCallGroup(evt.CallID, evt.GroupJID) + case *events.CallTerminate: + wa.clearIncomingCallGroup(evt.CallID) + } +} + func (wa *WhatsAppClient) handleIncomingVOIPCall(call *meowcaller.Call) { if call == nil { return @@ -42,6 +59,7 @@ func (wa *WhatsAppClient) handleIncomingVOIPCall(call *meowcaller.Call) { } func (wa *WhatsAppClient) handleVOIPCallEnded(callID, reason string) { + wa.clearIncomingCallGroup(callID) wa.clearWhatsAppRemoteHandRaises(callID) ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) log := wa.UserLogin.Log.With().Str("call_id", callID).Str("reason", reason).Logger() @@ -77,7 +95,8 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal Stringer("peer_jid", call.Peer()). Logger() peer := wa.matrixRTCAnnouncementPeer(ctx, call.Peer()) - portal, err := wa.Main.Bridge.GetPortalByKey(ctx, wa.makeWAPortalKey(peer)) + portalPeer := wa.incomingCallPortalPeer(call.ID(), peer) + portal, err := wa.Main.Bridge.GetPortalByKey(ctx, wa.makeWAPortalKey(portalPeer)) if err != nil { return err } @@ -101,6 +120,50 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal return wa.announceMatrixRTCCallInPortal(ctx, call, portal, peer, "incoming") } +func (wa *WhatsAppClient) trackIncomingCallGroup(callID string, group types.JID) { + if wa == nil || callID == "" || group.Server != types.GroupServer || group.User == "" { + return + } + wa.incomingCallGroupLock.Lock() + if wa.incomingCallGroups == nil { + wa.incomingCallGroups = make(map[string]incomingCallGroup) + } + now := time.Now() + for trackedCallID, tracked := range wa.incomingCallGroups { + if !tracked.ExpiresAt.After(now) { + delete(wa.incomingCallGroups, trackedCallID) + } + } + wa.incomingCallGroups[callID] = incomingCallGroup{ + JID: group.ToNonAD(), + ExpiresAt: now.Add(callEventMaxAge), + } + wa.incomingCallGroupLock.Unlock() +} + +func (wa *WhatsAppClient) incomingCallPortalPeer(callID string, fallback types.JID) types.JID { + if wa == nil || callID == "" { + return fallback + } + wa.incomingCallGroupLock.Lock() + tracked := wa.incomingCallGroups[callID] + delete(wa.incomingCallGroups, callID) + wa.incomingCallGroupLock.Unlock() + if tracked.JID.IsEmpty() || !tracked.ExpiresAt.After(time.Now()) { + return fallback + } + return tracked.JID +} + +func (wa *WhatsAppClient) clearIncomingCallGroup(callID string) { + if wa == nil || callID == "" { + return + } + wa.incomingCallGroupLock.Lock() + delete(wa.incomingCallGroups, callID) + wa.incomingCallGroupLock.Unlock() +} + func (wa *WhatsAppClient) joinMatrixRTCCallLink( ctx context.Context, portal *bridgev2.Portal, diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index baa450b..e1171ef 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -8,6 +8,7 @@ import ( "github.com/purpshell/meowcaller" "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -26,6 +27,60 @@ func TestShouldStartOutboundMatrixRTCCall(t *testing.T) { } } +func TestIncomingGroupCallUsesGroupPortalAndConsumesCorrelation(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + group := types.NewJID("120363000000000000", types.GroupServer) + fallback := types.NewJID("15550000001", types.DefaultUserServer) + + client.trackIncomingCallGroup("CALL", group) + if got := client.incomingCallPortalPeer("CALL", fallback); got != group { + t.Fatalf("incoming portal peer = %s, want group %s", got, group) + } + if got := client.incomingCallPortalPeer("CALL", fallback); got != fallback { + t.Fatalf("consumed incoming portal peer = %s, want fallback %s", got, fallback) + } +} + +func TestIncomingCallEventTracksOfferGroupBeforeManagedCall(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + group := types.NewJID("120363000000000000", types.GroupServer) + client.trackIncomingCallEvent(&events.CallOffer{ + BasicCallMeta: types.BasicCallMeta{ + CallID: "CALL", + GroupJID: group, + }, + }) + if got := client.incomingCallPortalPeer("CALL", types.EmptyJID); got != group { + t.Fatalf("incoming portal peer = %s, want group %s", got, group) + } +} + +func TestIncomingCallGroupIgnoresNonGroupJID(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + direct := types.NewJID("15550000001", types.DefaultUserServer) + client.trackIncomingCallGroup("CALL", direct) + if got := client.incomingCallPortalPeer("CALL", direct); got != direct { + t.Fatalf("incoming portal peer = %s, want direct fallback %s", got, direct) + } + if len(client.incomingCallGroups) != 0 { + t.Fatalf("tracked non-group calls = %d, want 0", len(client.incomingCallGroups)) + } +} + +func TestIncomingCallGroupIgnoresExpiredCorrelation(t *testing.T) { + group := types.NewJID("120363000000000000", types.GroupServer) + fallback := types.NewJID("15550000001", types.DefaultUserServer) + client := &WhatsAppClient{incomingCallGroups: map[string]incomingCallGroup{ + "CALL": { + JID: group, + ExpiresAt: time.Now().Add(-time.Second), + }, + }} + if got := client.incomingCallPortalPeer("CALL", fallback); got != fallback { + t.Fatalf("expired incoming portal peer = %s, want fallback %s", got, fallback) + } +} + func TestShouldStartOutboundMatrixRTCCallRejectsMessageMembership(t *testing.T) { evt := matrixRTCMemberEvent(event.MessageEventType) parsed, ok := voip.ParseMatrixRTCEvent(evt) From 326a47af7ac918287b00fa81242a22e854796edd Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:12:34 +0200 Subject: [PATCH 40/44] matrixrtc: restrict calls to login owners --- pkg/connector/commands_voip.go | 8 ++++++++ pkg/connector/matrixrtc.go | 12 ++++++++++++ pkg/connector/matrixrtc_test.go | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/pkg/connector/commands_voip.go b/pkg/connector/commands_voip.go index 4d7ee92..b048d94 100644 --- a/pkg/connector/commands_voip.go +++ b/pkg/connector/commands_voip.go @@ -9,6 +9,7 @@ import ( "github.com/purpshell/meowcaller" "maunium.net/go/mautrix/bridgev2/commands" "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/id" "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" ) @@ -392,6 +393,13 @@ func commandWhatsAppClient(ce *commands.Event) (*WhatsAppClient, error) { if login == nil { return nil, errors.New("the WhatsApp login is not available") } + var sender id.UserID + if ce.User != nil { + sender = ce.User.MXID + } + if !matrixRTCSenderOwnsLogin(sender, login) { + return nil, errors.New("the WhatsApp login for this portal belongs to another Matrix user") + } client, ok := login.Client.(*WhatsAppClient) if !ok || client == nil || !client.IsLoggedIn() { return nil, errors.New("the WhatsApp login is not connected") diff --git a/pkg/connector/matrixrtc.go b/pkg/connector/matrixrtc.go index 2d86f5a..7a232e6 100644 --- a/pkg/connector/matrixrtc.go +++ b/pkg/connector/matrixrtc.go @@ -98,6 +98,14 @@ func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *even log.Debug().Msg("Ignoring MatrixRTC event outside a bridged portal") return } + portalLogin := wa.Bridge.GetCachedUserLoginByID(portal.Receiver) + if !matrixRTCSenderOwnsLogin(parsed.Sender, portalLogin) { + log.Warn(). + Str("portal_receiver", string(portal.Receiver)). + Msg("Dropping MatrixRTC event from a user who does not own the portal login") + wa.Bridge.Matrix.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt)) + return + } activeCalls, err := wa.DB.MatrixRTCCall.GetActiveInRoom(ctx, parsed.RoomID) if err != nil { @@ -195,6 +203,10 @@ func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *even Msg("Handled MatrixRTC event for active bridged calls") } +func matrixRTCSenderOwnsLogin(sender id.UserID, login *bridgev2.UserLogin) bool { + return sender != "" && login != nil && login.User != nil && login.User.MXID == sender +} + func matrixRTCMembershipEventMatchesCall(evt voip.MatrixRTCEvent, call *wadb.MatrixRTCCall) bool { if call == nil || evt.EventID == "" || !voip.MatrixRTCEventHasJoinContent(evt) { return false diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index e1171ef..68d8673 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -9,6 +9,8 @@ import ( "github.com/purpshell/meowcaller" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -27,6 +29,23 @@ func TestShouldStartOutboundMatrixRTCCall(t *testing.T) { } } +func TestMatrixRTCSenderMustOwnPortalLogin(t *testing.T) { + login := &bridgev2.UserLogin{ + User: &bridgev2.User{ + User: &database.User{MXID: "@alice:example.com"}, + }, + } + if !matrixRTCSenderOwnsLogin("@alice:example.com", login) { + t.Fatal("matrixRTCSenderOwnsLogin rejected the login owner") + } + if matrixRTCSenderOwnsLogin("@mallory:example.com", login) { + t.Fatal("matrixRTCSenderOwnsLogin accepted another Matrix user") + } + if matrixRTCSenderOwnsLogin("@alice:example.com", nil) { + t.Fatal("matrixRTCSenderOwnsLogin accepted a missing login") + } +} + func TestIncomingGroupCallUsesGroupPortalAndConsumesCorrelation(t *testing.T) { client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} group := types.NewJID("120363000000000000", types.GroupServer) From 35720a37f23592ab0ff659c623e9482bdc98db1c Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:14:00 +0200 Subject: [PATCH 41/44] matrixrtc: suppress local reaction echoes --- pkg/connector/matrixrtc_reactions.go | 9 +++++---- pkg/connector/matrixrtc_test.go | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pkg/connector/matrixrtc_reactions.go b/pkg/connector/matrixrtc_reactions.go index f52c1fe..973ab53 100644 --- a/pkg/connector/matrixrtc_reactions.go +++ b/pkg/connector/matrixrtc_reactions.go @@ -110,7 +110,7 @@ func (wa *WhatsAppConnector) handleMatrixRTCCallControlEvent( } func (wa *WhatsAppClient) handleWhatsAppCallReaction(ctx context.Context, callID string, reaction meowcaller.CallReaction) { - if reaction.Removed { + if reaction.Removed || wa.isOwnWhatsAppCallParticipant(reaction.Sender) { return } emoji, supported := voip.NormalizeWhatsAppCallReaction(reaction.Emoji) @@ -244,10 +244,11 @@ func (wa *WhatsAppClient) clearWhatsAppRemoteHandRaises(callID string) { } func (wa *WhatsAppClient) isOwnWhatsAppCallParticipant(participant types.JID) bool { - if participant.IsEmpty() || wa.GetStore() == nil { + if participant.IsEmpty() || wa == nil || wa.Client == nil || wa.Client.Store == nil { return false } + device := wa.Client.Store participant = participant.ToNonAD() - return participant == wa.GetStore().GetLID().ToNonAD() || - participant == wa.GetStore().GetJID().ToNonAD() + return participant == device.GetLID().ToNonAD() || + participant == device.GetJID().ToNonAD() } diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index 68d8673..c60305a 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -7,6 +7,8 @@ import ( "time" "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/store" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix/bridgev2" @@ -46,6 +48,25 @@ func TestMatrixRTCSenderMustOwnPortalLogin(t *testing.T) { } } +func TestOwnWhatsAppCallParticipantMatchesPhoneAndLID(t *testing.T) { + phone := types.NewJID("15550000001", types.DefaultUserServer) + lid := types.NewJID("111", types.HiddenUserServer) + client := &WhatsAppClient{ + Client: &whatsmeow.Client{ + Store: &store.Device{ID: &phone, LID: lid}, + }, + } + if !client.isOwnWhatsAppCallParticipant(phone) { + t.Fatal("phone JID was not recognized as the local WhatsApp call participant") + } + if !client.isOwnWhatsAppCallParticipant(lid) { + t.Fatal("LID was not recognized as the local WhatsApp call participant") + } + if client.isOwnWhatsAppCallParticipant(types.NewJID("222", types.HiddenUserServer)) { + t.Fatal("remote LID was recognized as the local WhatsApp call participant") + } +} + func TestIncomingGroupCallUsesGroupPortalAndConsumesCorrelation(t *testing.T) { client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} group := types.NewJID("120363000000000000", types.GroupServer) From aa70741f50d51c1adafc864f8429cae4757b8f35 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:16:20 +0200 Subject: [PATCH 42/44] matrixrtc: serialize call starts and recover state --- pkg/connector/client.go | 2 + pkg/connector/matrixrtc_outgoing.go | 67 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 03afabc..b2ae726 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -130,6 +130,7 @@ type WhatsAppClient struct { voipHandBridgeLock sync.Mutex voipHandRaiseLock sync.Mutex voipHandRaises map[string]map[types.JID]bool + voipCallStartLock sync.Mutex incomingCallGroupLock sync.Mutex incomingCallGroups map[string]incomingCallGroup mediaRetryLock *semaphore.Weighted @@ -224,6 +225,7 @@ func (wa *WhatsAppClient) Connect(ctx context.Context) { if ctx.Err() != nil { return } + wa.cleanupStaleMatrixRTCCalls(ctx) wa.initMC() wa.startLoops() wa.Client.BackgroundEventCtx = wa.UserLogin.Log.WithContext(wa.Main.Bridge.BackgroundCtx) diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index 19f2d2c..f87b718 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -79,6 +79,49 @@ func (wa *WhatsAppClient) handleVOIPCallEnded(callID, reason string) { } } +func (wa *WhatsAppClient) cleanupStaleMatrixRTCCalls(ctx context.Context) { + if wa == nil || wa.Main == nil || wa.Main.DB == nil || wa.UserLogin == nil { + return + } + calls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + wa.UserLogin.Log.Err(err).Msg("Failed to query stale MatrixRTC calls during login load") + return + } + for _, call := range calls { + if call == nil { + continue + } + lastError := "" + if clearErr := wa.clearMatrixRTCMembership(ctx, call); clearErr != nil { + lastError = clearErr.Error() + wa.UserLogin.Log.Warn(). + Err(clearErr). + Str("call_id", call.WACallID). + Stringer("room_id", call.RoomID). + Msg("Failed to clear stale MatrixRTC membership during login load") + } + if markErr := wa.Main.DB.MatrixRTCCall.MarkEnded( + ctx, + wa.UserLogin.ID, + call.WACallID, + "ended", + "bridge_restart", + lastError, + time.Now(), + ); markErr != nil { + wa.UserLogin.Log.Err(markErr). + Str("call_id", call.WACallID). + Msg("Failed to mark stale MatrixRTC call ended during login load") + } + } + if len(calls) > 0 { + wa.UserLogin.Log.Info(). + Int("call_count", len(calls)). + Msg("Cleaned up stale MatrixRTC calls during login load") + } +} + func matrixRTCFinalEndReason(call *wadb.MatrixRTCCall, reason string) (string, string) { if call != nil && !call.EndedTS.IsZero() && (call.EndReason != "" || call.LastError != "") { if call.EndReason != "" { @@ -90,6 +133,8 @@ func matrixRTCFinalEndReason(call *wadb.MatrixRTCCall, reason string) (string, s } func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, call *meowcaller.Call) error { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() log := zerolog.Ctx(ctx).With(). Str("call_id", call.ID()). Stringer("peer_jid", call.Peer()). @@ -104,6 +149,17 @@ func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, cal log.Debug().Msg("No existing Matrix portal room for incoming MatrixRTC call announcement") return nil } + activeRoomCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveInRoom(ctx, portal.MXID) + if err != nil { + return err + } + if len(activeRoomCalls) > 0 { + log.Warn(). + Stringer("room_id", portal.MXID). + Int("active_call_count", len(activeRoomCalls)). + Msg("Rejecting incoming WhatsApp call because the Matrix room already has an active call") + return call.Reject() + } if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) if err != nil { @@ -170,12 +226,21 @@ func (wa *WhatsAppClient) joinMatrixRTCCallLink( tokenOrURL string, video bool, ) (*meowcaller.Call, error) { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() if portal == nil || portal.MXID == "" { return nil, fmt.Errorf("call links must be joined from an existing portal room") } if video && !wa.Main.Config.VOIP.Video.Enabled { return nil, fmt.Errorf("WhatsApp call-link video requires voip.video.enabled") } + activeRoomCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveInRoom(ctx, portal.MXID) + if err != nil { + return nil, err + } + if len(activeRoomCalls) > 0 { + return nil, fmt.Errorf("the Matrix room already has an active call") + } if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) if err != nil { @@ -314,6 +379,8 @@ func (wa *WhatsAppConnector) startOutboundMatrixRTCCall(ctx context.Context, por } func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal *bridgev2.Portal, trigger voip.MatrixRTCEvent) error { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() if wa.VOIP == nil || !wa.VOIP.Enabled() { return voip.ErrNotEnabled } From 36bd89088664949d48edb12e70ff7aa723a7b358 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:18:29 +0200 Subject: [PATCH 43/44] go.mod: tidy group call dependencies --- go.mod | 2 +- go.sum | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 9ba1a79..cf51840 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 github.com/livekit/protocol v1.49.0 github.com/livekit/server-sdk-go/v2 v2.17.0 + github.com/mattn/go-sqlite3 v1.14.48 github.com/pion/rtp v1.10.2 github.com/pion/webrtc/v4 v4.2.14 github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 @@ -66,7 +67,6 @@ require ( github.com/magefile/mage v1.17.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.48 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/nats.go v1.52.0 // indirect github.com/nats-io/nkeys v0.4.16 // indirect diff --git a/go.sum b/go.sum index eba43aa..bf5e17f 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,6 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8 h1:sVe8DiEWPu+oEkhbLUMPrTYXxaRQ/V3tUZld/5y2jYE= -github.com/purpshell/meowcaller v0.0.0-20260722143136-8faf887946a8/go.mod h1:FA3k7L98Dy2km9hvcdKJH1HMKPpi8bCONlA7R84NpN4= github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 h1:SnLX76CnagumooXRm63BK9Rn2/e/Th6aWWJJKTqOk2k= github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807/go.mod h1:kSME01MaSkwul6tSmExmgBWUOmfkw3DNpfnozsn01eE= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= @@ -249,8 +247,6 @@ go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391 h1:lsvBEY8MJfYdV61YbwikiQvb go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391/go.mod h1:xunp/oIQfFD68HHcNHfG0pOiHkvEtDhTweeIwKJ//+Q= go.mau.fi/webp v0.3.0 h1:gVHQZtz21Ziwj+CDuklbX9mqpsnDIFKxs/BJyV7iZzA= go.mau.fi/webp v0.3.0/go.mod h1:rlZFTev+dYxhvk+XNBP/5GcTt4gXmzAB4DU0aGUYIQo= -go.mau.fi/whatsmeow v0.0.0-20260720135917-a2381054887e h1:Kn2XQTpYRploXqtmJKjeX4ZYGevhIeQkO2PWGPGdwrY= -go.mau.fi/whatsmeow v0.0.0-20260720135917-a2381054887e/go.mod h1:Iy/xVSuVU2payR26MB1hv0UZUWRraEn4qKZ7+VRHulg= go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933 h1:7skZGs9q+rWKqYHok4ZufzhKpf6GmTKZnTjSm0aDtus= go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933/go.mod h1:Iy/xVSuVU2payR26MB1hv0UZUWRraEn4qKZ7+VRHulg= go.mau.fi/zeroconfig v0.2.0 h1:e/OGEERqVRRKlgaro7E6bh8xXiKFSXB3eNNIud7FUjU= From 3f8460006f6c84d77e3f79295262f5ecbc8709d1 Mon Sep 17 00:00:00 2001 From: Rajeh Taher Date: Thu, 30 Jul 2026 05:20:09 +0200 Subject: [PATCH 44/44] matrixrtc: announce call link waiting rooms --- pkg/connector/client.go | 3 ++ pkg/connector/matrixrtc_outgoing.go | 59 +++++++++++++++++++++++++++++ pkg/connector/matrixrtc_test.go | 26 +++++++++++++ pkg/connector/voip/manager.go | 18 +++++++++ 4 files changed, 106 insertions(+) diff --git a/pkg/connector/client.go b/pkg/connector/client.go index b2ae726..6f39a8c 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -95,6 +95,9 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. w.VOIP.SetHandRaiseHandler(func(callID string, state meowcaller.HandRaiseState) { go w.handleWhatsAppHandRaise(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) }) + w.VOIP.SetWaitingRoomHandler(func(callID string, state meowcaller.WaitingRoomState) { + go w.handleWhatsAppWaitingRoom(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) + }) w.Client.AddEventHandlerWithSuccessStatus(w.handleWAEvent) w.Client.SetForceActiveDeliveryReceipts(wa.Config.ForceActiveDeliveryReceipts) w.Client.InitialAutoReconnect = wa.Config.InitialAutoReconnect diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go index f87b718..2ddbc4e 100644 --- a/pkg/connector/matrixrtc_outgoing.go +++ b/pkg/connector/matrixrtc_outgoing.go @@ -3,6 +3,7 @@ package connector import ( "context" "fmt" + "slices" "strings" "time" @@ -122,6 +123,64 @@ func (wa *WhatsAppClient) cleanupStaleMatrixRTCCalls(ctx context.Context) { } } +func (wa *WhatsAppClient) handleWhatsAppWaitingRoom( + ctx context.Context, + callID string, + state meowcaller.WaitingRoomState, +) { + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil { + wa.UserLogin.Log.Err(err). + Str("call_id", callID). + Msg("Failed to load MatrixRTC call for WhatsApp waiting-room update") + return + } + if call == nil || !call.EndedTS.IsZero() || call.RoomID == "" { + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + _, err = intent.SendMessage(ctx, call.RoomID, event.EventMessage, &event.Content{ + Parsed: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: formatWaitingRoomNotice(state), + }, + }, nil) + if err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", callID). + Msg("Failed to send WhatsApp waiting-room update to Matrix") + } +} + +func formatWaitingRoomNotice(state meowcaller.WaitingRoomState) string { + approval := "disabled" + if state.Enabled { + approval = "enabled" + } + if state.InWaitingRoom { + return fmt.Sprintf("Waiting for approval to join the WhatsApp call link (approval %s).", approval) + } + if len(state.Users) == 0 { + return fmt.Sprintf("WhatsApp call-link waiting room is empty (approval %s).", approval) + } + participants := make([]string, 0, len(state.Users)) + for _, user := range state.Users { + identity := user.JID + if !user.PN.IsEmpty() { + identity = user.PN + } + participants = append(participants, identity.String()) + } + slices.Sort(participants) + return fmt.Sprintf( + "WhatsApp call-link waiting room has %d participant(s) pending (approval %s): %s", + len(participants), + approval, + strings.Join(participants, ", "), + ) +} + func matrixRTCFinalEndReason(call *wadb.MatrixRTCCall, reason string) (string, string) { if call != nil && !call.EndedTS.IsZero() && (call.EndReason != "" || call.LastError != "") { if call.EndReason != "" { diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go index c60305a..bb49ba6 100644 --- a/pkg/connector/matrixrtc_test.go +++ b/pkg/connector/matrixrtc_test.go @@ -3,6 +3,7 @@ package connector import ( "bytes" "encoding/json" + "strings" "testing" "time" @@ -121,6 +122,31 @@ func TestIncomingCallGroupIgnoresExpiredCorrelation(t *testing.T) { } } +func TestFormatWaitingRoomNotice(t *testing.T) { + waiting := formatWaitingRoomNotice(meowcaller.WaitingRoomState{ + Enabled: true, + InWaitingRoom: true, + }) + if !strings.Contains(waiting, "Waiting for approval") { + t.Fatalf("waiting-room self notice = %q", waiting) + } + + participants := formatWaitingRoomNotice(meowcaller.WaitingRoomState{ + Enabled: true, + Users: []meowcaller.WaitingRoomUser{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + }, + {JID: types.NewJID("111", types.HiddenUserServer)}, + }, + }) + if !strings.Contains(participants, "2 participant(s)") || + !strings.Contains(participants, "111@lid, 15550000002@s.whatsapp.net") { + t.Fatalf("waiting-room participant notice = %q", participants) + } +} + func TestShouldStartOutboundMatrixRTCCallRejectsMessageMembership(t *testing.T) { evt := matrixRTCMemberEvent(event.MessageEventType) parsed, ok := voip.ParseMatrixRTCEvent(evt) diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go index d5b390e..a5b96fd 100644 --- a/pkg/connector/voip/manager.go +++ b/pkg/connector/voip/manager.go @@ -57,6 +57,7 @@ type Manager struct { callEndNotify func(callID, reason string) callReactionNotify func(callID string, reaction meowcaller.CallReaction) handRaiseNotify func(callID string, state meowcaller.HandRaiseState) + waitingRoomNotify func(callID string, state meowcaller.WaitingRoomState) } func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Manager { @@ -145,6 +146,15 @@ func (m *Manager) SetHandRaiseHandler(handler func(callID string, state meowcall m.mu.Unlock() } +func (m *Manager) SetWaitingRoomHandler(handler func(callID string, state meowcaller.WaitingRoomState)) { + if m == nil { + return + } + m.mu.Lock() + m.waitingRoomNotify = handler + m.mu.Unlock() +} + func (m *Manager) SendReaction(callID, emoji string) error { if !m.Enabled() { return ErrNotEnabled @@ -938,6 +948,14 @@ func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { handler(call.ID(), state) } }) + call.OnWaitingRoomState(func(state meowcaller.WaitingRoomState) { + m.mu.Lock() + handler := m.waitingRoomNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), state) + } + }) } func parseVideoParticipantTarget(target string) (types.JID, error) {