diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c9fef73..343443b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -11,14 +11,14 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.26", "1.27"] - name: Lint ${{ matrix.go-version == '1.27' && '(latest)' || '(old)' }} + go-version: ["1.25", "1.26"] + name: Lint ${{ matrix.go-version == '1.26' && '(latest)' || '(old)' }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v6 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@v6 with: go-version: ${{ matrix.go-version }} cache: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d17865..e47e7b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,3 @@ -# v26.08 - -* Switched direct chats to use LIDs instead of phone numbers. - # v26.07 * Updated Docker image to Alpine 3.24. diff --git a/cmd/mautrix-whatsapp/main.go b/cmd/mautrix-whatsapp/main.go index 011e9b0..47d971c 100644 --- a/cmd/mautrix-whatsapp/main.go +++ b/cmd/mautrix-whatsapp/main.go @@ -18,7 +18,7 @@ var m = mxmain.BridgeMain{ Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "26.08", + Version: "26.07", SemCalVer: true, Connector: &connector.WhatsAppConnector{}, } diff --git a/go.mod b/go.mod index f7a6451..cf51840 100644 --- a/go.mod +++ b/go.mod @@ -1,54 +1,125 @@ module go.mau.fi/mautrix-whatsapp -go 1.26.0 +go 1.26 -toolchain go1.27.0 +toolchain go1.26.5 tool go.mau.fi/util/cmd/maubuild require ( github.com/lib/pq v1.12.3 + github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 + github.com/livekit/protocol v1.49.0 + github.com/livekit/server-sdk-go/v2 v2.17.0 + github.com/mattn/go-sqlite3 v1.14.48 + github.com/pion/rtp v1.10.2 + github.com/pion/webrtc/v4 v4.2.14 + github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 github.com/rs/zerolog v1.35.1 github.com/tidwall/gjson v1.19.0 - go.mau.fi/util v0.10.1-0.20260820140024-eb612d936fde + go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391 go.mau.fi/webp v0.3.0 - go.mau.fi/whatsmeow v0.0.0-20260821141805-33cfac511629 - golang.org/x/image v0.45.0 - golang.org/x/net v0.58.0 + go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/image v0.44.0 + golang.org/x/net v0.57.0 golang.org/x/sync v0.22.0 - google.golang.org/protobuf v1.36.12 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - maunium.net/go/mautrix v0.30.1-0.20260820142923-ae00f2c62e1e + maunium.net/go/mautrix v0.29.1-0.20260719130752-5743d9b6f27e ) require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + buf.build/go/protoyaml v0.7.0 // indirect + cel.dev/expr v0.25.2 // indirect filippo.io/edwards25519 v1.2.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/at-wat/ebml-go v0.18.0 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bep/debounce v1.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coder/websocket v1.8.15 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/dennwc/iters v1.2.2 // indirect github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect + github.com/frostbyte73/core v0.1.1 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/gammazero/deque v1.2.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/cel-go v0.28.1 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/hajimehoshi/go-mp3 v0.3.4 // indirect + github.com/jxskiss/base62 v1.1.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lithammer/shortuuid/v4 v4.2.0 // indirect + github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 // indirect + github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e // indirect + github.com/livekit/psrpc v0.7.2 // indirect + github.com/mackerelio/go-osstat v0.2.7 // indirect + github.com/magefile/mage v1.17.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.49 // indirect - github.com/petermattis/goid v0.0.0-20260816044145-ed329add6b1b // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca // indirect + github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/dtls/v3 v3.1.4 // indirect + github.com/pion/ice/v4 v4.2.7 // indirect + github.com/pion/interceptor v0.1.45 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/opus v0.1.0 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.16 // indirect + github.com/pion/sctp v1.10.0 // indirect + github.com/pion/sdp/v3 v3.0.18 // indirect + github.com/pion/srtp/v3 v3.0.11 // indirect + github.com/pion/stun/v3 v3.1.4 // indirect + github.com/pion/transport/v4 v4.0.2 // indirect + github.com/pion/turn/v5 v5.0.8 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.68.1 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/redis/go-redis/v9 v9.20.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + github.com/twitchtv/twirp v8.1.3+incompatible // indirect github.com/vektah/gqlparser/v2 v2.5.27 // indirect - github.com/yuin/goldmark v1.8.5 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + github.com/yuin/goldmark v1.8.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mau.fi/libsignal v0.2.2 // indirect go.mau.fi/zeroconfig v0.2.0 // indirect - golang.org/x/crypto v0.55.0 // indirect - golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect - golang.org/x/mod v0.40.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect + golang.org/x/mod v0.38.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect + gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 1adbb52..bf5e17f 100644 --- a/go.sum +++ b/go.sum @@ -1,55 +1,219 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/at-wat/ebml-go v0.18.0 h1:SNkpBFR4jCQV1rI4Bm1tSuIYnusxe2qQ4GHJia9eQg4= +github.com/at-wat/ebml-go v0.18.0/go.mod h1:w1cJs7zmGsb5nnSvhWGKLCxvfu4FVx5ERvYDIalj1ww= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= +github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elliotchance/orderedmap/v3 v3.1.0 h1:j4DJ5ObEmMBt/lcwIecKcoRxIQUEnw0L804lXYDt/pg= github.com/elliotchance/orderedmap/v3 v3.1.0/go.mod h1:G+Hc2RwaZvJMcS4JpGCOyViCnGeKf0bTYCGTO4uhjSo= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= +github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= +github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= +github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= +github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hajimehoshi/oto/v2 v2.3.1/go.mod h1:seWLbgHH7AyUMYKfKYT9pg7PhUu9/SisyJvNTT+ASQo= +github.com/jfreymuth/oggvorbis v1.0.5 h1:u+Ck+R0eLSRhgq8WTmffYnrVtSztJcYrl588DM4e3kQ= +github.com/jfreymuth/oggvorbis v1.0.5/go.mod h1:1U4pqWmghcoVsCJJ4fRBKv9peUJMBHixthRlBeD6uII= +github.com/jfreymuth/vorbis v1.0.2 h1:m1xH6+ZI4thH927pgKD8JOH4eaGRm18rEE9/0WKjvNE= +github.com/jfreymuth/vorbis v1.0.2/go.mod h1:DoftRo4AznKnShRl1GxiTFCseHr4zR9BN3TWXyuzrqQ= +github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= +github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= +github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97 h1:AyjUVuJuVd+5Kt+KEnIUoyZVAv8pejNDqs691I5e8jM= +github.com/livekit/media-sdk v0.0.0-20260605212526-4c11a51d3c97/go.mod h1:uWrLXY4JeLYynX39htMG49Dl4BhFYY+RCeoXaLdU+Lw= +github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e h1:SkgQRcG2VYEhh80Qb/zYZo8rWKJzNfJcfUQnXe6su2M= +github.com/livekit/mediatransportutil v0.0.0-20260605212259-862d4a7bcb1e/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= +github.com/livekit/protocol v1.49.0 h1:Q5nthDO1v7c0JHiWjMhgUQTlsKmCsBL/KCKxdHVaz00= +github.com/livekit/protocol v1.49.0/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= +github.com/livekit/psrpc v0.7.2 h1:6oZ+NODJ2pLyaT6VqDq1F4Qc/3TpDUSpyphj/P9MhQc= +github.com/livekit/psrpc v0.7.2/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= +github.com/livekit/server-sdk-go/v2 v2.17.0 h1:FzVQMoxHv0WIg164yGqSxLeV+h3aJomjAv1lFeR9MMw= +github.com/livekit/server-sdk-go/v2 v2.17.0/go.mod h1:5nzTfVBH2Jz+TW1SrfpqC7wrbcD1lT94KZCJ9hOMyvk= +github.com/mackerelio/go-osstat v0.2.7 h1:TCavZi10wF49bT6iQZ9eT2keGZQpC69MTDfdJej5e94= +github.com/mackerelio/go-osstat v0.2.7/go.mod h1:dwpYh5pIPmvk+IEwBKNIWRFMB92mrC08CmXOhDC7nQk= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= -github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= -github.com/petermattis/goid v0.0.0-20260816044145-ed329add6b1b h1:sS7HLzwS+dO+gxATgQfeZDEdUZe2pKAB3nGoUwP5zU0= -github.com/petermattis/goid v0.0.0-20260816044145-ed329add6b1b/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= +github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= +github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoPME= +github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= +github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca h1:GHSUVE4yOgX4E7kTRzpxCPbCOYkd3Kj8Dgdod30OI1E= +github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0= +github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk= +github.com/pion/dtls/v3 v3.1.4 h1:QhvtMflMfu9Kf0RcDC5BJBle4caPskByrKQR6uuYqpY= +github.com/pion/dtls/v3 v3.1.4/go.mod h1:cr/qotLISUw/9C1m83ZPNZtj9WnXkYLpfCptPqbkInc= +github.com/pion/ice/v4 v4.2.7 h1:zDEbC6MiEdhQpF8TxBOTws+NU6ZgGpveHrQq4Lc1kao= +github.com/pion/ice/v4 v4.2.7/go.mod h1:9SNPaq0c7El/ki8leJzyCkK10zsskprR3zTNbO3monY= +github.com/pion/interceptor v0.1.45 h1:6PUo/5829bIfRFIPPJQzuDn8EjxRTSB/CSD7QVCOaqo= +github.com/pion/interceptor v0.1.45/go.mod h1:gNDYM/uFKcLe/B3gS2/7+aw6z+RDiMy2qKTnF1LO31w= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= +github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= +github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= +github.com/pion/opus v0.1.0 h1:GgK/a3DNDrffKjUFsK39rZKqfv7bQ2S2eqRKt0BnqAE= +github.com/pion/opus v0.1.0/go.mod h1:t5Xog2n682JnawoykACE6nKVmupFvmJvkpM7x6bTv6g= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= +github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= +github.com/pion/rtp v1.10.2 h1:l+f6tTDcAH6xwepaAoW791ddhuYsJlqRATOzirO04Mo= +github.com/pion/rtp v1.10.2/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= +github.com/pion/sctp v1.10.0 h1:qeoD6swF/2M5bYRcAGayqSbTKX3m4AW29CiQxG1+Pfg= +github.com/pion/sctp v1.10.0/go.mod h1:N20Dq6LY+JvJDAh9VVh1JELngb2rQ8dPgds5yBWiPgw= +github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= +github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= +github.com/pion/srtp/v3 v3.0.11 h1:GiESUr54/K4UuPigfq/CvWUed80JenQAHXn0C2MQQIQ= +github.com/pion/srtp/v3 v3.0.11/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/stun/v3 v3.1.4 h1:/7ZL0j0dmLroKOq4GfkyKQ6asByYqntwyHSp5sYLcGY= +github.com/pion/stun/v3 v3.1.4/go.mod h1:ET7PFiXo1nrD2ZNVpbEHDuT0kCPVXhKmyWdiePNMw/U= +github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= +github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= +github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= +github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= +github.com/pion/turn/v5 v5.0.8 h1:pZUCtmwWCMkrRKqh/8pL3WoGADXBe0/lOPkN7oqFjK8= +github.com/pion/turn/v5 v5.0.8/go.mod h1:1VwvxElZaOdJU0liJ/WUSm/Tsh+n2OxS5ISSDxgOWxU= +github.com/pion/webrtc/v4 v4.2.14 h1:Q6zMs+fSDsYuhZcNlvFGBxCOMHVV9oYcDa6O9/HIGTc= +github.com/pion/webrtc/v4 v4.2.14/go.mod h1:87NVKP86+g4OMrRxWhjWfUjeXP4JrV6RTlUrIW+/Jak= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pStaY= +github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807 h1:SnLX76CnagumooXRm63BK9Rn2/e/Th6aWWJJKTqOk2k= +github.com/purpshell/meowcaller v0.0.0-20260726180203-6d9b7b2c1807/go.mod h1:kSME01MaSkwul6tSmExmgBWUOmfkw3DNpfnozsn01eE= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= +github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -65,47 +229,90 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= +github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTdwFp0s= github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= -github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA= -github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/yuin/goldmark v1.8.4 h1:oat/nd3U6NeQqFEL3xpEJq7d7c86NI+DbSNGAs4xnjA= +github.com/yuin/goldmark v1.8.4/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mau.fi/libsignal v0.2.2 h1:QV+XdzQkm3x3aSG7FcqfGSZuFXz83pRZPBFaPygHbOU= go.mau.fi/libsignal v0.2.2/go.mod h1:CRlIQg2J8uYTfDFvNoO8/KcZjs5cey0vbc6oj/bssY0= -go.mau.fi/util v0.10.1-0.20260820140024-eb612d936fde h1:eMHY9dMDkNuDMWhfTbMZHbbsxj7G6mfujjKei1HaFQM= -go.mau.fi/util v0.10.1-0.20260820140024-eb612d936fde/go.mod h1:z0ZZNt4hq3FZbUKnunexE/QscCx7VkLvQSvtggc/aE8= +go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391 h1:lsvBEY8MJfYdV61YbwikiQvb0Al/onbmLW5wfl/0tag= +go.mau.fi/util v0.9.12-0.20260719092501-f9c03d846391/go.mod h1:xunp/oIQfFD68HHcNHfG0pOiHkvEtDhTweeIwKJ//+Q= go.mau.fi/webp v0.3.0 h1:gVHQZtz21Ziwj+CDuklbX9mqpsnDIFKxs/BJyV7iZzA= go.mau.fi/webp v0.3.0/go.mod h1:rlZFTev+dYxhvk+XNBP/5GcTt4gXmzAB4DU0aGUYIQo= -go.mau.fi/whatsmeow v0.0.0-20260821141805-33cfac511629 h1:99p9fTS1G73a1aifCVaV5MX4hDq0lVUKWGoUNoXyE3A= -go.mau.fi/whatsmeow v0.0.0-20260821141805-33cfac511629/go.mod h1:aMd13H2xFFGH9cskcvxo4Aae+TmyFN38yw+HvsrpwVg= +go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933 h1:7skZGs9q+rWKqYHok4ZufzhKpf6GmTKZnTjSm0aDtus= +go.mau.fi/whatsmeow v0.0.0-20260722203353-e9a033b24933/go.mod h1:Iy/xVSuVU2payR26MB1hv0UZUWRraEn4qKZ7+VRHulg= go.mau.fi/zeroconfig v0.2.0 h1:e/OGEERqVRRKlgaro7E6bh8xXiKFSXB3eNNIud7FUjU= go.mau.fi/zeroconfig v0.2.0/go.mod h1:J0Vn0prHNOm493oZoQ84kq83ZaNCYZnq+noI1b1eN8w= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= -golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= -golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= -golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= -golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= -golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= +golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302 h1:xeVptzkP8BuJhoIjNizd2bRHfq9KB9HfOLZu90T04XM= +gopkg.in/hraban/opus.v2 v2.0.0-20230925203106-0188a62cb302/go.mod h1:/L5E7a21VWl8DeuCPKxQBdVG5cy+L0MRZ08B1wnqt7g= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= -maunium.net/go/mautrix v0.30.1-0.20260820142923-ae00f2c62e1e h1:ATvERyBIUXJrnL7ezP2xCLLGCNmGClmfzoQq4N3jBcA= -maunium.net/go/mautrix v0.30.1-0.20260820142923-ae00f2c62e1e/go.mod h1:Y02sBiAvfEVqK24bwVGCprmLATRZ7prWel3ZpB413e0= +maunium.net/go/mautrix v0.29.1-0.20260719130752-5743d9b6f27e h1:tPGnL/s5dfqhNoVLvmCKY3V60migQsNHBXAhUALBSd8= +maunium.net/go/mautrix v0.29.1-0.20260719130752-5743d9b6f27e/go.mod h1:PeLuIih5jnbwb2xKNNi0Te7AtINEQE2Uv3p6iG3RV80= diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index bb555e6..dd04696 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -12,7 +12,6 @@ import ( "time" "github.com/rs/zerolog" - "go.mau.fi/util/exmaps" "go.mau.fi/util/ptr" "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/proto/waE2E" @@ -229,32 +228,30 @@ func (wa *WhatsAppClient) handleWAHistorySync( } else { totalMessageCount += len(conv.GetMessages()) } - if jid.Server == types.DefaultUserServer { - lid, err := wa.GetStore().LIDs.GetLIDForPN(ctx, jid) + if jid.Server == types.HiddenUserServer { + pn, err := wa.GetStore().LIDs.GetPNForLID(ctx, jid) if err != nil { - log.Err(err).Stringer("pn", jid).Msg("Failed to get LID for phone number in history sync") - } else if lid.IsEmpty() { - log.Warn().Stringer("pn", jid).Msg("No LID found for phone number in history sync") + log.Err(err).Stringer("lid", jid).Msg("Failed to get PN for LID in history sync") + } else if pn.IsEmpty() { + log.Warn().Stringer("lid", jid).Msg("No PN found for LID in history sync") } else { log.Debug(). - Stringer("lid", lid). - Stringer("pn", jid). - Msg("Rerouting phone number DM to LID in history sync") - jid = lid + Stringer("lid", jid). + Stringer("pn", pn). + Msg("Rerouting LID DM to phone number in history sync") + jid = pn } } log.UpdateContext(func(c zerolog.Context) zerolog.Context { return c.Stringer("chat_jid", jid) }) - var firstItemTime, lastItemTime time.Time + var minTime, maxTime, firstItemTime, lastItemTime time.Time + var minTimeIndex, maxTimeIndex int ignoredTypes := 0 - rawMessages := conv.GetMessages() - messages := make([]*wadb.HistorySyncMessageTuple, 0, len(rawMessages)) - allowClamp := conv.GetCommentsCount() == 0 - var newerTS uint64 - for i, rawMsg := range rawMessages { + messages := make([]*wadb.HistorySyncMessageTuple, 0, len(conv.GetMessages())) + for i, rawMsg := range conv.GetMessages() { // Don't store messages that will just be skipped. msgEvt, err := wa.Client.ParseWebMessage(jid, rawMsg.GetMessage()) if err != nil { @@ -272,30 +269,20 @@ func (wa *WhatsAppClient) handleWAHistorySync( firstItemTime = msgEvt.Info.Timestamp } lastItemTime = msgEvt.Info.Timestamp + if minTime.IsZero() || msgEvt.Info.Timestamp.Before(minTime) { + minTime = msgEvt.Info.Timestamp + minTimeIndex = i + } + if maxTime.IsZero() || msgEvt.Info.Timestamp.After(maxTime) { + maxTime = msgEvt.Info.Timestamp + maxTimeIndex = i + } msgType := getMessageType(msgEvt.Message) if msgType == "ignore" || strings.HasPrefix(msgType, "unknown_protocol_") { ignoredTypes++ continue } - // Comments (replies) in announcement groups are not ordered by timestamp, so don't clamp them. - if rawMsg.GetMessage().GetCommentMetadata().GetCommentParentKey() != nil { - allowClamp = false - } - // WhatsApp has bugs where some random messages will have timestamps decades in the future. - // To ensure they don't mess up our ordering, require timestamps of older messages to be - // before the previous (newer) message. - if currentTS := rawMsg.GetMessage().GetMessageTimestamp(); newerTS > 0 && allowClamp && currentTS > newerTS { - log.Warn(). - Time("current_ts", time.Unix(int64(currentTS), 0)). - Time("prev_ts", time.Unix(int64(newerTS), 0)). - Int("msg_index", i). - Str("msg_id", rawMsg.GetMessage().GetKey().GetID()). - Msg("Clamping message timestamp") - rawMsg.Message.MessageTimestamp = ptr.Ptr(newerTS) - msgEvt.Info.Timestamp = time.Unix(int64(newerTS), 0) - } - newerTS = rawMsg.GetMessage().GetMessageTimestamp() marshaled, err := proto.Marshal(rawMsg) if err != nil { log.Warn().Err(err). @@ -310,8 +297,13 @@ func (wa *WhatsAppClient) handleWAHistorySync( log.Debug(). Int("wrapped_count", len(messages)). Int("ignored_msg_type_count", ignoredTypes). + Time("lowest_time", minTime). + Int("lowest_time_index", minTimeIndex). + Time("highest_time", maxTime). + Int("highest_time_index", maxTimeIndex). Time("first_item_time", firstItemTime). Time("last_item_time", lastItemTime). + Bool("highest_time_mismatch", firstItemTime != maxTime). Dict("metadata", zerolog.Dict(). Uint32("ephemeral_expiration", conv.GetEphemeralExpiration()). Int64("ephemeral_setting_timestamp", conv.GetEphemeralSettingTimestamp()). @@ -327,7 +319,7 @@ func (wa *WhatsAppClient) handleWAHistorySync( Msg("Collected messages to save from history sync conversation") if len(messages) > 0 { - err = wa.Main.DB.Conversation.Put(ctx, wadb.NewConversation(wa.UserLogin.ID, jid, conv, firstItemTime)) + err = wa.Main.DB.Conversation.Put(ctx, wadb.NewConversation(wa.UserLogin.ID, jid, conv, maxTime)) if err != nil { if stopOnError { return fmt.Errorf("failed to save conversation metadata for %s: %w", jid, err) @@ -482,9 +474,6 @@ func (wa *WhatsAppClient) FetchMessages(ctx context.Context, params bridgev2.Fet if err != nil { return nil, err } - if portalJID.Server == types.DefaultUserServer { - zerolog.Ctx(ctx).Warn().Stringer("portal_jid", portalJID).Msg("FetchMessages called for phone number portal") - } var markRead bool var startTime, endTime *time.Time var conv *wadb.Conversation @@ -539,7 +528,7 @@ func (wa *WhatsAppClient) FetchMessages(ctx context.Context, params bridgev2.Fet Forward: params.Forward, }, nil } else if hasMore { - return wa.fetchMessagesFromPhone(ctx, portalJID, params) + return wa.fetchMessagesFromPhone(ctx, params) } return &bridgev2.FetchMessagesResponse{ HasMore: false, @@ -605,7 +594,6 @@ func (wa *WhatsAppClient) convertHistorySyncMessages( newestTS := messages[0].GetMessageTimestamp() convertedMessages := make([]*bridgev2.BackfillMessage, 0, len(messages)) var mediaRequests []*wadb.MediaRequest - dups := make(exmaps.Set[networkid.MessageID]) for i, msg := range messages { evt, err := wa.Client.ParseWebMessage(portalJID, msg) if err != nil { @@ -629,18 +617,10 @@ func (wa *WhatsAppClient) convertHistorySyncMessages( continue } } - if !wa.ensureAltJIDs(ctx, &evt.Info.MessageSource, false) { - return nil, fmt.Errorf("failed to ensure alt JIDs for message %s", evt.Info.ID) - } isViewOnce := evt.IsViewOnce || evt.IsViewOnceV2 || evt.IsViewOnceV2Extension converted, mediaReq := wa.convertHistorySyncMessage( ctx, portal, &evt.Info, evt.Message, evt.RawMessage, isViewOnce, msg.Reactions, ) - // This is a hack to remove duplicates where the same message is inserted with both the LID and phone number sender - // TODO prevent those being inserted in the first place instead of hacking around it here - if !dups.Add(converted.ID) { - continue - } convertedMessages = append(convertedMessages, converted) if mediaReq != nil { mediaRequests = append(mediaRequests, mediaReq) @@ -671,7 +651,7 @@ func (wa *WhatsAppClient) convertHistorySyncMessages( }, nil } -func (wa *WhatsAppClient) fetchMessagesFromPhone(ctx context.Context, portalJID types.JID, params bridgev2.FetchMessagesParams) (*bridgev2.FetchMessagesResponse, error) { +func (wa *WhatsAppClient) fetchMessagesFromPhone(ctx context.Context, params bridgev2.FetchMessagesParams) (*bridgev2.FetchMessagesResponse, error) { if params.AnchorMessage == nil { return nil, fmt.Errorf("anchor message is required to fetch messages from phone") } @@ -683,9 +663,9 @@ func (wa *WhatsAppClient) fetchMessagesFromPhone(ctx context.Context, portalJID msgID := wa.Client.GenerateMessageID() reqData := wa.Client.BuildHistorySyncRequest(&types.MessageInfo{ MessageSource: types.MessageSource{ - Chat: portalJID, + Chat: parsed.Chat, Sender: parsed.Sender, - IsFromMe: wa.IsOwnJID(parsed.Sender), + IsFromMe: parsed.Sender.ToNonAD() == wa.JID.ToNonAD() || parsed.Sender.ToNonAD() == wa.Device.GetLID().ToNonAD(), IsGroup: parsed.Chat.Server == types.GroupServer, }, ID: parsed.ID, @@ -693,7 +673,6 @@ func (wa *WhatsAppClient) fetchMessagesFromPhone(ctx context.Context, portalJID }, 50) zerolog.Ctx(ctx).Debug(). Str("request_msg_id", msgID). - Stringer("portal_jid", portalJID). Any("anchor_msg_parsed", parsed). Any("request_data", reqData). Msg("Sending history sync request") @@ -722,20 +701,6 @@ func (wa *WhatsAppClient) handleOnDemandHistorySync(ctx context.Context, blob *w zerolog.Ctx(ctx).Err(err).Str("jid", conv.GetID()).Msg("Failed to parse portal JID") continue } - if portalJID.Server == types.DefaultUserServer { - lid, err := wa.GetStore().LIDs.GetLIDForPN(ctx, portalJID) - if err != nil { - zerolog.Ctx(ctx).Err(err).Stringer("lid", portalJID).Msg("Failed to get LID for phone number in on-demand history sync") - } else if lid.IsEmpty() { - zerolog.Ctx(ctx).Warn().Stringer("lid", portalJID).Msg("No LID found for phone number in on-demand history sync") - } else { - zerolog.Ctx(ctx).Debug(). - Stringer("lid", lid). - Stringer("pn", portalJID). - Msg("Rerouting phone number DM to LID in on-demand history sync") - portalJID = lid - } - } portal, err := wa.Main.Bridge.GetPortalByKey(ctx, wa.makeWAPortalKey(portalJID)) if err != nil { zerolog.Ctx(ctx).Err(err).Stringer("portal_jid", portalJID).Msg("Failed to get portal for on-demand history sync") @@ -784,12 +749,11 @@ func (wa *WhatsAppClient) convertHistorySyncMessage( } // TODO use proper intent intent := wa.Main.Bridge.Bot - msgID := waid.MakeMessageIDWithAltSender(info.Chat, info.Sender, info.SenderAlt, info.ID) wrapped := &bridgev2.BackfillMessage{ - ConvertedMessage: wa.Main.MsgConv.ToMatrix(ctx, portal, wa.Client, intent, msg, rawMsg, info, isViewOnce, true, nil), - Sender: wa.makeEventSender(ctx, pickLID(info.Sender, info.SenderAlt)), - ID: msgID, - TxnID: networkid.TransactionID(msgID), + ConvertedMessage: wa.Main.MsgConv.ToMatrix(ctx, portal, wa.Client, intent, msg, rawMsg, info, nil, isViewOnce, true, nil), + Sender: wa.makeEventSender(ctx, info.Sender), + ID: waid.MakeMessageID(info.Chat, info.Sender, info.ID), + TxnID: networkid.TransactionID(waid.MakeMessageID(info.Chat, info.Sender, info.ID)), Timestamp: info.Timestamp, StreamOrder: info.Timestamp.Unix(), Reactions: make([]*bridgev2.BackfillReaction, 0, len(reactions)), @@ -798,10 +762,10 @@ func (wa *WhatsAppClient) convertHistorySyncMessage( for _, reaction := range reactions { var sender types.JID if reaction.GetKey().GetFromMe() { - sender = wa.GetLID() + sender = wa.JID } else if reaction.GetKey().GetParticipant() != "" { sender, _ = types.ParseJID(*reaction.Key.Participant) - } else if info.Chat.Server == types.DefaultUserServer || info.Chat.Server == types.HiddenUserServer || info.Chat.Server == types.BotServer { + } else if info.Chat.Server == types.DefaultUserServer || info.Chat.Server == types.BotServer { sender = info.Chat } if sender.IsEmpty() { diff --git a/pkg/connector/capabilities.go b/pkg/connector/capabilities.go index 152e7ad..3e6f658 100644 --- a/pkg/connector/capabilities.go +++ b/pkg/connector/capabilities.go @@ -52,7 +52,7 @@ func (wa *WhatsAppConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilit } func (wa *WhatsAppConnector) GetBridgeInfoVersion() (info, caps int) { - return 1, 9 + return 1, 8 } const WAMaxFileSize = 2000 * 1024 * 1024 @@ -67,7 +67,7 @@ func supportedIfFFmpeg() event.CapabilitySupportLevel { } func capID() string { - base := "fi.mau.whatsapp.capabilities.2026_07_22" + base := "fi.mau.whatsapp.capabilities.2026_05_12" if ffmpeg.Supported() { return base + "+ffmpeg" } @@ -174,26 +174,21 @@ var whatsappCaps = &event.RoomFeatures{ event.MemberActionKick: event.CapLevelFullySupported, event.MemberActionLeave: event.CapLevelFullySupported, }, - MaxTextLength: MaxTextLength, - LocationMessage: event.CapLevelFullySupported, - Poll: event.CapLevelFullySupported, - PollEnd: event.CapLevelUnsupported, - PollHiddenVotes: event.CapLevelUnsupported, - PollDuplicateOptions: event.CapLevelUnsupported, - PollMaxOptions: 12, - PollOptionMaxLength: 100, - Reply: event.CapLevelFullySupported, - Edit: event.CapLevelFullySupported, - EditMaxAge: ptr.Ptr(jsontime.S(EditMaxAge)), - Delete: event.CapLevelFullySupported, - DeleteForMe: false, - DeleteMaxAge: ptr.Ptr(jsontime.S(2 * 24 * time.Hour)), - Reaction: event.CapLevelFullySupported, - ReactionCount: 1, - ReadReceipts: true, - TypingNotifications: true, - DisappearingTimer: waDisappearingCap, - DeleteChat: true, + MaxTextLength: MaxTextLength, + LocationMessage: event.CapLevelFullySupported, + Poll: event.CapLevelFullySupported, + Reply: event.CapLevelFullySupported, + Edit: event.CapLevelFullySupported, + EditMaxAge: ptr.Ptr(jsontime.S(EditMaxAge)), + Delete: event.CapLevelFullySupported, + DeleteForMe: false, + DeleteMaxAge: ptr.Ptr(jsontime.S(2 * 24 * time.Hour)), + Reaction: event.CapLevelFullySupported, + ReactionCount: 1, + ReadReceipts: true, + TypingNotifications: true, + DisappearingTimer: waDisappearingCap, + DeleteChat: true, } var whatsappDMCaps *event.RoomFeatures diff --git a/pkg/connector/chatinfo.go b/pkg/connector/chatinfo.go index 1461561..7ffbb14 100644 --- a/pkg/connector/chatinfo.go +++ b/pkg/connector/chatinfo.go @@ -99,18 +99,6 @@ func (wa *WhatsAppClient) applyChatSettings(ctx context.Context, chatID types.JI zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to get chat settings") return } - if !chat.Found { - chatID, err = wa.GetStore().GetAltJID(ctx, chatID) - if err != nil { - zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to get alternate JID to get chat settings") - return - } - chat, err = wa.GetStore().ChatSettings.GetChatSettings(ctx, chatID) - if err != nil { - zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to get chat settings with alternate JID") - return - } - } info.UserLocal = &bridgev2.UserLocalPortalInfo{ MutedUntil: ptr.Ptr(chat.MutedUntil), } @@ -153,20 +141,15 @@ const PrivateChatTopic = "WhatsApp private chat" const BotChatTopic = "WhatsApp chat with a bot" func (wa *WhatsAppClient) wrapDMInfo(ctx context.Context, jid types.JID) *bridgev2.ChatInfo { - ownID := wa.JID - if jid.Server == types.HiddenUserServer { - ownID = wa.GetLID() - } info := &bridgev2.ChatInfo{ - Type: ptr.Ptr(database.RoomTypeDM), Topic: ptr.Ptr(PrivateChatTopic), Members: &bridgev2.ChatMemberList{ IsFull: true, TotalMemberCount: 2, OtherUserID: waid.MakeUserID(jid), MemberMap: map[networkid.UserID]bridgev2.ChatMember{ - waid.MakeUserID(jid): {EventSender: wa.makeEventSender(ctx, jid)}, - waid.MakeUserID(ownID): {EventSender: wa.makeEventSender(ctx, ownID)}, + waid.MakeUserID(jid): {EventSender: wa.makeEventSender(ctx, jid)}, + waid.MakeUserID(wa.JID): {EventSender: wa.makeEventSender(ctx, wa.JID)}, }, PowerLevels: &bridgev2.PowerLevelOverrides{ Events: map[event.Type]int{ @@ -176,14 +159,13 @@ func (wa *WhatsAppClient) wrapDMInfo(ctx context.Context, jid types.JID) *bridge event.StateBeeperDisappearingTimer: 0, }, }, - ExcludeChangesFromTimeline: true, }, - ExcludeChangesFromTimeline: true, + Type: ptr.Ptr(database.RoomTypeDM), } if jid.Server == types.BotServer { info.Topic = ptr.Ptr(BotChatTopic) } - if wa.IsOwnJID(jid) { + if jid == wa.JID.ToNonAD() { // For chats with self, force-split the members so the user's own ghost is always in the room. info.Members.MemberMap = map[networkid.UserID]bridgev2.ChatMember{ waid.MakeUserID(jid): {EventSender: bridgev2.EventSender{Sender: waid.MakeUserID(jid)}}, @@ -207,7 +189,7 @@ func (wa *WhatsAppClient) wrapStatusBroadcastInfo(ctx context.Context) *bridgev2 Members: &bridgev2.ChatMemberList{ IsFull: false, MemberMap: map[networkid.UserID]bridgev2.ChatMember{ - waid.MakeUserID(wa.GetLID()): {EventSender: wa.makeEventSender(ctx, wa.GetLID())}, + waid.MakeUserID(wa.JID): {EventSender: wa.makeEventSender(ctx, wa.JID)}, }, }, Type: ptr.Ptr(database.RoomTypeDefault), @@ -284,13 +266,14 @@ func (wa *WhatsAppClient) wrapGroupInfo(ctx context.Context, info *types.GroupIn Ban: ptr.Ptr(nobodyPL), // TODO allow invites if bridge config says to allow them, or maybe if relay mode is enabled? Events: map[event.Type]int{ - event.StateRoomName: metaChangePL, - event.StateRoomAvatar: metaChangePL, - event.StateTopic: metaChangePL, - event.EventReaction: defaultPL, - event.EventRedaction: defaultPL, - event.EventUnstablePollResponse: defaultPL, + event.StateRoomName: metaChangePL, + event.StateRoomAvatar: metaChangePL, + event.StateTopic: metaChangePL, + event.EventReaction: defaultPL, + event.EventRedaction: defaultPL, + event.StateBeeperDisappearingTimer: metaChangePL, + // TODO always allow poll responses }, }, }, @@ -333,7 +316,7 @@ func (wa *WhatsAppClient) wrapGroupInfo(ctx context.Context, info *types.GroupIn } } if info.IsParent && !hasSelf && info.AddressingMode == types.AddressingModeLID { - wrapped.Members.MemberMap.Add(bridgev2.ChatMember{EventSender: wa.makeEventSender(ctx, wa.GetLID())}) + wrapped.Members.MemberMap.Add(bridgev2.ChatMember{EventSender: wa.makeEventSender(ctx, wa.Device.LID)}) } if !info.LinkedParentJID.IsEmpty() { @@ -540,8 +523,8 @@ func (wa *WhatsAppClient) wrapNewsletterInfo(ctx context.Context, info *types.Ne Members: &bridgev2.ChatMemberList{ TotalMemberCount: info.ThreadMeta.SubscriberCount, MemberMap: map[networkid.UserID]bridgev2.ChatMember{ - waid.MakeUserID(wa.GetLID()): { - EventSender: wa.makeEventSender(ctx, wa.GetLID()), + waid.MakeUserID(wa.JID): { + EventSender: wa.makeEventSender(ctx, wa.JID), PowerLevel: &ownPowerLevel, }, }, @@ -550,12 +533,12 @@ func (wa *WhatsAppClient) wrapNewsletterInfo(ctx context.Context, info *types.Ne StateDefault: ptr.Ptr(nobodyPL), Ban: ptr.Ptr(nobodyPL), Events: map[event.Type]int{ - event.StateRoomName: adminPL, - event.StateRoomAvatar: adminPL, - event.StateTopic: adminPL, - event.EventReaction: defaultPL, - event.EventRedaction: defaultPL, - event.EventUnstablePollResponse: defaultPL, + event.StateRoomName: adminPL, + event.StateRoomAvatar: adminPL, + event.StateTopic: adminPL, + event.EventReaction: defaultPL, + event.EventRedaction: defaultPL, + // TODO always allow poll responses }, }, }, diff --git a/pkg/connector/client.go b/pkg/connector/client.go index b3ecdba..6f39a8c 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -24,6 +24,7 @@ import ( "sync/atomic" "time" + "github.com/purpshell/meowcaller" "github.com/rs/zerolog" "go.mau.fi/util/exsync" "go.mau.fi/whatsmeow" @@ -40,6 +41,7 @@ import ( "maunium.net/go/mautrix/bridgev2/status" "maunium.net/go/mautrix/event" + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" "go.mau.fi/mautrix-whatsapp/pkg/waid" ) @@ -54,9 +56,9 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. directMediaRetries: make(map[networkid.MessageID]*directMediaRetry), mediaRetryLock: semaphore.NewWeighted(wa.Config.HistorySync.MediaRequests.MaxAsyncHandle), pushNamesSynced: exsync.NewEvent(), + createDedup: exsync.NewSet[types.MessageID](), appStateFullSyncAttempted: make(map[appstate.WAPatchName]time.Time), - - disableNewsletter: store.BaseClientPayload.GetUserAgent().GetPlatform() == waWa6.ClientPayload_UserAgent_MACOS, + incomingCallGroups: make(map[string]incomingCallGroup), } login.Client = w @@ -71,12 +73,11 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. if err != nil { return err } - w.LID = w.Device.GetLID() if w.Device != nil { log := w.UserLogin.Log.With().Str("component", "whatsmeow").Logger() w.Client = whatsmeow.NewClient(w.Device, waLog.Zerolog(log)) - w.Client.AddEventHandlerWithSuccessStatus(w.handleWAEvent) + w.Client.AddEventHandler(w.trackIncomingCallEvent) w.Client.SynchronousAck = true w.Client.EnableDecryptedEventBuffer = bridgev2.PortalEventBuffer == 0 w.Client.ManualHistorySyncDownload = true @@ -85,6 +86,19 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. w.Client.GetMessageForRetry = w.trackNotFoundRetry w.Client.PreRetryCallback = w.trackFoundRetry w.Client.BackgroundEventCtx = w.UserLogin.Log.WithContext(wa.Bridge.BackgroundCtx) + w.VOIP = voip.NewManager(w.Client, makeVOIPConfig(wa.Config.VOIP), w.UserLogin.Log.With().Str("component", "voip").Logger()) + w.VOIP.SetIncomingCallHandler(w.handleIncomingVOIPCall) + w.VOIP.SetCallEndHandler(w.handleVOIPCallEnded) + w.VOIP.SetCallReactionHandler(func(callID string, reaction meowcaller.CallReaction) { + go w.handleWhatsAppCallReaction(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, reaction) + }) + w.VOIP.SetHandRaiseHandler(func(callID string, state meowcaller.HandRaiseState) { + go w.handleWhatsAppHandRaise(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) + }) + w.VOIP.SetWaitingRoomHandler(func(callID string, state meowcaller.WaitingRoomState) { + go w.handleWhatsAppWaitingRoom(withoutCancelOrBackground(w.Main.Bridge.BackgroundCtx), callID, state) + }) + w.Client.AddEventHandlerWithSuccessStatus(w.handleWAEvent) w.Client.SetForceActiveDeliveryReceipts(wa.Config.ForceActiveDeliveryReceipts) w.Client.InitialAutoReconnect = wa.Config.InitialAutoReconnect w.Client.UseRetryMessageStore = wa.Config.UseWhatsAppRetryStore @@ -104,25 +118,30 @@ type WhatsAppClient struct { Main *WhatsAppConnector UserLogin *bridgev2.UserLogin Client *whatsmeow.Client + VOIP *voip.Manager Device *store.Device JID types.JID - LID types.JID MC mClient - historySyncWakeup chan struct{} - stopLoops atomic.Pointer[context.CancelFunc] - resyncQueue map[types.JID]resyncQueueItem - resyncQueueLock sync.Mutex - nextResync time.Time - directMediaRetries map[networkid.MessageID]*directMediaRetry - directMediaLock sync.Mutex - mediaRetryLock *semaphore.Weighted - offlineSyncWaiter atomic.Pointer[chan error] - isNewLogin bool - pushNamesSynced *exsync.Event - lastPresence types.Presence - - disableNewsletter bool + historySyncWakeup chan struct{} + stopLoops atomic.Pointer[context.CancelFunc] + resyncQueue map[types.JID]resyncQueueItem + resyncQueueLock sync.Mutex + nextResync time.Time + directMediaRetries map[networkid.MessageID]*directMediaRetry + directMediaLock sync.Mutex + voipHandBridgeLock sync.Mutex + voipHandRaiseLock sync.Mutex + voipHandRaises map[string]map[types.JID]bool + voipCallStartLock sync.Mutex + incomingCallGroupLock sync.Mutex + incomingCallGroups map[string]incomingCallGroup + mediaRetryLock *semaphore.Weighted + offlineSyncWaiter atomic.Pointer[chan error] + isNewLogin bool + pushNamesSynced *exsync.Event + lastPresence types.Presence + createDedup *exsync.Set[types.MessageID] appStateRecoveryLock sync.Mutex appStateFullSyncAttempted map[appstate.WAPatchName]time.Time @@ -189,19 +208,7 @@ func (wa *WhatsAppClient) RegisterPushNotifications(ctx context.Context, pushTyp } func (wa *WhatsAppClient) IsThisUser(_ context.Context, userID networkid.UserID) bool { - return userID == waid.MakeUserID(wa.JID) || userID == waid.MakeUserID(wa.GetLID()) -} - -func (wa *WhatsAppClient) IsOwnJID(jid types.JID) bool { - return (jid.Server == types.DefaultUserServer && jid.User == wa.JID.User) || - (jid.Server == types.HiddenUserServer && jid.User == wa.GetLID().User) -} - -func (wa *WhatsAppClient) GetLID() types.JID { - if wa.LID.IsEmpty() && !wa.JID.IsEmpty() { - wa.LID = wa.GetStore().GetLID() - } - return wa.LID + return userID == waid.MakeUserID(wa.JID) } func (wa *WhatsAppClient) Connect(ctx context.Context) { @@ -221,6 +228,7 @@ func (wa *WhatsAppClient) Connect(ctx context.Context) { if ctx.Err() != nil { return } + wa.cleanupStaleMatrixRTCCalls(ctx) wa.initMC() wa.startLoops() wa.Client.BackgroundEventCtx = wa.UserLogin.Log.WithContext(wa.Main.Bridge.BackgroundCtx) @@ -379,6 +387,9 @@ func (wa *WhatsAppClient) callStopLoops() { func (wa *WhatsAppClient) Disconnect() { wa.callStopLoops() + if wa.VOIP != nil { + wa.VOIP.AbortAll() + } if cli := wa.Client; cli != nil { cli.Disconnect() } diff --git a/pkg/connector/commands_voip.go b/pkg/connector/commands_voip.go new file mode 100644 index 0000000..b048d94 --- /dev/null +++ b/pkg/connector/commands_voip.go @@ -0,0 +1,530 @@ +package connector + +import ( + "errors" + "fmt" + "slices" + "strings" + + "github.com/purpshell/meowcaller" + "maunium.net/go/mautrix/bridgev2/commands" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +var HelpSectionCalls = commands.HelpSection{Name: "Calls", Order: 27} + +var cmdCallParticipants = &commands.FullHandler{ + Func: fnCallParticipants, + Name: "call-participants", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "List the WhatsApp participants in the active call.", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallAdd = &commands.FullHandler{ + Func: fnCallAdd, + Name: "call-add", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Invite a WhatsApp user to the active call.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallRing = &commands.FullHandler{ + Func: fnCallRing, + Name: "call-ring", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Ring a non-connected WhatsApp participant already in the active call.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallVideoSelect = &commands.FullHandler{ + Func: fnCallVideoSelect, + Name: "call-video-select", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Select which WhatsApp group participant is shown on the Matrix camera track.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallLinkCreate = &commands.FullHandler{ + Func: fnCallLinkCreate, + Name: "call-link-create", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Create a reusable WhatsApp call link.", + Args: "[audio|video]", + }, + RequiresLogin: true, +} + +var cmdCallLinkPreview = &commands.FullHandler{ + Func: fnCallLinkPreview, + Name: "call-link-preview", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Preview a WhatsApp call link without joining it.", + Args: " [audio|video]", + }, + RequiresLogin: true, +} + +var cmdCallLinkJoin = &commands.FullHandler{ + Func: fnCallLinkJoin, + Name: "call-link-join", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Join a WhatsApp call link and ring it into the current Matrix room.", + Args: " [audio|video]", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallWaiting = &commands.FullHandler{ + Func: fnCallWaiting, + Name: "call-waiting", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Show the waiting room for the active WhatsApp call link.", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallApproval = &commands.FullHandler{ + Func: fnCallApproval, + Name: "call-approval", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Enable or disable approval for the active WhatsApp call link.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallAdmit = &commands.FullHandler{ + Func: fnCallAdmit, + Name: "call-admit", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Admit a user from the active WhatsApp call link waiting room.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +var cmdCallDeny = &commands.FullHandler{ + Func: fnCallDeny, + Name: "call-deny", + Help: commands.HelpMeta{ + Section: HelpSectionCalls, + Description: "Deny a user from the active WhatsApp call link waiting room.", + Args: "", + }, + RequiresLogin: true, + RequiresPortal: true, +} + +func fnCallParticipants(ce *commands.Event) { + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + state, ok, err := client.VOIP.GroupState(call.WACallID) + if err != nil { + ce.Reply("Failed to read the active call roster: %v", err) + return + } + if !ok { + ce.Reply("WhatsApp has not advertised a group roster for this call yet.") + return + } + ce.Reply(formatGroupCallRoster(state)) +} + +func fnCallAdd(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.AddParticipant(ce.Ctx, call.WACallID, target); err != nil { + ce.Reply("Failed to invite the participant: %v", err) + return + } + ce.Reply("Invited `%s` to the active WhatsApp call.", target) +} + +func fnCallRing(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.RingParticipant(ce.Ctx, call.WACallID, target); err != nil { + ce.Reply("Failed to ring the participant: %v", err) + return + } + ce.Reply("Rang `%s` in the active WhatsApp call.", target) +} + +func fnCallVideoSelect(ce *commands.Event) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.SelectVideoParticipant(call.WACallID, target); err != nil { + ce.Reply("Failed to select the WhatsApp video participant: %v", err) + return + } + ce.Reply("Selected `%s` for the WhatsApp camera track.", target) +} + +func fnCallLinkCreate(ce *commands.Event) { + video, err := callMediaArg(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-create [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + link, err := client.VOIP.CreateCallLink(ce.Ctx, video) + if err != nil { + ce.Reply("Failed to create the WhatsApp call link: %v", err) + return + } + ce.Reply("Created a WhatsApp %s call link:\n\n%s", callMediaName(video), link.URL) +} + +func fnCallLinkPreview(ce *commands.Event) { + token, video, err := callLinkArgs(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-preview [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + preview, err := client.VOIP.PreviewCallLink(ce.Ctx, token, video) + if err != nil { + ce.Reply("Failed to preview the WhatsApp call link: %v", err) + return + } + creator := preview.Creator + if !preview.CreatorPhoneNumber.IsEmpty() { + creator = preview.CreatorPhoneNumber + } + ce.Reply( + "**WhatsApp %s call link**\n\nCreator: `%s`\n\nApproval required: **%t**\n\nYou are an admin: **%t**", + callMediaName(preview.Video), creator, preview.ApprovalRequired, preview.IsAdmin, + ) +} + +func fnCallLinkJoin(ce *commands.Event) { + token, video, err := callLinkArgs(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-link-join [audio|video]`") + return + } + client, err := commandWhatsAppClient(ce) + if err != nil { + ce.Reply("Failed to resolve the WhatsApp login: %v", err) + return + } + call, err := client.joinMatrixRTCCallLink(ce.Ctx, ce.Portal, token, video) + if err != nil { + ce.Reply("Failed to join the WhatsApp call link: %v", err) + return + } + if state, ok, _ := client.VOIP.WaitingRoomState(call.ID()); ok && state.InWaitingRoom { + ce.Reply("Joined the WhatsApp call link waiting room. Element will ring in this room while approval is pending.") + } else { + ce.Reply("Joined the WhatsApp call link. Element will ring in this room.") + } +} + +func fnCallWaiting(ce *commands.Event) { + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + state, ok, err := client.VOIP.WaitingRoomState(call.WACallID) + if err != nil { + ce.Reply("Failed to read the waiting room: %v", err) + return + } + if !ok { + ce.Reply("The active call has no WhatsApp call-link waiting-room state.") + return + } + ce.Reply(formatWaitingRoomState(state)) +} + +func fnCallApproval(ce *commands.Event) { + if len(ce.Args) != 1 { + ce.Reply("Usage: `$cmdprefix call-approval `") + return + } + enabled, err := parseCallApproval(ce.Args[0]) + if err != nil { + ce.Reply("Usage: `$cmdprefix call-approval `") + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if err = client.VOIP.SetApprovalRequired(ce.Ctx, call.WACallID, enabled); err != nil { + ce.Reply("Failed to change call-link approval: %v", err) + return + } + ce.Reply("WhatsApp call-link approval is now **%s**.", map[bool]string{true: "enabled", false: "disabled"}[enabled]) +} + +func fnCallAdmit(ce *commands.Event) { + fnCallWaitingParticipant(ce, true) +} + +func fnCallDeny(ce *commands.Event) { + fnCallWaitingParticipant(ce, false) +} + +func fnCallWaitingParticipant(ce *commands.Event, admit bool) { + target, ok := callTargetArg(ce) + if !ok { + return + } + client, call, err := activePortalCall(ce) + if err != nil { + ce.Reply("Failed to find the active call: %v", err) + return + } + if admit { + err = client.VOIP.AdmitParticipant(ce.Ctx, call.WACallID, target) + } else { + err = client.VOIP.DenyParticipant(ce.Ctx, call.WACallID, target) + } + if err != nil { + ce.Reply("Failed to update the waiting-room participant: %v", err) + return + } + action := "Admitted" + if !admit { + action = "Denied" + } + ce.Reply("%s `%s` in the WhatsApp call-link waiting room.", action, target) +} + +func callTargetArg(ce *commands.Event) (string, bool) { + if len(ce.Args) != 1 { + ce.Reply("Usage: `$cmdprefix %s `", ce.Command) + return "", false + } + return strings.TrimSpace(ce.Args[0]), true +} + +func activePortalCall(ce *commands.Event) (*WhatsAppClient, *wadb.MatrixRTCCall, error) { + if ce.Portal == nil { + return nil, nil, errors.New("this command can only be used in a portal room") + } + client, err := commandWhatsAppClient(ce) + if err != nil { + return nil, nil, err + } + calls, err := client.Main.DB.MatrixRTCCall.GetActiveInRoom(ce.Ctx, ce.Portal.MXID) + if err != nil { + return nil, nil, fmt.Errorf("query active calls: %w", err) + } + call, err := selectActiveCallForLogin(calls, client.UserLogin.ID) + if err != nil { + return nil, nil, err + } + return client, call, nil +} + +func commandWhatsAppClient(ce *commands.Event) (*WhatsAppClient, error) { + var loginID networkid.UserLoginID + if ce.Portal != nil { + loginID = ce.Portal.Receiver + } else if login := ce.User.GetDefaultLogin(); login != nil { + loginID = login.ID + } + login := ce.Bridge.GetCachedUserLoginByID(loginID) + if login == nil { + return nil, errors.New("the WhatsApp login is not available") + } + var sender id.UserID + if ce.User != nil { + sender = ce.User.MXID + } + if !matrixRTCSenderOwnsLogin(sender, login) { + return nil, errors.New("the WhatsApp login for this portal belongs to another Matrix user") + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || !client.IsLoggedIn() { + return nil, errors.New("the WhatsApp login is not connected") + } + if client.VOIP == nil || !client.VOIP.Enabled() { + return nil, errors.New("WhatsApp calling is not enabled") + } + return client, nil +} + +func selectActiveCallForLogin(calls []*wadb.MatrixRTCCall, loginID networkid.UserLoginID) (*wadb.MatrixRTCCall, error) { + var selected *wadb.MatrixRTCCall + for _, call := range calls { + if call == nil || call.UserLoginID != loginID { + continue + } + if selected != nil { + return nil, errors.New("multiple active calls are tracked in this room") + } + selected = call + } + if selected == nil { + return nil, errors.New("there is no active call in this room") + } + return selected, nil +} + +func formatGroupCallRoster(state meowcaller.GroupCallState) string { + participants := slices.Clone(state.Participants) + slices.SortFunc(participants, func(a, b meowcaller.GroupCallParticipant) int { + return strings.Compare(a.JID.String(), b.JID.String()) + }) + lines := make([]string, 0, len(participants)+1) + lines = append(lines, fmt.Sprintf("**WhatsApp call participants (transaction %d):**", state.TransactionID)) + for _, participant := range participants { + identity := participant.JID + if !participant.PN.IsEmpty() { + identity = participant.PN + } + detail := fmt.Sprintf("%s; %d device(s)", participant.State, len(participant.Devices)) + if participant.HandRaised { + detail += "; hand raised" + } + lines = append(lines, fmt.Sprintf("- `%s`: %s", identity, detail)) + } + if state.RekeyRequested { + lines = append(lines, "- WhatsApp requested a group media rekey.") + } + return strings.Join(lines, "\n") +} + +func formatWaitingRoomState(state meowcaller.WaitingRoomState) string { + lines := []string{ + fmt.Sprintf( + "**WhatsApp call-link waiting room (transaction %d):** approval **%s**, admin **%t**, waiting **%t**", + state.TransactionID, + map[bool]string{true: "enabled", false: "disabled"}[state.Enabled], + state.IsAdmin, + state.InWaitingRoom, + ), + } + users := slices.Clone(state.Users) + slices.SortFunc(users, func(a, b meowcaller.WaitingRoomUser) int { + return strings.Compare(a.JID.String(), b.JID.String()) + }) + for _, user := range users { + identity := user.JID + if !user.PN.IsEmpty() { + identity = user.PN + } + lines = append(lines, fmt.Sprintf("- `%s`: %s", identity, user.State)) + } + if len(users) == 0 { + lines = append(lines, "- No users are waiting.") + } + return strings.Join(lines, "\n") +} + +func callLinkArgs(args []string) (token string, video bool, err error) { + if len(args) < 1 || len(args) > 2 { + return "", false, errors.New("invalid call-link arguments") + } + token = strings.TrimSpace(args[0]) + if token == "" { + return "", false, errors.New("call-link token is empty") + } + if len(args) == 2 { + video, err = callMediaArg(args[1:]) + return + } + video = strings.HasPrefix(strings.ToLower(token), "https://call.whatsapp.com/video/") + return +} + +func callMediaArg(args []string) (bool, error) { + if len(args) == 0 { + return false, nil + } + if len(args) != 1 { + return false, errors.New("invalid call media") + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "audio": + return false, nil + case "video": + return true, nil + default: + return false, errors.New("invalid call media") + } +} + +func callMediaName(video bool) string { + if video { + return "video" + } + return "audio" +} + +func parseCallApproval(raw string) (bool, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "on", "true", "enable", "enabled": + return true, nil + case "off", "false", "disable", "disabled": + return false, nil + default: + return false, errors.New("invalid approval state") + } +} diff --git a/pkg/connector/commands_voip_test.go b/pkg/connector/commands_voip_test.go new file mode 100644 index 0000000..75007fe --- /dev/null +++ b/pkg/connector/commands_voip_test.go @@ -0,0 +1,144 @@ +package connector + +import ( + "strings" + "testing" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/bridgev2/networkid" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func TestSelectActiveCallForLogin(t *testing.T) { + alice := networkid.UserLoginID("alice") + bob := networkid.UserLoginID("bob") + calls := []*wadb.MatrixRTCCall{ + {UserLoginID: bob, WACallID: "bob-call"}, + {UserLoginID: alice, WACallID: "alice-call"}, + } + call, err := selectActiveCallForLogin(calls, alice) + if err != nil { + t.Fatalf("selectActiveCallForLogin returned error: %v", err) + } + if call.WACallID != "alice-call" { + t.Fatalf("selected call = %q, want alice-call", call.WACallID) + } +} + +func TestSelectActiveCallForLoginRejectsMissingAndAmbiguousCalls(t *testing.T) { + loginID := networkid.UserLoginID("alice") + if _, err := selectActiveCallForLogin(nil, loginID); err == nil { + t.Fatal("selectActiveCallForLogin accepted an empty call list") + } + calls := []*wadb.MatrixRTCCall{ + {UserLoginID: loginID, WACallID: "first"}, + {UserLoginID: loginID, WACallID: "second"}, + } + if _, err := selectActiveCallForLogin(calls, loginID); err == nil { + t.Fatal("selectActiveCallForLogin accepted multiple calls for one login") + } +} + +func TestFormatGroupCallRoster(t *testing.T) { + state := meowcaller.GroupCallState{ + TransactionID: 42, + RekeyRequested: true, + Participants: []meowcaller.GroupCallParticipant{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + State: "connected", + Devices: []meowcaller.GroupCallDevice{ + {JID: types.NewJID("222", types.HiddenUserServer)}, + }, + HandRaised: true, + }, + { + JID: types.NewJID("111", types.HiddenUserServer), + State: "ringing", + }, + }, + } + got := formatGroupCallRoster(state) + for _, want := range []string{ + "transaction 42", + "`111@lid`: ringing; 0 device(s)", + "`15550000002@s.whatsapp.net`: connected; 1 device(s); hand raised", + "requested a group media rekey", + } { + if !strings.Contains(got, want) { + t.Errorf("formatted roster missing %q:\n%s", want, got) + } + } +} + +func TestCallLinkArgsInfersMediaFromURL(t *testing.T) { + token, video, err := callLinkArgs([]string{"https://call.whatsapp.com/video/TOKEN"}) + if err != nil { + t.Fatalf("callLinkArgs returned error: %v", err) + } + if token != "https://call.whatsapp.com/video/TOKEN" || !video { + t.Fatalf("callLinkArgs = (%q, %t), want video URL and true", token, video) + } + + token, video, err = callLinkArgs([]string{"TOKEN", "video"}) + if err != nil { + t.Fatalf("callLinkArgs with explicit media returned error: %v", err) + } + if token != "TOKEN" || !video { + t.Fatalf("callLinkArgs = (%q, %t), want TOKEN and true", token, video) + } +} + +func TestCallMediaArgRejectsUnknownMedia(t *testing.T) { + if _, err := callMediaArg([]string{"screen"}); err == nil { + t.Fatal("callMediaArg accepted an unknown media kind") + } + if video, err := callMediaArg(nil); err != nil || video { + t.Fatalf("callMediaArg default = (%t, %v), want audio and nil", video, err) + } +} + +func TestParseCallApproval(t *testing.T) { + for _, raw := range []string{"on", "true", "enabled"} { + if enabled, err := parseCallApproval(raw); err != nil || !enabled { + t.Errorf("parseCallApproval(%q) = (%t, %v), want true and nil", raw, enabled, err) + } + } + for _, raw := range []string{"off", "false", "disabled"} { + if enabled, err := parseCallApproval(raw); err != nil || enabled { + t.Errorf("parseCallApproval(%q) = (%t, %v), want false and nil", raw, enabled, err) + } + } + if _, err := parseCallApproval("maybe"); err == nil { + t.Fatal("parseCallApproval accepted an invalid value") + } +} + +func TestFormatWaitingRoomState(t *testing.T) { + state := meowcaller.WaitingRoomState{ + Enabled: true, + IsAdmin: true, + InWaitingRoom: false, + TransactionID: 7, + Users: []meowcaller.WaitingRoomUser{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + State: "pending", + }, + }, + } + got := formatWaitingRoomState(state) + for _, want := range []string{ + "transaction 7", + "approval **enabled**", + "`15550000002@s.whatsapp.net`: pending", + } { + if !strings.Contains(got, want) { + t.Errorf("formatted waiting room missing %q:\n%s", want, got) + } + } +} diff --git a/pkg/connector/config.go b/pkg/connector/config.go index e51c80a..7d15948 100644 --- a/pkg/connector/config.go +++ b/pkg/connector/config.go @@ -54,6 +54,7 @@ type Config struct { UseWhatsAppRetryStore bool `yaml:"use_whatsapp_retry_store"` AnimatedSticker msgconv.AnimatedStickerConfig `yaml:"animated_sticker"` + VOIP VOIPConfig `yaml:"voip"` HistorySync struct { MaxInitialConversations int `yaml:"max_initial_conversations"` @@ -78,6 +79,58 @@ type Config struct { displaynameTemplate *template.Template `yaml:"-"` } +type VOIPConfig struct { + Enabled bool `yaml:"enabled"` + MatrixSurface string `yaml:"matrix_surface"` + IncomingPolicy string `yaml:"incoming_policy"` + MaxActiveCallsPerLogin int `yaml:"max_active_calls_per_login"` + MatrixRTC MatrixRTCConfig `yaml:"matrixrtc"` + LiveKit LiveKitConfig `yaml:"livekit"` + Audio VOIPAudioConfig `yaml:"audio"` + Video VOIPVideoConfig `yaml:"video"` + Diagnostics VOIPDiagnostics `yaml:"diagnostics"` +} + +type MatrixRTCConfig struct { + LiveKitServiceURL string `yaml:"livekit_service_url"` + RequireLiveKitFocus bool `yaml:"require_livekit_focus"` + MembershipEventCompat string `yaml:"membership_event_compat"` + NotificationEventCompat string `yaml:"notification_event_compat"` + UseDelayedEvents bool `yaml:"use_delayed_events"` + ParticipantMode string `yaml:"participant_mode"` + FallbackParticipantMXID string `yaml:"fallback_participant_mxid"` +} + +type LiveKitConfig struct { + ConnectTimeout time.Duration `yaml:"connect_timeout"` + PublishSilenceBeforeWhatsAppAnswer bool `yaml:"publish_silence_before_whatsapp_answer"` + AutoSubscribe bool `yaml:"auto_subscribe"` + AudioUplinkPolicy string `yaml:"audio_uplink_policy"` + SelectedParticipantTimeout time.Duration `yaml:"selected_participant_timeout"` +} + +type VOIPAudioConfig struct { + Enabled bool `yaml:"enabled"` + JitterBuffer time.Duration `yaml:"jitter_buffer_ms"` + OpusBackend string `yaml:"opus_backend"` + SilenceOnUnderrun bool `yaml:"silence_on_underrun"` + MaxMixParticipants int `yaml:"max_mix_participants"` +} + +type VOIPVideoConfig struct { + Enabled bool `yaml:"enabled"` + SelectedSourcePolicy string `yaml:"selected_source_policy"` + MaxWidth int `yaml:"max_width"` + MaxHeight int `yaml:"max_height"` + MaxFPS int `yaml:"max_fps"` +} + +type VOIPDiagnostics struct { + HealthcheckFocusOnStartup bool `yaml:"healthcheck_focus_on_startup"` + EnableMeowcallerDiagnostics bool `yaml:"enable_meowcaller_diagnostics"` + MediaTraceDir string `yaml:"media_trace_dir"` +} + type umConfig Config func (c *Config) UnmarshalYAML(node *yaml.Node) error { @@ -99,9 +152,74 @@ func (c *Config) PostProcess() error { if err != nil { return fmt.Errorf("failed to execute displayname template: %w", err) } + if err = c.validateVOIP(); err != nil { + return err + } return nil } +func (c *Config) validateVOIP() error { + if !c.VOIP.Enabled { + return nil + } + if c.VOIP.MatrixSurface != "matrixrtc_livekit" { + return fmt.Errorf("voip.matrix_surface must be matrixrtc_livekit") + } + if !oneOf(c.VOIP.IncomingPolicy, "notice", "ring", "auto_answer") { + return fmt.Errorf("voip.incoming_policy must be one of notice, ring, auto_answer") + } + if c.VOIP.MaxActiveCallsPerLogin <= 0 { + return fmt.Errorf("voip.max_active_calls_per_login must be greater than 0") + } + if !oneOf(c.VOIP.MatrixRTC.MembershipEventCompat, "auto", "msc4143", "msc3401") { + return fmt.Errorf("voip.matrixrtc.membership_event_compat must be one of auto, msc4143, msc3401") + } + if !oneOf(c.VOIP.MatrixRTC.NotificationEventCompat, "auto", "disabled") { + return fmt.Errorf("voip.matrixrtc.notification_event_compat must be one of auto, disabled") + } + if !oneOf(c.VOIP.MatrixRTC.ParticipantMode, "whatsapp_ghost", "bridge_user") { + return fmt.Errorf("voip.matrixrtc.participant_mode must be one of whatsapp_ghost, bridge_user") + } + if c.VOIP.LiveKit.ConnectTimeout <= 0 { + return fmt.Errorf("voip.livekit.connect_timeout must be greater than 0") + } + if !oneOf(c.VOIP.LiveKit.AudioUplinkPolicy, "dominant_speaker", "mix_all", "selected_participant") { + return fmt.Errorf("voip.livekit.audio_uplink_policy must be one of dominant_speaker, mix_all, selected_participant") + } + if c.VOIP.Audio.Enabled { + if c.VOIP.Audio.JitterBuffer <= 0 { + return fmt.Errorf("voip.audio.jitter_buffer_ms must be greater than 0") + } + if c.VOIP.Audio.OpusBackend == "" { + return fmt.Errorf("voip.audio.opus_backend must be set") + } + if c.VOIP.Audio.MaxMixParticipants <= 0 { + return fmt.Errorf("voip.audio.max_mix_participants must be greater than 0") + } + } + if c.VOIP.Video.Enabled { + if !oneOf(c.VOIP.Video.SelectedSourcePolicy, "active_speaker", "selected_participant") { + return fmt.Errorf("voip.video.selected_source_policy must be one of active_speaker, selected_participant") + } + if c.VOIP.Video.MaxWidth <= 0 || c.VOIP.Video.MaxHeight <= 0 || c.VOIP.Video.MaxFPS <= 0 { + return fmt.Errorf("voip.video max_width, max_height and max_fps must be greater than 0") + } + } + if c.VOIP.Diagnostics.EnableMeowcallerDiagnostics && c.VOIP.Diagnostics.MediaTraceDir == "" { + return fmt.Errorf("voip.diagnostics.media_trace_dir must be set when meowcaller diagnostics are enabled") + } + return nil +} + +func oneOf(value string, allowed ...string) bool { + for _, item := range allowed { + if value == item { + return true + } + } + return false +} + func upgradeConfig(helper up.Helper) { helper.Copy(up.Str, "os_name") helper.Copy(up.Str, "browser_name") @@ -135,6 +253,36 @@ func upgradeConfig(helper up.Helper) { helper.Copy(up.Int, "animated_sticker", "args", "height") helper.Copy(up.Int, "animated_sticker", "args", "fps") + helper.Copy(up.Bool, "voip", "enabled") + helper.Copy(up.Str, "voip", "matrix_surface") + helper.Copy(up.Str, "voip", "incoming_policy") + helper.Copy(up.Int, "voip", "max_active_calls_per_login") + helper.Copy(up.Str|up.Null, "voip", "matrixrtc", "livekit_service_url") + helper.Copy(up.Bool, "voip", "matrixrtc", "require_livekit_focus") + helper.Copy(up.Str, "voip", "matrixrtc", "membership_event_compat") + helper.Copy(up.Str, "voip", "matrixrtc", "notification_event_compat") + helper.Copy(up.Bool, "voip", "matrixrtc", "use_delayed_events") + helper.Copy(up.Str, "voip", "matrixrtc", "participant_mode") + helper.Copy(up.Str|up.Null, "voip", "matrixrtc", "fallback_participant_mxid") + helper.Copy(up.Str|up.Int, "voip", "livekit", "connect_timeout") + helper.Copy(up.Bool, "voip", "livekit", "publish_silence_before_whatsapp_answer") + helper.Copy(up.Bool, "voip", "livekit", "auto_subscribe") + helper.Copy(up.Str, "voip", "livekit", "audio_uplink_policy") + helper.Copy(up.Str|up.Int, "voip", "livekit", "selected_participant_timeout") + helper.Copy(up.Bool, "voip", "audio", "enabled") + helper.Copy(up.Str|up.Int, "voip", "audio", "jitter_buffer_ms") + helper.Copy(up.Str, "voip", "audio", "opus_backend") + helper.Copy(up.Bool, "voip", "audio", "silence_on_underrun") + helper.Copy(up.Int, "voip", "audio", "max_mix_participants") + helper.Copy(up.Bool, "voip", "video", "enabled") + helper.Copy(up.Str, "voip", "video", "selected_source_policy") + helper.Copy(up.Int, "voip", "video", "max_width") + helper.Copy(up.Int, "voip", "video", "max_height") + helper.Copy(up.Int, "voip", "video", "max_fps") + helper.Copy(up.Bool, "voip", "diagnostics", "healthcheck_focus_on_startup") + helper.Copy(up.Bool, "voip", "diagnostics", "enable_meowcaller_diagnostics") + helper.Copy(up.Str|up.Null, "voip", "diagnostics", "media_trace_dir") + helper.Copy(up.Int, "history_sync", "max_initial_conversations") helper.Copy(up.Bool, "history_sync", "request_full_sync") helper.Copy(up.Str|up.Int, "history_sync", "dispatch_wait") @@ -205,6 +353,7 @@ func (wa *WhatsAppConnector) GetConfig() (string, any, up.Upgrader) { {"proxy"}, {"displayname_template"}, {"call_start_notices"}, + {"voip"}, {"history_sync"}, }, Base: ExampleConfig, diff --git a/pkg/connector/config_test.go b/pkg/connector/config_test.go new file mode 100644 index 0000000..5f2b93f --- /dev/null +++ b/pkg/connector/config_test.go @@ -0,0 +1,33 @@ +package connector + +import ( + "os" + "testing" + + "go.yaml.in/yaml/v3" +) + +func TestExampleConfigDoesNotAdvertiseUnsupportedVideoModes(t *testing.T) { + data, err := os.ReadFile("example-config.yaml") + if err != nil { + t.Fatal(err) + } + + var config map[string]any + if err = yaml.Unmarshal(data, &config); err != nil { + t.Fatal(err) + } + voip, ok := config["voip"].(map[string]any) + if !ok { + t.Fatal("example config has no voip section") + } + video, ok := voip["video"].(map[string]any) + if !ok { + t.Fatal("example config has no voip.video section") + } + for _, unsupported := range []string{"require_h264", "allow_transcode"} { + if _, exists := video[unsupported]; exists { + t.Errorf("example config advertises unsupported voip.video.%s option", unsupported) + } + } +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index e262c82..7213cdd 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -28,24 +28,27 @@ import ( "time" "github.com/lib/pq" + "github.com/rs/zerolog" "go.mau.fi/util/dbutil" - "go.mau.fi/util/exsync" "go.mau.fi/util/random" "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/proto/waCompanionReg" "go.mau.fi/whatsmeow/store" "go.mau.fi/whatsmeow/store/sqlstore" whatsmeowUpgrades "go.mau.fi/whatsmeow/store/sqlstore/upgrades" + "go.mau.fi/whatsmeow/types" waLog "go.mau.fi/whatsmeow/util/log" "google.golang.org/protobuf/proto" "maunium.net/go/mautrix/bridgev2" "maunium.net/go/mautrix/bridgev2/commands" + "maunium.net/go/mautrix/bridgev2/database" "maunium.net/go/mautrix/bridgev2/networkid" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" "go.mau.fi/mautrix-whatsapp/pkg/msgconv" + "go.mau.fi/mautrix-whatsapp/pkg/waid" ) type WhatsAppConnector struct { @@ -62,7 +65,8 @@ type WhatsAppConnector struct { mediaEditCacheLock sync.RWMutex stopMediaEditCacheLoop atomic.Pointer[context.CancelFunc] - unmigratedDMs *exsync.Set[networkid.PortalKey] + matrixRTCOutboundStartLock sync.Mutex + matrixRTCOutboundStartExpires map[string]time.Time } func init() { @@ -109,11 +113,16 @@ func (wa *WhatsAppConnector) Init(bridge *bridgev2.Bridge) { } wa.DB = wadb.New(bridge.ID, bridge.DB.Database, bridge.Log.With().Str("db_section", "whatsapp").Logger()) wa.MsgConv.DB = wa.DB + wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) wa.Bridge.Commands.(*commands.Processor).AddHandlers( cmdAccept, cmdSync, cmdInviteLink, cmdResolveLink, cmdJoin, + cmdCallParticipants, cmdCallAdd, cmdCallRing, cmdCallVideoSelect, + cmdCallLinkCreate, cmdCallLinkPreview, cmdCallLinkJoin, + cmdCallWaiting, cmdCallApproval, cmdCallAdmit, cmdCallDeny, ) wa.mediaEditCache = make(MediaEditCache) - wa.unmigratedDMs = exsync.NewSet[networkid.PortalKey]() + wa.initMatrixRTCEventHooks() + wa.startMatrixRTCHealthcheck() whatsmeowDBLog := bridge.Log.With().Str("db_section", "whatsmeow").Logger() wa.DeviceStore = sqlstore.NewWithWrappedDB( @@ -157,7 +166,78 @@ func (wa *WhatsAppConnector) Start(ctx context.Context) error { return bridgev2.DBUpgradeError{Err: err, Section: "whatsapp"} } - return wa.migrateToLIDDMs(ctx) + if !wa.Bridge.Background && wa.Bridge.DB.KV.Get(ctx, "whatsapp_lid_dms_deleted") == "false" { + wa.deleteLIDDMsMigration(ctx) + } + + return nil +} + +func (wa *WhatsAppConnector) deleteLIDDMsMigration(ctx context.Context) { + log := zerolog.Ctx(ctx).With().Str("action", "delete lid dms").Logger() + portals, err := wa.Bridge.GetAllPortalsWithMXID(ctx) + if err != nil { + log.Err(err).Msg("Failed to get portals for LID DM deletion") + return + } + defer wa.Bridge.DB.KV.Set(ctx, "whatsapp_lid_dms_deleted", "true") + if len(portals) == 0 { + log.Debug().Msg("No portals found") + return + } + portalsByKey := make(map[networkid.PortalKey]*bridgev2.Portal, len(portals)) + for _, p := range portals { + if p.Receiver == "" || p.RoomType != database.RoomTypeDM { + continue + } + portalsByKey[p.PortalKey] = p + } + _, err = wa.DB.Exec(ctx, "DELETE FROM whatsapp_history_sync_conversation WHERE chat_jid LIKE '%@lid'") + if err != nil { + log.Err(err).Msg("Failed to remove LID conversations from history sync") + } + for key, portal := range portalsByKey { + parsedID, err := waid.ParsePortalID(key.ID) + if err != nil { + log.Warn().Err(err).Str("portal_id", string(key.ID)).Msg("Failed to parse portal ID") + continue + } else if parsedID.Server != types.HiddenUserServer { + continue + } + var pnStr string + err = wa.DB.QueryRow(ctx, "SELECT pn FROM whatsmeow_lid_map WHERE lid=$1", parsedID.User).Scan(&pnStr) + if err != nil { + log.Warn().Err(err).Str("portal_id", string(key.ID)).Msg("Failed to get PN for LID portal") + continue + } + key.ID = waid.MakePortalID(types.JID{User: pnStr, Server: types.DefaultUserServer}) + _, pnPortalExists := portalsByKey[key] + if !pnPortalExists { + log.Warn().Str("portal_id", string(key.ID)).Msg("PN portal does not exist, not deleting LID DM") + continue + } + err = portal.Delete(ctx) + if err != nil { + log.Err(err). + Object("portal_key", portal.PortalKey). + Stringer("portal_mxid", portal.MXID). + Msg("Failed to delete LID DM portal from database") + continue + } + err = wa.Bridge.Bot.DeleteRoom(ctx, portal.MXID, false) + if err != nil { + log.Err(err). + Object("portal_key", portal.PortalKey). + Stringer("portal_mxid", portal.MXID). + Msg("Failed to delete LID DM portal from Matrix") + continue + } + log.Debug(). + Object("portal_key", portal.PortalKey). + Stringer("portal_mxid", portal.MXID). + Msg("Deleted LID DM portal") + } + log.Info().Msg("Finished deleting LID DM portals") } func (wa *WhatsAppConnector) Stop() { diff --git a/pkg/connector/directmedia.go b/pkg/connector/directmedia.go index 8a7afd9..89a54fc 100644 --- a/pkg/connector/directmedia.go +++ b/pkg/connector/directmedia.go @@ -52,7 +52,7 @@ func (wa *WhatsAppConnector) SetUseDirectMedia() { } var ErrReloadNeeded = mautrix.RespError{ - ErrCode: "COM.BEEPER.MEDIA_RELOAD_NEEDED", + ErrCode: "FI.MAU.WHATSAPP_RELOAD_NEEDED", Err: "Media is no longer available on WhatsApp servers and must be re-requested from your phone", StatusCode: http.StatusNotFound, } @@ -212,7 +212,7 @@ func (wa *WhatsAppConnector) makeDirectMediaResponse( log := zerolog.Ctx(ctx) err := waClient.Client.DownloadToFile(ctx, dm, f) if keys != nil && (errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith403) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith404) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith410) || errors.Is(err, whatsmeow.ErrNoURLPresent)) { - val := params["com.beeper.interactive_download_request"] + val := params["fi.mau.whatsapp.reload_media"] if val == "false" || (!wa.Config.DirectMediaAutoRequest && val != "true") { return nil, ErrReloadNeeded } diff --git a/pkg/connector/events.go b/pkg/connector/events.go index 1f4e118..cb51e64 100644 --- a/pkg/connector/events.go +++ b/pkg/connector/events.go @@ -39,9 +39,22 @@ import ( "go.mau.fi/mautrix-whatsapp/pkg/waid" ) +func (wa *WhatsAppClient) getPortalKeyByMessageSource(ms types.MessageSource) networkid.PortalKey { + jid := ms.Chat + if ms.IsIncomingBroadcast() { + if ms.IsFromMe { + jid = ms.BroadcastListOwner.ToNonAD() + } else { + jid = ms.Sender.ToNonAD() + } + } + return wa.makeWAPortalKey(jid) +} + type MessageInfoWrapper struct { - Info types.MessageInfo - wa *WhatsAppClient + OrigSource types.MessageSource + Info types.MessageInfo + wa *WhatsAppClient } func (evt *MessageInfoWrapper) ShouldCreatePortal() bool { @@ -49,26 +62,7 @@ func (evt *MessageInfoWrapper) ShouldCreatePortal() bool { } func (evt *MessageInfoWrapper) GetPortalKey() networkid.PortalKey { - ms := evt.Info.MessageSource - jid := ms.Chat - if ms.IsIncomingBroadcast() { - if ms.IsFromMe { - // TODO can this still be a phone number? - jid = ms.BroadcastListOwner.ToNonAD() - } else { - jid = ms.Sender.ToNonAD() - if jid.Server == types.DefaultUserServer && !ms.SenderAlt.IsEmpty() { - jid = ms.SenderAlt.ToNonAD() - } - } - } else if jid.Server == types.DefaultUserServer { - if !ms.IsFromMe && ms.Chat.ToNonAD() == ms.Sender.ToNonAD() && !ms.SenderAlt.IsEmpty() { - jid = ms.SenderAlt.ToNonAD() - } else if !ms.RecipientAlt.IsEmpty() { - jid = ms.RecipientAlt.ToNonAD() - } - } - return evt.wa.makeWAPortalKey(jid) + return evt.wa.getPortalKeyByMessageSource(evt.Info.MessageSource) } func (evt *MessageInfoWrapper) AddLogContext(c zerolog.Context) zerolog.Context { @@ -79,19 +73,12 @@ func (evt *MessageInfoWrapper) GetTimestamp() time.Time { return evt.Info.Timestamp } -func pickLID(main, alt types.JID) types.JID { - if main.Server == types.DefaultUserServer && alt.Server == types.HiddenUserServer { - return alt - } - return main -} - func (evt *MessageInfoWrapper) GetSender() bridgev2.EventSender { - return evt.wa.makeEventSender(evt.wa.Main.Bridge.BackgroundCtx, pickLID(evt.Info.Sender, evt.Info.SenderAlt)) + return evt.wa.makeEventSender(evt.wa.Main.Bridge.BackgroundCtx, evt.Info.Sender) } func (evt *MessageInfoWrapper) GetID() networkid.MessageID { - return waid.MakeMessageIDWithAltSender(evt.Info.Chat, evt.Info.Sender, evt.Info.SenderAlt, evt.Info.ID) + return waid.MakeMessageID(evt.Info.Chat, evt.Info.Sender, evt.Info.ID) } func (evt *MessageInfoWrapper) GetTransactionID() networkid.TransactionID { @@ -148,6 +135,14 @@ func (evt *WAMessageEvent) PreHandle(ctx context.Context, portal *bridgev2.Porta return } meta := portal.Metadata.(*waid.PortalMetadata) + if meta.AddressingMode == types.AddressingModeLID && evt.Info.Sender.Server == types.DefaultUserServer { + evt.Info.Sender, evt.Info.SenderAlt = evt.Info.SenderAlt, evt.Info.Sender + zerolog.Ctx(ctx).Debug(). + Stringer("lid", evt.Info.Sender). + Stringer("pn", evt.Info.SenderAlt). + Str("message_id", evt.Info.ID). + Msg("Forced phone number sender to LID in group message") + } if meta.AddressingMode == types.AddressingModeLID || meta.LIDMigrationAttempted { return } @@ -166,6 +161,13 @@ func (evt *WAMessageEvent) PreHandle(ctx context.Context, portal *bridgev2.Porta log.Info().Msg("Resyncing group members as it appears to have switched to LID addressing mode") portal.UpdateInfo(ctx, evt.wa.wrapGroupInfo(ctx, info), evt.wa.UserLogin, nil, time.Time{}) log.Debug().Msg("Finished resyncing after LID change") + if evt.Info.Sender.Server == types.DefaultUserServer && evt.Info.SenderAlt.Server == types.HiddenUserServer { + evt.Info.Sender, evt.Info.SenderAlt = evt.Info.SenderAlt, evt.Info.Sender + log.Debug(). + Stringer("new_sender", evt.Info.Sender). + Stringer("new_sender_alt", evt.Info.SenderAlt). + Msg("Overriding sender to LID after resyncing group members") + } } func (evt *WAMessageEvent) PostHandle(ctx context.Context, portal *bridgev2.Portal) { @@ -199,7 +201,7 @@ func (evt *WAMessageEvent) ConvertEdit(ctx context.Context, portal *bridgev2.Por ctx = context.WithValue(ctx, msgconv.ContextKeyEditTargetID, evt.Message.GetProtocolMessage().GetKey().GetID()) cm := evt.wa.Main.MsgConv.ToMatrix( - ctx, portal, evt.wa.Client, intent, editedMsg, evt.MsgEvent.RawMessage, &evt.Info, evt.isViewOnce(), false, previouslyConvertedPart, + ctx, portal, evt.wa.Client, intent, editedMsg, evt.MsgEvent.RawMessage, &evt.Info, &evt.OrigSource, evt.isViewOnce(), false, previouslyConvertedPart, ) if evt.isUndecryptableUpsertSubEvent && isFailedMedia(cm) { evt.postHandle = func() { @@ -284,7 +286,7 @@ func (evt *WAMessageEvent) HandleExisting(ctx context.Context, portal *bridgev2. func (evt *WAMessageEvent) ConvertMessage(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI) (*bridgev2.ConvertedMessage, error) { evt.wa.EnqueuePortalResync(portal, false) converted := evt.wa.Main.MsgConv.ToMatrix( - ctx, portal, evt.wa.Client, intent, evt.Message, evt.MsgEvent.RawMessage, &evt.Info, evt.isViewOnce(), false, nil, + ctx, portal, evt.wa.Client, intent, evt.Message, evt.MsgEvent.RawMessage, &evt.Info, &evt.OrigSource, evt.isViewOnce(), false, nil, ) if isFailedMedia(converted) { evt.postHandle = func() { @@ -390,9 +392,7 @@ func (evt *WAUndecryptableMessage) GetStreamOrder() int64 { type WAMediaRetry struct { *events.MediaRetry - wa *WhatsAppClient - senderLID types.JID - chatLID types.JID + wa *WhatsAppClient } func (evt *WAMediaRetry) GetType() bridgev2.RemoteEventType { @@ -400,7 +400,7 @@ func (evt *WAMediaRetry) GetType() bridgev2.RemoteEventType { } func (evt *WAMediaRetry) GetPortalKey() networkid.PortalKey { - return evt.wa.makeWAPortalKey(pickLID(evt.ChatID, evt.chatLID)) + return evt.wa.makeWAPortalKey(evt.ChatID) } func (evt *WAMediaRetry) AddLogContext(c zerolog.Context) zerolog.Context { @@ -414,23 +414,16 @@ func (evt *WAMediaRetry) AddLogContext(c zerolog.Context) zerolog.Context { func (evt *WAMediaRetry) getRealSender() types.JID { sender := evt.SenderID - if sender.IsEmpty() { - if evt.FromMe { - if evt.ChatID.Server == types.HiddenUserServer { - sender = evt.wa.GetLID().ToNonAD() - } else { - sender = evt.wa.JID.ToNonAD() - } - } else if evt.ChatID.Server == types.DefaultUserServer || evt.ChatID.Server == types.HiddenUserServer || evt.ChatID.Server == types.BotServer { - sender = evt.ChatID.ToNonAD() - } + if evt.FromMe { + sender = evt.wa.JID.ToNonAD() + } else if sender.IsEmpty() && (evt.ChatID.Server == types.DefaultUserServer || evt.ChatID.Server == types.BotServer) { + sender = evt.ChatID.ToNonAD() } return sender } func (evt *WAMediaRetry) GetSender() bridgev2.EventSender { - realSender := pickLID(evt.getRealSender(), evt.senderLID) - return evt.wa.makeEventSender(evt.wa.Main.Bridge.BackgroundCtx, realSender) + return evt.wa.makeEventSender(evt.wa.Main.Bridge.BackgroundCtx, evt.getRealSender()) } func (evt *WAMediaRetry) GetTargetMessage() networkid.MessageID { diff --git a/pkg/connector/example-config.yaml b/pkg/connector/example-config.yaml index 564f25e..30c545c 100644 --- a/pkg/connector/example-config.yaml +++ b/pkg/connector/example-config.yaml @@ -70,6 +70,67 @@ initial_auto_reconnect: true # retry receipts if the bridge is restarted after the message is sent. use_whatsapp_retry_store: false +# MatrixRTC/LiveKit call bridging for WhatsApp calls. +voip: + # Enables real call bridging through Element Call. When false, call_start_notices behavior remains. + enabled: false + # Only MatrixRTC with a LiveKit focus is supported by this bridge path. + matrix_surface: matrixrtc_livekit + # notice - keep call_start_notices fallback only + # ring - ring MatrixRTC/Element Call and answer WhatsApp only after a Matrix user joins + # auto_answer - test-only behavior that answers WhatsApp immediately + incoming_policy: ring + # Limit concurrent WhatsApp call legs per login. A direct, group, or call-link call counts as one leg. + max_active_calls_per_login: 1 + + matrixrtc: + # If null, discover from .well-known org.matrix.msc4143.rtc_foci. + livekit_service_url: null + # Refuse real call bridging when no LiveKit focus can be discovered. + require_livekit_focus: true + # auto, msc4143, or msc3401. + membership_event_compat: auto + # auto or disabled. Used for MatrixRTC call ringing/decline events when supported. + notification_event_compat: auto + # Use homeserver delayed events to expire MatrixRTC membership when supported. + use_delayed_events: true + # whatsapp_ghost or bridge_user. + participant_mode: whatsapp_ghost + # Optional explicit Matrix user to use when participant_mode is bridge_user. + fallback_participant_mxid: null + + livekit: + connect_timeout: 10s + # Publish silence before WhatsApp answers so Element Call shows a stable participant. + publish_silence_before_whatsapp_answer: true + auto_subscribe: true + # dominant_speaker, mix_all, or selected_participant. + audio_uplink_policy: dominant_speaker + selected_participant_timeout: 30s + + audio: + enabled: true + jitter_buffer_ms: 180ms + opus_backend: libopus + silence_on_underrun: true + max_mix_participants: 4 + + video: + # Video is passed through as H.264. Other codecs are ignored. + enabled: false + # active_speaker or selected_participant. Both keep one group camera stable; + # change the selected participant with !wa call-video-select. + selected_source_policy: active_speaker + max_width: 1280 + max_height: 720 + max_fps: 30 + + diagnostics: + healthcheck_focus_on_startup: true + # Unsafe: meowcaller diagnostics can contain call secrets and media. + enable_meowcaller_diagnostics: false + media_trace_dir: null + # Settings for converting animated stickers. animated_sticker: # Format to which animated stickers should be converted. diff --git a/pkg/connector/handlematrix.go b/pkg/connector/handlematrix.go index c55f523..9aea18b 100644 --- a/pkg/connector/handlematrix.go +++ b/pkg/connector/handlematrix.go @@ -103,14 +103,11 @@ func (wa *WhatsAppClient) handleConvertedMatrixMessage(ctx context.Context, msg if err != nil { return nil, err } - if chatJID.Server == types.DefaultUserServer { - zerolog.Ctx(ctx).Warn().Stringer("portal_jid", chatJID).Msg("Matrix message received in phone number portal") - } if chatJID == types.StatusBroadcastJID && wa.Main.Config.DisableStatusBroadcastSend { return nil, ErrBroadcastSendDisabled } wrappedMsgID := waid.MakeMessageID(chatJID, wa.JID, req.ID) - wrappedMsgID2 := waid.MakeMessageID(chatJID, wa.GetLID(), req.ID) + wrappedMsgID2 := waid.MakeMessageID(chatJID, wa.GetStore().GetLID(), req.ID) msg.AddPendingToIgnore(networkid.TransactionID(wrappedMsgID)) msg.AddPendingToIgnore(networkid.TransactionID(wrappedMsgID2)) zerolog.Ctx(ctx).Trace().Any("payload", waMsg).Msg("Outgoing message payload") @@ -119,7 +116,7 @@ func (wa *WhatsAppClient) handleConvertedMatrixMessage(ctx context.Context, msg return nil, err } var pickedMessageID networkid.MessageID - if resp.Sender == wa.GetLID() { + if resp.Sender == wa.GetStore().GetLID() && chatJID.Server != types.DefaultUserServer { pickedMessageID = wrappedMsgID2 msg.RemovePending(networkid.TransactionID(wrappedMsgID)) } else { @@ -140,17 +137,18 @@ func (wa *WhatsAppClient) handleConvertedMatrixMessage(ctx context.Context, msg }, nil } -func (wa *WhatsAppClient) PreHandleMatrixReaction(ctx context.Context, msg *bridgev2.MatrixReaction) (bridgev2.MatrixReactionPreResponse, error) { +func (wa *WhatsAppClient) PreHandleMatrixReaction(_ context.Context, msg *bridgev2.MatrixReaction) (bridgev2.MatrixReactionPreResponse, error) { portalJID, err := waid.ParsePortalID(msg.Portal.ID) if err != nil { return bridgev2.MatrixReactionPreResponse{}, fmt.Errorf("failed to parse portal ID: %w", err) } else if portalJID == types.StatusBroadcastJID { return bridgev2.MatrixReactionPreResponse{}, ErrBroadcastReactionUnsupported } - sender := wa.GetLID() - if portalJID.Server == types.DefaultUserServer { - zerolog.Ctx(ctx).Warn().Stringer("portal_jid", portalJID).Msg("Matrix reaction received in phone number portal") - sender = wa.JID + sender := wa.JID + if portalJID.Server == types.HiddenUserServer || + msg.Portal.Metadata.(*waid.PortalMetadata).CommunityAnnouncementGroup || + msg.Portal.Metadata.(*waid.PortalMetadata).AddressingMode == types.AddressingModeLID { + sender = wa.GetStore().GetLID() } return bridgev2.MatrixReactionPreResponse{ SenderID: waid.MakeUserID(sender), @@ -322,7 +320,7 @@ func (wa *WhatsAppClient) HandleMatrixReadReceipt(ctx context.Context, receipt * if err != nil { continue } - if wa.IsOwnJID(parsed.Sender) { + if parsed.Sender.User == wa.GetStore().GetLID().User || parsed.Sender.User == wa.JID.User { continue } var key types.JID @@ -638,7 +636,7 @@ func (wa *WhatsAppClient) getLastMessageInfo(ctx context.Context, chatJID types. lastTS = msgs[0].Timestamp parsed, _ := waid.ParseMessageID(msgs[0].ID) if parsed != nil { - fromMe := wa.IsOwnJID(parsed.Sender) + fromMe := parsed.Sender.ToNonAD() == wa.JID.ToNonAD() || parsed.Sender.ToNonAD() == wa.GetStore().GetLID().ToNonAD() var participant *string if chatJID.Server == types.GroupServer { participant = ptr.Ptr(parsed.Sender.String()) diff --git a/pkg/connector/handlewhatsapp.go b/pkg/connector/handlewhatsapp.go index 509c589..99845ac 100644 --- a/pkg/connector/handlewhatsapp.go +++ b/pkg/connector/handlewhatsapp.go @@ -93,7 +93,9 @@ func (wa *WhatsAppClient) handleWAEvent(rawEvt any) (success bool) { success = wa.handleWACallStart(ctx, evt.GroupJID, evt.CallCreator, evt.CallCreatorAlt, evt.CallID, "", evt.Timestamp) case *events.CallOfferNotice: success = wa.handleWACallStart(ctx, evt.GroupJID, evt.CallCreator, evt.CallCreatorAlt, evt.CallID, evt.Type, evt.Timestamp) - case *events.CallTerminate, *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: + case *events.CallTerminate: + wa.clearIncomingCallGroup(evt.CallID) + case *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: // ignore case *events.IdentityChange: wa.handleWAIdentityChange(ctx, evt) @@ -113,7 +115,8 @@ func (wa *WhatsAppClient) handleWAEvent(rawEvt any) (success bool) { case *events.HistorySync: wa.UserLogin.Log.Warn().Msg("Unexpected history sync event received") case *events.MediaRetry: - success = wa.handleWAMediaRetry(ctx, evt) + wa.phoneSeen(evt.Timestamp) + success = wa.UserLogin.QueueRemoteEvent(&WAMediaRetry{MediaRetry: evt, wa: wa}).Success case *events.GroupInfo: success = wa.handleWAGroupInfoChange(ctx, evt) @@ -157,7 +160,7 @@ func (wa *WhatsAppClient) handleWAEvent(rawEvt any) (success bool) { if err != nil { log.Err(err).Msg("Failed to update push name in store") } - _, _, err = wa.GetStore().Contacts.PutPushName(ctx, wa.GetLID().ToNonAD(), evt.Action.GetName()) + _, _, err = wa.GetStore().Contacts.PutPushName(ctx, wa.GetStore().GetLID().ToNonAD(), evt.Action.GetName()) if err != nil { log.Err(err).Msg("Failed to update push name in store") } @@ -258,26 +261,65 @@ func (wa *WhatsAppClient) handleWAEvent(rawEvt any) (success bool) { return } -func (wa *WhatsAppClient) ensureAltJIDs(ctx context.Context, info *types.MessageSource, checkPhones bool) bool { - var err error - if info.Sender.Server == types.DefaultUserServer && info.SenderAlt.IsEmpty() { - info.SenderAlt, err = wa.GetStore().LIDs.GetLIDForPN(ctx, info.Sender) +func (wa *WhatsAppClient) rerouteWAMessage(ctx context.Context, evtType string, info *types.MessageSource, msgID any) { + if (info.Chat.Server == types.HiddenUserServer || info.Chat.Server == types.BroadcastServer) && + info.Sender.Server == types.HiddenUserServer && info.SenderAlt.IsEmpty() { + info.SenderAlt, _ = wa.GetStore().LIDs.GetPNForLID(ctx, info.Sender) + } + if info.Chat.Server == types.HiddenUserServer && info.IsFromMe && info.RecipientAlt.IsEmpty() { + info.RecipientAlt, _ = wa.GetStore().LIDs.GetPNForLID(ctx, info.Chat) + } + if info.Chat.Server == types.HiddenUserServer && info.Sender.ToNonAD() == info.Chat && info.SenderAlt.Server == types.DefaultUserServer { + wa.UserLogin.Log.Debug(). + Stringer("lid", info.Sender). + Stringer("pn", info.SenderAlt). + Any("message_id", msgID). + Str("evt_type", evtType). + Msg("Forced LID DM sender to phone number in incoming message") + info.Sender, info.SenderAlt = info.SenderAlt, info.Sender + info.Chat = info.Sender.ToNonAD() + } else if info.Chat.Server == types.HiddenUserServer && info.IsFromMe && info.RecipientAlt.Server == types.DefaultUserServer { + wa.UserLogin.Log.Debug(). + Stringer("lid", info.Chat). + Stringer("pn", info.RecipientAlt). + Any("message_id", msgID). + Str("evt_type", evtType). + Msg("Forced LID DM sender to phone number in own message sent from another device") + info.Chat = info.RecipientAlt.ToNonAD() + if info.Sender.Server == types.HiddenUserServer { + info.Sender, info.SenderAlt = info.SenderAlt, info.Sender + if info.Sender.IsEmpty() { + info.Sender = wa.GetStore().GetJID() + info.Sender.Device = info.SenderAlt.Device + } + } + } else if info.Chat.Server == types.BroadcastServer && info.Sender.Server == types.HiddenUserServer && info.SenderAlt.Server == types.DefaultUserServer { + wa.UserLogin.Log.Debug(). + Stringer("lid", info.Sender). + Stringer("pn", info.SenderAlt). + Stringer("chat", info.Chat). + Any("message_id", msgID). + Str("evt_type", evtType). + Msg("Forced LID broadcast list sender to phone number in incoming message") + info.Sender, info.SenderAlt = info.SenderAlt, info.Sender + } else if info.Sender.Server == types.BotServer && info.Chat.Server == types.HiddenUserServer { + chatPN, err := wa.GetStore().LIDs.GetPNForLID(ctx, info.Chat) if err != nil { - zerolog.Ctx(ctx).Err(err).Stringer("sender", info.Sender).Msg("Failed to get LID for sender") - return false + wa.UserLogin.Log.Err(err). + Any("message_id", msgID). + Stringer("lid", info.Chat). + Str("evt_type", evtType). + Msg("Failed to get phone number of DM for incoming bot message") + } else if !chatPN.IsEmpty() { + wa.UserLogin.Log.Debug(). + Stringer("lid", info.Chat). + Stringer("pn", chatPN). + Any("message_id", msgID). + Str("evt_type", evtType). + Msg("Forced LID chat to phone number in bot message") + info.Chat = chatPN } } - if info.Chat.Server == types.DefaultUserServer && info.IsFromMe && info.RecipientAlt.IsEmpty() { - info.RecipientAlt, err = wa.GetStore().LIDs.GetLIDForPN(ctx, info.Chat) - if err != nil { - zerolog.Ctx(ctx).Err(err).Stringer("chat", info.Chat).Msg("Failed to get LID for chat") - return false - } - } - if checkPhones { - return wa.checkAllPhonesInMessage(ctx, info) - } - return true } func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Message) (success bool) { @@ -285,12 +327,6 @@ func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Messa if evt.Info.Chat == types.StatusBroadcastJID && !wa.Main.Config.EnableStatusBroadcast { return } - if evt.Info.Chat.Server == types.NewsletterServer && wa.disableNewsletter { - return - } - if !wa.ensureAltJIDs(ctx, &evt.Info.MessageSource, true) { - return false - } parsedMessageType := getMessageType(evt.Message) if encReact := evt.Message.GetEncReactionMessage(); encReact != nil { decrypted, err := wa.Client.DecryptReaction(ctx, evt) @@ -324,6 +360,8 @@ func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Messa evt.UnwrapRaw() parsedMessageType = getMessageType(evt.Message) } + origSource := evt.Info.MessageSource + wa.rerouteWAMessage(ctx, "message", &evt.Info.MessageSource, evt.Info.ID) wa.UserLogin.Log.Trace(). Any("info", evt.Info). Any("payload", evt.Message). @@ -374,8 +412,9 @@ func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Messa res := wa.UserLogin.QueueRemoteEvent(&WAMessageEvent{ MessageInfoWrapper: &MessageInfoWrapper{ - Info: evt.Info, - wa: wa, + OrigSource: origSource, + Info: evt.Info, + wa: wa, }, Message: evt.Message, MsgEvent: evt, @@ -387,9 +426,7 @@ func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Messa } func (wa *WhatsAppClient) handleWAUndecryptableMessage(ctx context.Context, evt *events.UndecryptableMessage) bool { - if !wa.ensureAltJIDs(ctx, &evt.Info.MessageSource, true) { - return false - } + wa.rerouteWAMessage(ctx, "undecryptable message", &evt.Info.MessageSource, evt.Info.ID) wa.UserLogin.Log.Debug(). Any("info", evt.Info). Bool("unavailable", evt.IsUnavailable). @@ -412,44 +449,12 @@ func (wa *WhatsAppClient) handleWAUndecryptableMessage(ctx context.Context, evt return res.Success } -func (wa *WhatsAppClient) handleWAMediaRetry(ctx context.Context, evt *events.MediaRetry) bool { - wa.phoneSeen(evt.Timestamp) - var senderLID, chatLID types.JID - var err error - if evt.SenderID.Server == types.DefaultUserServer { - senderLID, err = wa.GetStore().LIDs.GetLIDForPN(ctx, evt.SenderID) - if err != nil { - wa.UserLogin.Log.Err(err). - Stringer("sender_id", evt.SenderID). - Msg("Failed to get LID for media retry sender") - return false - } - } - if evt.ChatID.Server == types.DefaultUserServer { - chatLID, err = wa.GetStore().LIDs.GetLIDForPN(ctx, evt.ChatID) - if err != nil { - wa.UserLogin.Log.Err(err). - Stringer("chat_id", evt.ChatID). - Msg("Failed to get LID for media retry chat") - return false - } - } - res := wa.UserLogin.QueueRemoteEvent(&WAMediaRetry{ - MediaRetry: evt, - wa: wa, - senderLID: senderLID, - chatLID: chatLID, - }) - return res.Success -} - func (wa *WhatsAppClient) handleWAReceipt(ctx context.Context, evt *events.Receipt) (success bool) { + origChat := evt.Chat + wa.rerouteWAMessage(ctx, "receipt", &evt.MessageSource, evt.MessageIDs) if evt.IsFromMe && evt.Sender.Device == 0 { wa.phoneSeen(evt.Timestamp) } - if !wa.ensureAltJIDs(ctx, &evt.MessageSource, true) { - return false - } var evtType bridgev2.RemoteEventType switch evt.Type { case types.ReceiptTypeRead, types.ReceiptTypeReadSelf: @@ -461,34 +466,28 @@ func (wa *WhatsAppClient) handleWAReceipt(ctx context.Context, evt *events.Recei default: return true } - targets := make([]networkid.MessageID, 0, len(evt.MessageIDs)) - messageSender := wa.GetLID() + targets := make([]networkid.MessageID, len(evt.MessageIDs)) + messageSender := wa.JID if !evt.MessageSender.IsEmpty() { messageSender = evt.MessageSender - } - var chatAlt types.JID - if evt.Chat.Server == types.DefaultUserServer { - chatLID, _ := wa.GetStore().LIDs.GetLIDForPN(ctx, evt.Chat) - if !chatLID.IsEmpty() { - chatAlt = evt.Chat - evt.Chat = chatLID + // Second part of rerouting receipts in LID chats + if messageSender == origChat && evt.Chat != origChat { + messageSender = evt.Chat + } + } else if evt.Chat.Server == types.GroupServer && evt.Sender.Server == types.HiddenUserServer { + lid := wa.GetStore().GetLID() + if !lid.IsEmpty() { + messageSender = lid } } - for _, id := range evt.MessageIDs { - targets = append(targets, waid.MakeMessageID(evt.Chat, messageSender, id)) - if !chatAlt.IsEmpty() { - targets = append(targets, waid.MakeMessageID(chatAlt, messageSender, id)) - } - } - senderLID := evt.Sender - if senderLID.Server == types.DefaultUserServer && !evt.SenderAlt.IsEmpty() { - senderLID = evt.SenderAlt + for i, id := range evt.MessageIDs { + targets[i] = waid.MakeMessageID(evt.Chat, messageSender, id) } res := wa.UserLogin.QueueRemoteEvent(&simplevent.Receipt{ EventMeta: simplevent.EventMeta{ Type: evtType, PortalKey: wa.makeWAPortalKey(evt.Chat), - Sender: wa.makeEventSender(ctx, senderLID), + Sender: wa.makeEventSender(ctx, evt.Sender), Timestamp: evt.Timestamp, }, Targets: targets, @@ -497,11 +496,11 @@ func (wa *WhatsAppClient) handleWAReceipt(ctx context.Context, evt *events.Recei } func (wa *WhatsAppClient) handleWAChatPresence(ctx context.Context, evt *events.ChatPresence) { - if evt.Chat.Server == types.DefaultUserServer && evt.Sender.ToNonAD() == evt.Chat { + if evt.Chat.Server == types.HiddenUserServer && evt.Sender.ToNonAD() == evt.Chat { if evt.SenderAlt.IsEmpty() { - evt.SenderAlt, _ = wa.GetStore().LIDs.GetLIDForPN(ctx, evt.Sender) + evt.SenderAlt, _ = wa.GetStore().LIDs.GetPNForLID(ctx, evt.Sender) } - if evt.SenderAlt.Server == types.HiddenUserServer { + if evt.SenderAlt.Server == types.DefaultUserServer { evt.Sender, evt.SenderAlt = evt.SenderAlt, evt.Sender evt.Chat = evt.Sender.ToNonAD() } @@ -538,7 +537,6 @@ func (wa *WhatsAppClient) handleWALogout(reason events.ConnectFailureReason, onC wa.Disconnect() wa.Client = nil wa.JID = types.EmptyJID - wa.LID = types.EmptyJID wa.UserLogin.Metadata.(*waid.UserLoginMetadata).WADeviceID = 0 wa.UserLogin.BridgeState.Send(status.BridgeState{ StateEvent: status.StateBadCredentials, @@ -552,15 +550,12 @@ func (wa *WhatsAppClient) handleWACallStart(ctx context.Context, group, sender, if !wa.Main.Config.CallStartNotices || time.Since(ts) > callEventMaxAge { return true } - if sender.Server == types.DefaultUserServer && senderAlt.IsEmpty() { - senderAlt, _ = wa.GetStore().LIDs.GetLIDForPN(ctx, sender) - } - if sender.Server == types.DefaultUserServer && senderAlt.Server == types.HiddenUserServer { + if sender.Server == types.HiddenUserServer && senderAlt.Server == types.DefaultUserServer { wa.UserLogin.Log.Debug(). - Stringer("lid", senderAlt). - Stringer("pn", sender). + Stringer("lid", sender). + Stringer("pn", senderAlt). Str("call_id", id). - Msg("Forced phone number caller to LID in incoming call") + Msg("Forced LID caller to phone number in incoming call") sender, senderAlt = senderAlt, sender } chat := group @@ -607,12 +602,6 @@ func (wa *WhatsAppClient) handleWAIdentityChange(ctx context.Context, evt *event if !wa.Main.Config.IdentityChangeNotices { return } - if evt.JID.Server == types.DefaultUserServer { - lid, _ := wa.GetStore().LIDs.GetLIDForPN(ctx, evt.JID) - if !lid.IsEmpty() { - evt.JID = lid - } - } wa.UserLogin.QueueRemoteEvent(&simplevent.Message[*events.IdentityChange]{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventMessage, @@ -663,14 +652,13 @@ func (wa *WhatsAppClient) handleWADeleteChat(ctx context.Context, evt *events.De func (wa *WhatsAppClient) handleWADeleteForMe(ctx context.Context, evt *events.DeleteForMe) bool { chatJID := wa.maybeConvertJIDToLID(ctx, evt.ChatJID) - senderJID := wa.maybeConvertJIDToLID(ctx, evt.SenderJID) return wa.UserLogin.QueueRemoteEvent(&simplevent.MessageRemove{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventMessageRemove, PortalKey: wa.makeWAPortalKey(chatJID), Timestamp: evt.Timestamp, }, - TargetMessage: waid.MakeMessageID(chatJID, senderJID, evt.MessageID), + TargetMessage: waid.MakeMessageID(chatJID, evt.SenderJID, evt.MessageID), OnlyForMe: true, }).Success } @@ -681,7 +669,7 @@ func (wa *WhatsAppClient) handleWAMarkChatAsRead(ctx context.Context, evt *event EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventReadReceipt, PortalKey: wa.makeWAPortalKey(chatJID), - Sender: wa.makeEventSender(ctx, wa.GetLID()), + Sender: wa.makeEventSender(ctx, wa.JID), Timestamp: evt.Timestamp, }, ReadUpTo: evt.Timestamp, @@ -770,6 +758,9 @@ func (wa *WhatsAppClient) handleWAGroupInfoChange(ctx context.Context, evt *even } func (wa *WhatsAppClient) handleWAJoinedGroup(ctx context.Context, evt *events.JoinedGroup) bool { + if wa.createDedup.Pop(evt.CreateKey) { + return true + } return wa.UserLogin.QueueRemoteEvent(&simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventChatResync, @@ -782,9 +773,6 @@ func (wa *WhatsAppClient) handleWAJoinedGroup(ctx context.Context, evt *events.J } func (wa *WhatsAppClient) handleWANewsletterJoin(ctx context.Context, evt *events.NewsletterJoin) bool { - if wa.disableNewsletter { - return true - } return wa.UserLogin.QueueRemoteEvent(&simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventChatResync, @@ -883,12 +871,7 @@ func (wa *WhatsAppClient) handleWAAppStateSyncComplete(ctx context.Context, evt } else { log.Info(). Time("recovery_ts", ts). - Bool("recovery_evt", evt.Recovery). Msg("Unmarked app state recovery as attempted after successful full sync") - wa.UserLogin.TrackAnalytics("WhatsApp Appstate Recovery Success", map[string]any{ - "patch_name": evt.Name, - "from_recovery": evt.Recovery, - }) } } else if ts, exists = wa.appStateFullSyncAttempted[evt.Name]; exists { delete(wa.appStateFullSyncAttempted, evt.Name) @@ -945,9 +928,6 @@ func (wa *WhatsAppClient) handleWAAppStateSyncError(ctx context.Context, evt *ev if err != nil { log.Err(err).Msg("Failed to save login metadata after marking app state recovery as attempted") } - wa.UserLogin.TrackAnalytics("WhatsApp Appstate Recovery Request", map[string]any{ - "patch_name": evt.Name, - }) go func() { resp, err := wa.Client.SendPeerMessage(ctx, whatsmeow.BuildAppStateRecoveryRequest(evt.Name)) if err != nil { diff --git a/pkg/connector/id.go b/pkg/connector/id.go index c0aa06b..c07e431 100644 --- a/pkg/connector/id.go +++ b/pkg/connector/id.go @@ -49,7 +49,7 @@ func (wa *WhatsAppClient) makeEventSender(ctx context.Context, id types.JID) bri } } return bridgev2.EventSender{ - IsFromMe: wa.IsOwnJID(id), + IsFromMe: id.User == wa.GetStore().GetJID().User || id.User == wa.GetStore().GetLID().User, Sender: waid.MakeUserID(id), SenderLogin: waid.MakeUserLoginID(senderLoginJID), } @@ -60,25 +60,24 @@ func (wa *WhatsAppClient) messageIDToKey(id *waid.ParsedMessageID) *waCommon.Mes RemoteJID: ptr.Ptr(id.Chat.String()), ID: ptr.Ptr(id.ID), } - if wa.IsOwnJID(id.Sender) { + if id.Sender.User == wa.GetStore().GetJID().User || id.Sender.User == wa.GetStore().GetLID().User { key.FromMe = ptr.Ptr(true) } - if id.Chat.Server != types.MessengerServer && id.Chat.Server != types.DefaultUserServer && - id.Chat.Server != types.HiddenUserServer && id.Chat.Server != types.BotServer { + if id.Chat.Server != types.MessengerServer && id.Chat.Server != types.DefaultUserServer && id.Chat.Server != types.HiddenUserServer && id.Chat.Server != types.BotServer { key.Participant = ptr.Ptr(id.Sender.String()) } return key } -func (wa *WhatsAppClient) maybeConvertJIDToLID(ctx context.Context, jid types.JID) types.JID { - if jid.Server == types.DefaultUserServer { - if lidForPN, err := wa.GetStore().LIDs.GetLIDForPN(ctx, jid); err != nil { +func (wa *WhatsAppClient) maybeConvertJIDToLID(ctx context.Context, chatJID types.JID) types.JID { + if chatJID.Server == types.HiddenUserServer { + if pn, err := wa.GetStore().LIDs.GetPNForLID(ctx, chatJID); err != nil { wa.UserLogin.Log.Err(err). - Stringer("pn", jid). - Msg("Failed to get LID for phone number chat") - } else if !lidForPN.IsEmpty() { - return lidForPN + Stringer("lid", chatJID). + Msg("Failed to get phone number for LID chat") + } else if !pn.IsEmpty() { + return pn.ToNonAD() } } - return jid + return chatJID } diff --git a/pkg/connector/lidmigrate.go b/pkg/connector/lidmigrate.go deleted file mode 100644 index cfab099..0000000 --- a/pkg/connector/lidmigrate.go +++ /dev/null @@ -1,212 +0,0 @@ -// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2026 Tulir Asokan -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package connector - -import ( - "context" - "fmt" - "time" - - "github.com/rs/zerolog" - "go.mau.fi/util/dbutil" - "go.mau.fi/whatsmeow/types" - - "maunium.net/go/mautrix/bridgev2" - "maunium.net/go/mautrix/bridgev2/networkid" - "maunium.net/go/mautrix/event" - - "go.mau.fi/mautrix-whatsapp/pkg/waid" -) - -func (wa *WhatsAppClient) FindAltTargetMessage(ctx context.Context, targetMsg networkid.MessageID, evt bridgev2.RemoteEventWithTargetMessage) (alts []networkid.MessageID, err error) { - parsed, err := waid.ParseMessageID(targetMsg) - if err != nil { - return nil, fmt.Errorf("failed to parse target message ID: %w", err) - } - altSender, err := wa.GetStore().GetAltJID(ctx, parsed.Sender) - if err != nil { - return nil, err - } - var altChat types.JID - if parsed.Chat.Server == types.HiddenUserServer { - altChat, err = wa.GetStore().LIDs.GetPNForLID(ctx, parsed.Chat) - if err != nil { - return nil, err - } - } - if !altSender.IsEmpty() { - altSenderID := *parsed - altSenderID.Sender = altSender - alts = append(alts, altSenderID.String()) - } - if !altChat.IsEmpty() { - altChatID := *parsed - altChatID.Chat = altChat - if altSender.Server == types.DefaultUserServer { - altChatID.Sender = altSender - } - alts = append(alts, altChatID.String()) - } - return -} - -func (wa *WhatsAppClient) checkAllPhonesInMessage(ctx context.Context, info *types.MessageSource) (ok bool) { - for _, jid := range []types.JID{info.Sender, info.SenderAlt, info.Chat, info.RecipientAlt, info.BroadcastListOwner} { - if !wa.reIDPhoneDMToLIDIfNeeded(ctx, jid) { - return false - } - } - return true -} - -func (wa *WhatsAppClient) reIDPhoneDMToLIDIfNeeded(ctx context.Context, pn types.JID) (ok bool) { - if pn.Server != types.DefaultUserServer { - return true - } - portalKey := wa.makeWAPortalKey(pn) - if wa.Main.unmigratedDMs.Has(portalKey) { - lid, err := wa.GetStore().LIDs.GetLIDForPN(ctx, pn) - if err != nil { - zerolog.Ctx(ctx).Err(err).Stringer("pn", pn).Msg("Failed to get LID for PN") - return false - } else if lid.IsEmpty() { - zerolog.Ctx(ctx).Warn().Stringer("pn", pn).Msg("No found LID for phone number") - return true - } - zerolog.Ctx(ctx).Info(). - Object("portal_key", portalKey). - Stringer("pn", pn). - Stringer("lid", lid). - Msg("Received event for portal in unmigrated DMs list, trying migration") - _, err = wa.Main.reIDPhoneDMToLID(ctx, pn, lid, wa.UserLogin.ID) - if err != nil { - zerolog.Ctx(ctx).Err(err).Msg("Failed to re-ID phone DM to LID") - return false - } - } - return true -} - -func (wa *WhatsAppConnector) reIDPhoneDMToLID(ctx context.Context, pn, lid types.JID, receiver networkid.UserLoginID) (bridgev2.ReIDResult, error) { - pnKey := networkid.PortalKey{ - ID: waid.MakePortalID(pn), - Receiver: receiver, - } - lidKey := networkid.PortalKey{ - ID: waid.MakePortalID(lid), - Receiver: receiver, - } - result, portal, err := wa.Bridge.ReIDPortal(ctx, pnKey, lidKey) - if err != nil { - return result, err - } - if result == bridgev2.ReIDResultSourceReIDd || result == bridgev2.ReIDResultTargetDeletedAndSourceReIDd { - var pnGhost, lidGhost *bridgev2.Ghost - pnGhost, err = wa.Bridge.GetGhostByID(ctx, waid.MakeUserID(pn)) - if err != nil { - return result, fmt.Errorf("failed to get PN ghost: %w", err) - } - lidGhost, err = wa.Bridge.GetGhostByID(ctx, waid.MakeUserID(lid)) - if err != nil { - return result, fmt.Errorf("failed to get LID ghost: %w", err) - } - _, err = pnGhost.Intent.SendState(ctx, portal.MXID, event.StateMember, pnGhost.Intent.GetMXID().String(), &event.Content{ - Parsed: &event.MemberEventContent{Membership: event.MembershipLeave, Reason: "Migrating to LIDs"}, - Raw: map[string]any{"com.beeper.exclude_from_timeline": true}, - }, time.Time{}) - if err != nil { - return result, fmt.Errorf("failed to send leave event for PN ghost: %w", err) - } - _, err = wa.Bridge.Bot.SendState(ctx, portal.MXID, event.StateMember, lidGhost.Intent.GetMXID().String(), &event.Content{ - Parsed: &event.MemberEventContent{Membership: event.MembershipInvite, Reason: "Migrating to LIDs"}, - Raw: map[string]any{"com.beeper.exclude_from_timeline": true}, - }, time.Time{}) - if err != nil { - return result, fmt.Errorf("failed to send invite event for LID ghost: %w", err) - } - _, err = lidGhost.Intent.SendState(ctx, portal.MXID, event.StateMember, lidGhost.Intent.GetMXID().String(), &event.Content{ - Parsed: &event.MemberEventContent{Membership: event.MembershipJoin, Reason: "Migrating to LIDs"}, - Raw: map[string]any{"com.beeper.exclude_from_timeline": true}, - }, time.Time{}) - if err != nil { - return result, fmt.Errorf("failed to send join event for LID ghost: %w", err) - } - portal.OtherUserID = lidGhost.ID - err = portal.Save(ctx) - if err != nil { - return result, fmt.Errorf("failed to save portal after re-ID: %w", err) - } - portal.UpdateBridgeInfo(ctx) - } - return result, nil -} - -var scanPortalKey = dbutil.ConvertRowFn[networkid.PortalKey](func(row dbutil.Scannable) (key networkid.PortalKey, err error) { - err = row.Scan(&key.ID, &key.Receiver) - return -}) - -func (wa *WhatsAppConnector) migrateToLIDDMs(ctx context.Context) error { - if wa.Bridge.Background { - if wa.Bridge.DB.KV.Get(ctx, "whatsapp_lid_dms_migrated") == "true" { - return nil - } - return fmt.Errorf("can't migrate to LID DMs in background mode") - } - log := zerolog.Ctx(ctx).With().Str("action", "migrate to lid dms").Logger() - const findPNPortals = "SELECT id, receiver FROM portal WHERE bridge_id=$1 AND room_type='dm' AND id LIKE '%@s.whatsapp.net'" - pnPortalKeys, err := scanPortalKey.NewRowIter(wa.Bridge.DB.Query(ctx, findPNPortals, wa.Bridge.ID)).AsList() - if err != nil { - return fmt.Errorf("failed to get phone number portals: %w", err) - } - var updatedPortals, missingLID int - for _, key := range pnPortalKeys { - pnJID, err := waid.ParsePortalID(key.ID) - if err != nil { - log.Warn().Err(err).Str("portal_id", string(key.ID)).Msg("Failed to parse portal ID") - continue - } else if pnJID.Server != types.DefaultUserServer { - continue - } - lid, err := wa.DeviceStore.LIDMap.GetLIDForPN(ctx, pnJID) - if err != nil { - return fmt.Errorf("failed to get LID for PN portal %s: %w", key.ID, err) - } else if lid.IsEmpty() { - log.Warn().Stringer("pn", pnJID).Msg("No LID for PN portal") - wa.unmigratedDMs.Add(key) - missingLID++ - continue - } - res, err := wa.reIDPhoneDMToLID(ctx, pnJID, lid, key.Receiver) - if err != nil { - return fmt.Errorf("failed to re-ID %s to %s: %w", pnJID, lid, err) - } - updatedPortals++ - log.Info(). - Stringer("pn", pnJID). - Stringer("lid", lid). - Stringer("result", res). - Msg("Re-ID'd phone number DM portal") - } - log.Info(). - Int("updated_portals", updatedPortals). - Int("total_pn_portals", len(pnPortalKeys)). - Int("missing_lid", missingLID). - Msg("Finished re-IDing phone number DM portals") - wa.Bridge.DB.KV.Set(ctx, "whatsapp_lid_dms_migrated", "true") - return nil -} diff --git a/pkg/connector/login.go b/pkg/connector/login.go index a62b294..5da419f 100644 --- a/pkg/connector/login.go +++ b/pkg/connector/login.go @@ -135,8 +135,7 @@ var ( _ bridgev2.LoginProcessWebAuthn = (*WALogin)(nil) ) -const LoginConnectWait = 30 * time.Second -const LoginPairPhoneWait = 30 * time.Second +const LoginConnectWait = 15 * time.Second func (wl *WALogin) Start(ctx context.Context) (*bridgev2.LoginStep, error) { wl.Main.firstClientConnectOnce.Do(wl.Main.onFirstClientConnect) @@ -194,21 +193,19 @@ func (wl *WALogin) StartWithOverride(ctx context.Context, old *bridgev2.UserLogi } func (wl *WALogin) SubmitUserInput(ctx context.Context, input map[string]string) (*bridgev2.LoginStep, error) { + ctx, cancel := context.WithTimeout(ctx, LoginConnectWait) + defer cancel() err := wl.Client.Connect() if err != nil { wl.Log.Err(err).Msg("Failed to connect to WhatsApp for phone code login") return nil, err } - connectCtx, cancelConnect := context.WithTimeout(ctx, LoginConnectWait) - err = wl.WaitForQRs.Wait(connectCtx) - cancelConnect() + err = wl.WaitForQRs.Wait(ctx) if err != nil { wl.Log.Warn().Err(err).Msg("Timed out waiting for connection") return nil, fmt.Errorf("failed to wait for connection: %w", err) } - pairCtx, cancelPair := context.WithTimeout(ctx, LoginPairPhoneWait) - defer cancelPair() - pairingCode, err := wl.Client.PairPhone(pairCtx, input["phone_number"], true, whatsmeow.PairClientChrome, "Chrome (Linux)") + pairingCode, err := wl.Client.PairPhone(ctx, input["phone_number"], true, whatsmeow.PairClientChrome, "Chrome (Linux)") if err != nil { wl.Log.Err(err).Msg("Failed to request phone code login") if errors.Is(err, whatsmeow.ErrPhoneNumberTooShort) { diff --git a/pkg/connector/matrixrtc.go b/pkg/connector/matrixrtc.go new file mode 100644 index 0000000..7a232e6 --- /dev/null +++ b/pkg/connector/matrixrtc.go @@ -0,0 +1,340 @@ +package connector + +import ( + "context" + "strings" + "time" + + "github.com/rs/zerolog" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +const ( + matrixRTCHealthcheckTimeout = 15 * time.Second + matrixRTCOutboundStartDedupWindow = 30 * time.Second +) + +func withoutCancelOrBackground(ctx context.Context) context.Context { + if ctx == nil { + return context.Background() + } + return context.WithoutCancel(ctx) +} + +func (wa *WhatsAppConnector) initMatrixRTCEventHooks() { + matrixConnector, ok := wa.Bridge.Matrix.(*matrix.Connector) + if !ok || matrixConnector.EventProcessor == nil { + wa.Bridge.Log.Debug().Msg("Matrix connector does not expose an event processor for MatrixRTC hooks") + return + } + for _, evtType := range voip.SupportedMatrixRTCEventTypes() { + matrixConnector.EventProcessor.On(evtType, wa.handleMatrixRTCEvent) + } + wa.Bridge.Log.Debug().Int("event_type_count", len(voip.SupportedMatrixRTCEventTypes())).Msg("Registered MatrixRTC event hooks") +} + +func (wa *WhatsAppConnector) startMatrixRTCHealthcheck() { + if !wa.Config.VOIP.Enabled || !wa.Config.VOIP.Diagnostics.HealthcheckFocusOnStartup { + return + } + go func() { + ctx, cancel := context.WithTimeout(withoutCancelOrBackground(wa.Bridge.BackgroundCtx), matrixRTCHealthcheckTimeout) + defer cancel() + focus, err := voip.DiscoverLiveKitFocus(ctx, nil, wa.Bridge.Matrix.ServerName(), wa.Config.VOIP.MatrixRTC.LiveKitServiceURL) + log := wa.Bridge.Log.With().Str("component", "voip_healthcheck").Logger() + if err != nil { + event := log.Warn() + if wa.Config.VOIP.MatrixRTC.RequireLiveKitFocus { + event = log.Error() + } + event.Err(err).Msg("MatrixRTC LiveKit focus healthcheck failed") + return + } + log.Info(). + Str("focus_type", focus.Type). + Str("livekit_service_url", focus.LiveKitServiceURL). + Msg("MatrixRTC LiveKit focus healthcheck passed") + }() +} + +func (wa *WhatsAppConnector) handleMatrixRTCEvent(ctx context.Context, evt *event.Event) { + if !wa.Config.VOIP.Enabled { + return + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + return + } + + log := zerolog.Ctx(ctx).With(). + Str("matrixrtc_kind", string(parsed.Kind)). + Str("matrix_event_type", parsed.Type.Type). + Stringer("matrix_room_id", parsed.RoomID). + Stringer("matrix_sender", parsed.Sender). + Str("matrix_call_id", parsed.CallID). + Logger() + + if parsed.Sender == wa.Bridge.Bot.GetMXID() || wa.Bridge.IsGhostMXID(parsed.Sender) { + log.Debug().Msg("Ignoring MatrixRTC event sent by the bridge") + return + } + if !wa.Bridge.Config.Permissions.Get(parsed.Sender).SendEvents { + log.Debug().Msg("Dropping MatrixRTC event from user with no permission to send events") + wa.Bridge.Matrix.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt)) + return + } + + portal, err := wa.Bridge.GetPortalByMXID(ctx, parsed.RoomID) + if err != nil { + log.Err(err).Msg("Failed to look up portal for MatrixRTC event") + return + } else if portal == nil { + log.Debug().Msg("Ignoring MatrixRTC event outside a bridged portal") + return + } + portalLogin := wa.Bridge.GetCachedUserLoginByID(portal.Receiver) + if !matrixRTCSenderOwnsLogin(parsed.Sender, portalLogin) { + log.Warn(). + Str("portal_receiver", string(portal.Receiver)). + Msg("Dropping MatrixRTC event from a user who does not own the portal login") + wa.Bridge.Matrix.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt)) + return + } + + activeCalls, err := wa.DB.MatrixRTCCall.GetActiveInRoom(ctx, parsed.RoomID) + if err != nil { + log.Err(err).Msg("Failed to look up active MatrixRTC calls for room") + return + } + if isMatrixRTCCallControlEvent(parsed) { + wa.handleMatrixRTCCallControlEvent(ctx, parsed, activeCalls, log) + return + } + if len(activeCalls) == 0 { + if !shouldStartOutboundMatrixRTCCall(evt, parsed, wa.Config.VOIP.MatrixRTC.MembershipEventCompat) { + log.Debug().Msg("Ignoring MatrixRTC event without active bridged calls in the room") + return + } + if !wa.reserveMatrixRTCOutboundStart(parsed.RoomID.String()) { + log.Debug().Msg("Ignoring duplicate outbound MatrixRTC start in dedupe window") + return + } + if err = wa.startOutboundMatrixRTCCall(ctx, portal, parsed); err != nil { + log.Err(err).Msg("Failed to start outbound WhatsApp call from MatrixRTC event") + if cleanupErr := wa.cleanupFailedOutboundMatrixRTCStart(ctx, parsed); cleanupErr != nil { + log.Err(cleanupErr).Msg("Failed to clean up failed outbound MatrixRTC event") + } + } + return + } + + var matched, handled, activated, ended int + for _, activeCall := range activeCalls { + matched++ + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, activeCall.UserLoginID) + callLog := log.With(). + Str("wa_call_id", activeCall.WACallID). + Str("user_login_id", string(activeCall.UserLoginID)). + Stringer("matrix_participant_mxid", activeCall.MatrixParticipantMXID). + Str("matrix_session_id", activeCall.MatrixSessionID). + Logger() + if err != nil { + callLog.Err(err).Msg("Failed to look up WhatsApp login for MatrixRTC call") + continue + } else if login == nil { + callLog.Debug().Msg("MatrixRTC call references a missing WhatsApp login") + continue + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + callLog.Debug().Msg("WhatsApp login has no VOIP manager for MatrixRTC event") + continue + } + if matrixRTCMembershipEventMatchesCall(parsed, activeCall) && + activeCall.SelectedMembershipEventID != parsed.EventID { + if activeCall.SelectedHandRaiseEventID != "" { + if err = client.VOIP.SetHandRaised(activeCall.WACallID, false); err != nil { + callLog.Warn().Err(err).Msg("Failed to lower WhatsApp hand after MatrixRTC membership replacement") + } + activeCall.SelectedHandRaiseEventID = "" + } + activeCall.SelectedMembershipEventID = parsed.EventID + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + callLog.Err(err).Msg("Failed to persist replacement MatrixRTC membership event") + } + } + handled++ + endedCalls := client.VOIP.HandleMatrixRTCCallEvent(ctx, parsed, activeCall.WACallID) + if shouldEndMatrixRTCCallFromMembership(parsed, activeCall.SelectedPublisherID) { + endedCalls += client.VOIP.HandleMatrixRTCCallEvent(ctx, voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindRTCDecline, + RoomID: parsed.RoomID, + Sender: parsed.Sender, + CallID: parsed.CallID, + }, activeCall.WACallID) + } + ended += endedCalls + if shouldActivateMatrixRTCCall(parsed, activeCall.State) { + if err = client.activateMatrixRTCCall(ctx, activeCall, parsed); err != nil { + callLog.Err(err).Msg("Failed to activate MatrixRTC LiveKit bridge for WhatsApp call") + } else { + activated++ + } + } + if endedCalls > 0 { + err = wa.DB.MatrixRTCCall.MarkEnded(ctx, activeCall.UserLoginID, activeCall.WACallID, "ended", string(parsed.Kind), "", time.Now()) + if err != nil { + callLog.Err(err).Msg("Failed to mark MatrixRTC call ended after MatrixRTC event") + } + } + } + log.Debug(). + Int("active_call_count", len(activeCalls)). + Int("matched_call_count", matched). + Int("handled_call_count", handled). + Int("activated_call_count", activated). + Int("ended_call_count", ended). + Msg("Handled MatrixRTC event for active bridged calls") +} + +func matrixRTCSenderOwnsLogin(sender id.UserID, login *bridgev2.UserLogin) bool { + return sender != "" && login != nil && login.User != nil && login.User.MXID == sender +} + +func matrixRTCMembershipEventMatchesCall(evt voip.MatrixRTCEvent, call *wadb.MatrixRTCCall) bool { + if call == nil || evt.EventID == "" || !voip.MatrixRTCEventHasJoinContent(evt) { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + return matrixRTCEventMatchesParticipant(evt, call.SelectedPublisherID) + default: + return false + } +} + +func shouldActivateMatrixRTCCall(evt voip.MatrixRTCEvent, callState string) bool { + if callState != "ringing" { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + return voip.MatrixRTCEventHasJoinContent(evt) + default: + return false + } +} + +func shouldStartOutboundMatrixRTCCall(evt *event.Event, parsed voip.MatrixRTCEvent, membershipCompat string) bool { + if parsed.Type.Class != event.StateEventType || !voip.MatrixRTCEventHasJoinContent(parsed) { + return false + } + switch parsed.Kind { + case voip.MatrixRTCEventKindRTCMembership: + if !matrixRTCCompatAllowsModern(membershipCompat) { + return false + } + case voip.MatrixRTCEventKindGroupCallMember: + if !matrixRTCCompatAllowsLegacy(membershipCompat) { + return false + } + default: + return false + } + return !matrixRTCPrevContentHasJoinContent(evt) +} + +func shouldEndMatrixRTCCallFromMembership(evt voip.MatrixRTCEvent, selectedParticipantID string) bool { + switch evt.Kind { + case voip.MatrixRTCEventKindRTCMembership, voip.MatrixRTCEventKindGroupCallMember: + default: + return false + } + if selectedParticipantID == "" || voip.MatrixRTCEventHasJoinContent(evt) { + return false + } + return matrixRTCEventMatchesParticipant(evt, selectedParticipantID) +} + +func matrixRTCEventMatchesParticipant(evt voip.MatrixRTCEvent, participantID string) bool { + if participantID == "" { + return false + } + if matrixRTCTriggerParticipantID(evt) == participantID { + return true + } + if evt.StateKey == "" || evt.Sender == "" { + return false + } + if legacyID := legacyMatrixRTCParticipantIDFromStateKey(evt.Sender, evt.StateKey); legacyID == participantID { + return true + } + if modernID := modernMatrixRTCParticipantIDFromStateKey(evt.Sender, evt.StateKey); modernID == participantID { + return true + } + return false +} + +func legacyMatrixRTCParticipantIDFromStateKey(sender id.UserID, stateKey string) string { + prefix := "_" + string(sender) + "_" + const suffix = "_m.call" + if !strings.HasPrefix(stateKey, prefix) || !strings.HasSuffix(stateKey, suffix) { + return "" + } + deviceID := strings.TrimSuffix(strings.TrimPrefix(stateKey, prefix), suffix) + if deviceID == "" { + return "" + } + return voip.MatrixRTCMemberID(sender, deviceID) +} + +func modernMatrixRTCParticipantIDFromStateKey(sender id.UserID, stateKey string) string { + prefix := string(sender) + "_" + if !strings.HasPrefix(stateKey, prefix) { + return "" + } + deviceID := strings.TrimPrefix(stateKey, prefix) + if deviceID == "" { + return "" + } + return voip.MatrixRTCMemberID(sender, deviceID) +} + +func (wa *WhatsAppConnector) reserveMatrixRTCOutboundStart(roomID string) bool { + if roomID == "" { + return false + } + now := time.Now() + wa.matrixRTCOutboundStartLock.Lock() + defer wa.matrixRTCOutboundStartLock.Unlock() + if wa.matrixRTCOutboundStartExpires == nil { + wa.matrixRTCOutboundStartExpires = make(map[string]time.Time) + } + for trackedRoomID, expires := range wa.matrixRTCOutboundStartExpires { + if !expires.After(now) { + delete(wa.matrixRTCOutboundStartExpires, trackedRoomID) + } + } + if expires := wa.matrixRTCOutboundStartExpires[roomID]; expires.After(now) { + return false + } + wa.matrixRTCOutboundStartExpires[roomID] = now.Add(matrixRTCOutboundStartDedupWindow) + return true +} + +func matrixRTCPrevContentHasJoinContent(evt *event.Event) bool { + if evt == nil || evt.Unsigned.PrevContent == nil { + return false + } + prevEvt := *evt + prevEvt.Content = *evt.Unsigned.PrevContent + prevEvt.Unsigned.PrevContent = nil + parsedPrev, ok := voip.ParseMatrixRTCEvent(&prevEvt) + return ok && voip.MatrixRTCEventHasJoinContent(parsedPrev) +} diff --git a/pkg/connector/matrixrtc_outgoing.go b/pkg/connector/matrixrtc_outgoing.go new file mode 100644 index 0000000..2ddbc4e --- /dev/null +++ b/pkg/connector/matrixrtc_outgoing.go @@ -0,0 +1,950 @@ +package connector + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + "github.com/purpshell/meowcaller" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + mxbridge "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" + "go.mau.fi/mautrix-whatsapp/pkg/waid" +) + +const ( + matrixRTCRingLifetime = 90 * time.Second + matrixRTCMembershipLifetime = 4 * time.Hour + matrixRTCStickyDuration = time.Hour +) + +type incomingCallGroup struct { + JID types.JID + ExpiresAt time.Time +} + +func (wa *WhatsAppClient) trackIncomingCallEvent(rawEvt any) { + switch evt := rawEvt.(type) { + case *events.CallOffer: + wa.trackIncomingCallGroup(evt.CallID, evt.GroupJID) + case *events.CallOfferNotice: + wa.trackIncomingCallGroup(evt.CallID, evt.GroupJID) + case *events.CallTerminate: + wa.clearIncomingCallGroup(evt.CallID) + } +} + +func (wa *WhatsAppClient) handleIncomingVOIPCall(call *meowcaller.Call) { + if call == nil { + return + } + ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) + err := wa.announceIncomingMatrixRTCCall(ctx, call) + if err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Msg("Failed to announce incoming WhatsApp call over MatrixRTC") + } +} + +func (wa *WhatsAppClient) handleVOIPCallEnded(callID, reason string) { + wa.clearIncomingCallGroup(callID) + wa.clearWhatsAppRemoteHandRaises(callID) + ctx := wa.UserLogin.Log.WithContext(withoutCancelOrBackground(wa.Main.Bridge.BackgroundCtx)) + log := wa.UserLogin.Log.With().Str("call_id", callID).Str("reason", reason).Logger() + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil { + log.Err(err).Msg("Failed to look up MatrixRTC call record after WhatsApp call ended") + return + } else if call == nil { + return + } + if err = wa.clearMatrixRTCMembership(ctx, call); err != nil { + log.Err(err).Msg("Failed to clear MatrixRTC membership after WhatsApp call ended") + } + endReason, lastError := matrixRTCFinalEndReason(call, reason) + if err = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, wa.UserLogin.ID, callID, "ended", endReason, lastError, time.Now()); err != nil { + log.Err(err).Msg("Failed to mark MatrixRTC call ended") + } +} + +func (wa *WhatsAppClient) cleanupStaleMatrixRTCCalls(ctx context.Context) { + if wa == nil || wa.Main == nil || wa.Main.DB == nil || wa.UserLogin == nil { + return + } + calls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + wa.UserLogin.Log.Err(err).Msg("Failed to query stale MatrixRTC calls during login load") + return + } + for _, call := range calls { + if call == nil { + continue + } + lastError := "" + if clearErr := wa.clearMatrixRTCMembership(ctx, call); clearErr != nil { + lastError = clearErr.Error() + wa.UserLogin.Log.Warn(). + Err(clearErr). + Str("call_id", call.WACallID). + Stringer("room_id", call.RoomID). + Msg("Failed to clear stale MatrixRTC membership during login load") + } + if markErr := wa.Main.DB.MatrixRTCCall.MarkEnded( + ctx, + wa.UserLogin.ID, + call.WACallID, + "ended", + "bridge_restart", + lastError, + time.Now(), + ); markErr != nil { + wa.UserLogin.Log.Err(markErr). + Str("call_id", call.WACallID). + Msg("Failed to mark stale MatrixRTC call ended during login load") + } + } + if len(calls) > 0 { + wa.UserLogin.Log.Info(). + Int("call_count", len(calls)). + Msg("Cleaned up stale MatrixRTC calls during login load") + } +} + +func (wa *WhatsAppClient) handleWhatsAppWaitingRoom( + ctx context.Context, + callID string, + state meowcaller.WaitingRoomState, +) { + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil { + wa.UserLogin.Log.Err(err). + Str("call_id", callID). + Msg("Failed to load MatrixRTC call for WhatsApp waiting-room update") + return + } + if call == nil || !call.EndedTS.IsZero() || call.RoomID == "" { + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + _, err = intent.SendMessage(ctx, call.RoomID, event.EventMessage, &event.Content{ + Parsed: &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: formatWaitingRoomNotice(state), + }, + }, nil) + if err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", callID). + Msg("Failed to send WhatsApp waiting-room update to Matrix") + } +} + +func formatWaitingRoomNotice(state meowcaller.WaitingRoomState) string { + approval := "disabled" + if state.Enabled { + approval = "enabled" + } + if state.InWaitingRoom { + return fmt.Sprintf("Waiting for approval to join the WhatsApp call link (approval %s).", approval) + } + if len(state.Users) == 0 { + return fmt.Sprintf("WhatsApp call-link waiting room is empty (approval %s).", approval) + } + participants := make([]string, 0, len(state.Users)) + for _, user := range state.Users { + identity := user.JID + if !user.PN.IsEmpty() { + identity = user.PN + } + participants = append(participants, identity.String()) + } + slices.Sort(participants) + return fmt.Sprintf( + "WhatsApp call-link waiting room has %d participant(s) pending (approval %s): %s", + len(participants), + approval, + strings.Join(participants, ", "), + ) +} + +func matrixRTCFinalEndReason(call *wadb.MatrixRTCCall, reason string) (string, string) { + if call != nil && !call.EndedTS.IsZero() && (call.EndReason != "" || call.LastError != "") { + if call.EndReason != "" { + reason = call.EndReason + } + return reason, call.LastError + } + return reason, "" +} + +func (wa *WhatsAppClient) announceIncomingMatrixRTCCall(ctx context.Context, call *meowcaller.Call) error { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() + log := zerolog.Ctx(ctx).With(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Logger() + peer := wa.matrixRTCAnnouncementPeer(ctx, call.Peer()) + portalPeer := wa.incomingCallPortalPeer(call.ID(), peer) + portal, err := wa.Main.Bridge.GetPortalByKey(ctx, wa.makeWAPortalKey(portalPeer)) + if err != nil { + return err + } + if portal == nil || portal.MXID == "" { + log.Debug().Msg("No existing Matrix portal room for incoming MatrixRTC call announcement") + return nil + } + activeRoomCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveInRoom(ctx, portal.MXID) + if err != nil { + return err + } + if len(activeRoomCalls) > 0 { + log.Warn(). + Stringer("room_id", portal.MXID). + Int("active_call_count", len(activeRoomCalls)). + Msg("Rejecting incoming WhatsApp call because the Matrix room already has an active call") + return call.Reject() + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + log.Warn(). + Int("active_call_count", len(activeCalls)). + Int("max_active_calls", wa.Main.Config.VOIP.MaxActiveCallsPerLogin). + Msg("Rejecting incoming WhatsApp call because the MatrixRTC active call limit was reached") + return call.Reject() + } + } + return wa.announceMatrixRTCCallInPortal(ctx, call, portal, peer, "incoming") +} + +func (wa *WhatsAppClient) trackIncomingCallGroup(callID string, group types.JID) { + if wa == nil || callID == "" || group.Server != types.GroupServer || group.User == "" { + return + } + wa.incomingCallGroupLock.Lock() + if wa.incomingCallGroups == nil { + wa.incomingCallGroups = make(map[string]incomingCallGroup) + } + now := time.Now() + for trackedCallID, tracked := range wa.incomingCallGroups { + if !tracked.ExpiresAt.After(now) { + delete(wa.incomingCallGroups, trackedCallID) + } + } + wa.incomingCallGroups[callID] = incomingCallGroup{ + JID: group.ToNonAD(), + ExpiresAt: now.Add(callEventMaxAge), + } + wa.incomingCallGroupLock.Unlock() +} + +func (wa *WhatsAppClient) incomingCallPortalPeer(callID string, fallback types.JID) types.JID { + if wa == nil || callID == "" { + return fallback + } + wa.incomingCallGroupLock.Lock() + tracked := wa.incomingCallGroups[callID] + delete(wa.incomingCallGroups, callID) + wa.incomingCallGroupLock.Unlock() + if tracked.JID.IsEmpty() || !tracked.ExpiresAt.After(time.Now()) { + return fallback + } + return tracked.JID +} + +func (wa *WhatsAppClient) clearIncomingCallGroup(callID string) { + if wa == nil || callID == "" { + return + } + wa.incomingCallGroupLock.Lock() + delete(wa.incomingCallGroups, callID) + wa.incomingCallGroupLock.Unlock() +} + +func (wa *WhatsAppClient) joinMatrixRTCCallLink( + ctx context.Context, + portal *bridgev2.Portal, + tokenOrURL string, + video bool, +) (*meowcaller.Call, error) { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() + if portal == nil || portal.MXID == "" { + return nil, fmt.Errorf("call links must be joined from an existing portal room") + } + if video && !wa.Main.Config.VOIP.Video.Enabled { + return nil, fmt.Errorf("WhatsApp call-link video requires voip.video.enabled") + } + activeRoomCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveInRoom(ctx, portal.MXID) + if err != nil { + return nil, err + } + if len(activeRoomCalls) > 0 { + return nil, fmt.Errorf("the Matrix room already has an active call") + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return nil, err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + return nil, fmt.Errorf("active MatrixRTC call limit reached for login %s", wa.UserLogin.ID) + } + } + call, err := wa.VOIP.JoinCallLink(ctx, tokenOrURL, video) + if err != nil { + return nil, err + } + peer := wa.matrixRTCAnnouncementPeer(ctx, call.Peer()) + if err = wa.announceMatrixRTCCallInPortal(ctx, call, portal, peer, "call_link"); err != nil { + _ = call.Hangup() + return nil, err + } + return call, nil +} + +func (wa *WhatsAppClient) announceMatrixRTCCallInPortal( + ctx context.Context, + call *meowcaller.Call, + portal *bridgev2.Portal, + peer types.JID, + direction string, +) error { + if call == nil { + return fmt.Errorf("WhatsApp call is nil") + } + if portal == nil || portal.MXID == "" { + return fmt.Errorf("Matrix portal room is not available") + } + focus, err := voip.DiscoverLiveKitFocus(ctx, nil, wa.Main.Bridge.Matrix.ServerName(), wa.Main.Config.VOIP.MatrixRTC.LiveKitServiceURL) + if err != nil { + return err + } + intent, err := wa.matrixRTCParticipantIntent(ctx, peer) + if err != nil { + return err + } + + now := time.Now() + deviceID := voip.MatrixRTCDeviceID(string(wa.UserLogin.ID), call.ID()) + session := voip.MatrixRTCSession{ + UserID: intent.GetMXID(), + DeviceID: deviceID, + MemberID: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + Intent: matrixRTCCallIntent(call), + Focus: *focus, + Created: now, + Expires: matrixRTCMembershipLifetime, + StickyKey: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + } + record := &wadb.MatrixRTCCall{ + UserLoginID: wa.UserLogin.ID, + WACallID: call.ID(), + RoomID: portal.MXID, + PortalKey: portal.PortalKey, + PeerJID: peer, + Direction: direction, + MediaKind: session.Intent, + FocusType: focus.Type, + LiveKitServiceURL: focus.LiveKitServiceURL, + LiveKitRoom: portal.MXID.String(), + MatrixParticipantMXID: intent.GetMXID(), + MatrixSessionID: deviceID, + AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, + State: "ringing", + CreatedTS: now, + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + return err + } + if err = wa.sendMatrixRTCRing(ctx, intent, portal.MXID, call.ID(), &session); err != nil { + _ = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, wa.UserLogin.ID, call.ID(), "ended", "matrixrtc_announce_failed", err.Error(), time.Now()) + return err + } + record.BridgeMembershipEventID = session.MembershipEventID + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + return err + } + zerolog.Ctx(ctx).Info(). + Str("call_id", call.ID()). + Str("direction", direction). + Stringer("room_id", portal.MXID). + Stringer("participant_mxid", intent.GetMXID()). + Str("device_id", deviceID). + Msg("Announced WhatsApp call over MatrixRTC") + return nil +} + +func (wa *WhatsAppClient) matrixRTCAnnouncementPeer(ctx context.Context, peer types.JID) types.JID { + peer = peer.ToNonAD() + if peer.Server != types.HiddenUserServer { + return peer + } + pn, err := wa.GetStore().LIDs.GetPNForLID(ctx, peer) + if err != nil { + zerolog.Ctx(ctx).Err(err). + Stringer("lid", peer). + Msg("Failed to get phone number for LID incoming MatrixRTC call") + return peer + } else if pn.IsEmpty() { + return peer + } + pn = pn.ToNonAD() + zerolog.Ctx(ctx).Debug(). + Stringer("lid", peer). + Stringer("pn", pn). + Msg("Using phone number portal for incoming MatrixRTC call from LID") + return pn +} + +func (wa *WhatsAppConnector) startOutboundMatrixRTCCall(ctx context.Context, portal *bridgev2.Portal, trigger voip.MatrixRTCEvent) error { + if portal == nil { + return nil + } + if portal.Receiver == "" { + return fmt.Errorf("portal has no receiver login for outbound MatrixRTC call") + } + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, portal.Receiver) + if err != nil { + return err + } else if login == nil || login.Client == nil { + return fmt.Errorf("receiver login %s not found for outbound MatrixRTC call", portal.Receiver) + } else if !login.Client.IsLoggedIn() { + return bridgev2.ErrNotLoggedIn + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + return fmt.Errorf("receiver login %s has no WhatsApp VOIP manager", portal.Receiver) + } + return client.startOutboundMatrixRTCCall(ctx, portal, trigger) +} + +func (wa *WhatsAppClient) startOutboundMatrixRTCCall(ctx context.Context, portal *bridgev2.Portal, trigger voip.MatrixRTCEvent) error { + wa.voipCallStartLock.Lock() + defer wa.voipCallStartLock.Unlock() + if wa.VOIP == nil || !wa.VOIP.Enabled() { + return voip.ErrNotEnabled + } + peer, err := waid.ParsePortalID(portal.ID) + if err != nil { + return err + } + if !matrixRTCPortalSupportsWhatsAppCalls(peer) { + return fmt.Errorf("MatrixRTC WhatsApp calls are not supported in %s portals", peer.Server) + } + mediaKind, downgradedMedia := matrixRTCOutboundMediaKind(trigger) + if mediaKind == "" { + return fmt.Errorf("outbound WhatsApp MatrixRTC calls only support audio/video, not %q", trigger.Intent) + } + if downgradedMedia { + wa.UserLogin.Log.Warn(). + Stringer("room_id", trigger.RoomID). + Str("requested_media_kind", trigger.Intent). + Str("media_kind", mediaKind). + Msg("Downgrading outbound MatrixRTC call media kind") + } + if mediaKind == "video" && !wa.Main.Config.VOIP.Video.Enabled { + return fmt.Errorf("outbound WhatsApp MatrixRTC video calls require voip.video.enabled") + } + if wa.Main.Config.VOIP.MaxActiveCallsPerLogin > 0 { + activeCalls, err := wa.Main.DB.MatrixRTCCall.GetActiveForLogin(ctx, wa.UserLogin.ID) + if err != nil { + return err + } + if len(activeCalls) >= wa.Main.Config.VOIP.MaxActiveCallsPerLogin { + return fmt.Errorf("active MatrixRTC call limit reached for login %s", wa.UserLogin.ID) + } + } + focus, err := wa.matrixRTCLiveKitFocusForTrigger(ctx, trigger) + if err != nil { + return err + } + intent, err := wa.matrixRTCParticipantIntent(ctx, peer) + if err != nil { + return err + } + + var call *meowcaller.Call + if peer.Server == types.GroupServer { + call, err = wa.VOIP.DialGroupByID(ctx, peer.ToNonAD().String(), mediaKind == "video") + } else { + call, err = wa.VOIP.Dial(ctx, peer.ToNonAD().String(), mediaKind == "video") + } + if err != nil { + return err + } + now := time.Now() + deviceID := voip.MatrixRTCDeviceID(string(wa.UserLogin.ID), call.ID()) + record := &wadb.MatrixRTCCall{ + UserLoginID: wa.UserLogin.ID, + WACallID: call.ID(), + RoomID: trigger.RoomID, + PortalKey: portal.PortalKey, + PeerJID: peer, + Direction: "outgoing", + MediaKind: mediaKind, + FocusType: focus.Type, + LiveKitServiceURL: focus.LiveKitServiceURL, + LiveKitRoom: trigger.RoomID.String(), + MatrixParticipantMXID: intent.GetMXID(), + MatrixSessionID: deviceID, + SelectedPublisherID: matrixRTCTriggerParticipantID(trigger), + SelectedMembershipEventID: trigger.EventID, + AudioPolicy: wa.Main.Config.VOIP.LiveKit.AudioUplinkPolicy, + State: "joining_livekit", + CreatedTS: now, + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, record); err != nil { + _ = call.Hangup() + return err + } + session := &voip.MatrixRTCSession{ + UserID: intent.GetMXID(), + DeviceID: deviceID, + MemberID: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + Intent: mediaKind, + Focus: *focus, + Created: now, + Expires: matrixRTCMembershipLifetime, + StickyKey: voip.MatrixRTCMemberID(intent.GetMXID(), deviceID), + } + if err = wa.sendMatrixRTCMembership(ctx, intent, trigger.RoomID, session); err != nil { + return wa.failMatrixRTCActivation(ctx, record, "matrixrtc_membership_failed", err) + } + record.BridgeMembershipEventID = session.MembershipEventID + if err = wa.connectOutboundMatrixRTCCall(ctx, record, trigger); err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("room_id", trigger.RoomID). + Stringer("peer_jid", peer). + Msg("Failed to connect outbound MatrixRTC call to LiveKit") + return err + } + wa.UserLogin.Log.Info(). + Str("call_id", call.ID()). + Stringer("room_id", trigger.RoomID). + Stringer("peer_jid", peer). + Stringer("matrix_participant_mxid", trigger.Sender). + Msg("Started outbound WhatsApp call from MatrixRTC") + return nil +} + +func (wa *WhatsAppClient) sendMatrixRTCRing(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, waCallID string, session *voip.MatrixRTCSession) error { + now := time.Now() + notificationMode := wa.Main.Config.VOIP.MatrixRTC.NotificationEventCompat + if matrixRTCCompatAllowsModern(notificationMode) { + resp, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCNotificationEventType(), voip.BuildRTCNotificationContent(now, matrixRTCRingLifetime, session.Intent), 0) + if err != nil { + return err + } + if resp != nil { + session.NotificationEventID = resp.EventID + } + } + if matrixRTCCompatAllowsLegacy(notificationMode) { + _, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.LegacyCallNotifyEventType(), voip.BuildLegacyCallNotifyContent(waCallID, session.Intent), 0) + if err != nil { + return err + } + } + return wa.sendMatrixRTCMembership(ctx, intent, roomID, session) +} + +func (wa *WhatsAppClient) sendMatrixRTCMembership(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, session *voip.MatrixRTCSession) error { + now := time.Now() + membershipMode := wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat + modernMessageSent := false + if matrixRTCCompatAllowsModern(membershipMode) { + content := voip.BuildRTCMembershipContent(*session) + resp, err := sendMatrixRTCMessage(ctx, intent, roomID, voip.RTCMembershipEventType(event.MessageEventType), content, matrixRTCStickyDuration) + if err != nil { + return err + } + if resp != nil { + session.MembershipEventID = resp.EventID + } + modernMessageSent = true + stateKey := voip.MatrixRTCStateKey(session.UserID, session.DeviceID) + stateResp, err := intent.SendState(ctx, roomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: content}, now) + if err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", roomID). + Str("state_key", stateKey). + Msg("Failed to send MatrixRTC membership state event after sticky message membership") + } else if session.MembershipEventID == "" && stateResp != nil { + session.MembershipEventID = stateResp.EventID + } + } + if matrixRTCCompatAllowsLegacy(membershipMode) { + resp, err := intent.SendState(ctx, roomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: voip.BuildLegacyCallMemberContent(*session)}, now) + if err != nil { + if modernMessageSent { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", roomID). + Msg("Failed to send legacy MatrixRTC membership state event after modern membership") + return nil + } + return err + } + if session.MembershipEventID == "" && resp != nil { + session.MembershipEventID = resp.EventID + } + } + return nil +} + +func (wa *WhatsAppClient) clearMatrixRTCMembership(ctx context.Context, call *wadb.MatrixRTCCall) error { + if call.RoomID == "" || call.MatrixParticipantMXID == "" { + return nil + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + if intent == nil { + intent = wa.Main.Bridge.Bot + } + stickyKey := voip.MatrixRTCMemberID(call.MatrixParticipantMXID, call.MatrixSessionID) + emptyContent := voip.EmptyMatrixRTCContent(stickyKey) + now := time.Now() + membershipMode := wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat + modernMessageSent := false + if matrixRTCCompatAllowsModern(membershipMode) { + if _, err := sendMatrixRTCMessage(ctx, intent, call.RoomID, voip.RTCMembershipEventType(event.MessageEventType), emptyContent, matrixRTCStickyDuration); err != nil { + return err + } + modernMessageSent = true + stateKey := voip.MatrixRTCStateKey(call.MatrixParticipantMXID, call.MatrixSessionID) + if _, err := intent.SendState(ctx, call.RoomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", call.RoomID). + Str("state_key", stateKey). + Msg("Failed to clear MatrixRTC membership state event after sticky message cleanup") + } + } + if matrixRTCCompatAllowsLegacy(membershipMode) { + if _, err := intent.SendState(ctx, call.RoomID, voip.GroupCallMemberEventType(), "", &event.Content{Raw: map[string]any{}}, now); err != nil { + if modernMessageSent { + wa.UserLogin.Log.Warn(). + Err(err). + Stringer("room_id", call.RoomID). + Msg("Failed to clear legacy MatrixRTC membership state event after modern cleanup") + return nil + } + return err + } + } + return nil +} + +func (wa *WhatsAppConnector) cleanupFailedOutboundMatrixRTCStart(ctx context.Context, trigger voip.MatrixRTCEvent) error { + if trigger.RoomID == "" || wa.Bridge == nil || wa.Bridge.Bot == nil { + return nil + } + intent := wa.Bridge.Bot + now := time.Now() + membershipMode := wa.Config.VOIP.MatrixRTC.MembershipEventCompat + + if trigger.Kind == voip.MatrixRTCEventKindRTCMembership && matrixRTCCompatAllowsModern(membershipMode) { + emptyContent := voip.EmptyMatrixRTCContent(matrixRTCTriggerStickyKey(trigger)) + if _, err := sendMatrixRTCMessage(ctx, intent, trigger.RoomID, voip.RTCMembershipEventType(event.MessageEventType), emptyContent, matrixRTCStickyDuration); err != nil { + return err + } + stateKey := matrixRTCTriggerStateKey(trigger) + if stateKey != "" { + if _, err := intent.SendState(ctx, trigger.RoomID, voip.RTCMembershipEventType(event.StateEventType), stateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + return err + } + } + } + + if trigger.Kind == voip.MatrixRTCEventKindGroupCallMember && matrixRTCCompatAllowsLegacy(membershipMode) { + if _, err := intent.SendState(ctx, trigger.RoomID, voip.GroupCallMemberEventType(), trigger.StateKey, &event.Content{Raw: map[string]any{}}, now); err != nil { + return err + } + } + return nil +} + +func (wa *WhatsAppClient) activateMatrixRTCCall(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) error { + if call == nil { + return nil + } + call.State = "joining_livekit" + call.LastError = "" + call.SelectedPublisherID = matrixRTCTriggerParticipantID(trigger) + call.SelectedMembershipEventID = trigger.EventID + if err := wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + return err + } + authResp, err := wa.requestMatrixRTCLiveKitAuth(ctx, call, trigger) + if err != nil { + return err + } + if err = wa.VOIP.BridgeCallToLiveKit(ctx, call.WACallID, authResp, call.SelectedPublisherID); err != nil { + return wa.failMatrixRTCActivation(ctx, call, "livekit_bridge_failed", err) + } + now := time.Now() + call.State = "active" + call.JoinedTS = now + call.AnsweredTS = now + if authResp.RoomName != "" { + call.LiveKitRoom = authResp.RoomName + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + return err + } + wa.UserLogin.Log.Info(). + Str("call_id", call.WACallID). + Stringer("room_id", call.RoomID). + Stringer("trigger_sender", trigger.Sender). + Msg("Activated MatrixRTC LiveKit bridge for WhatsApp call") + return nil +} + +func (wa *WhatsAppClient) connectOutboundMatrixRTCCall(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) error { + authResp, err := wa.requestMatrixRTCLiveKitAuth(ctx, call, trigger) + if err != nil { + return err + } + if err = wa.VOIP.BridgeCallToLiveKit(ctx, call.WACallID, authResp, call.SelectedPublisherID); err != nil { + return wa.failMatrixRTCActivation(ctx, call, "livekit_bridge_failed", err) + } + now := time.Now() + call.State = "active" + call.JoinedTS = now + if authResp.RoomName != "" { + call.LiveKitRoom = authResp.RoomName + } + return wa.Main.DB.MatrixRTCCall.Put(ctx, call) +} + +func (wa *WhatsAppClient) requestMatrixRTCLiveKitAuth(ctx context.Context, call *wadb.MatrixRTCCall, trigger voip.MatrixRTCEvent) (*voip.LiveKitAuthResponse, error) { + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + openIDToken, err := requestMatrixOpenIDToken(ctx, intent) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "matrix_openid_failed", err) + } + if matrixRTCCompatAllowsLegacy(wa.Main.Config.VOIP.MatrixRTC.MembershipEventCompat) { + authResp, err := voip.RequestLegacyLiveKitAuth(ctx, nil, call.LiveKitServiceURL, matrixRTCLegacyLiveKitAuthRequest(call, openIDToken)) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "livekit_auth_failed", err) + } + return authResp, nil + } + authResp, err := voip.RequestLiveKitAuth(ctx, nil, call.LiveKitServiceURL, matrixRTCLiveKitAuthRequest(call, openIDToken)) + if err != nil { + return nil, wa.failMatrixRTCActivation(ctx, call, "livekit_auth_failed", err) + } + return authResp, nil +} + +func (wa *WhatsAppClient) failMatrixRTCActivation(ctx context.Context, call *wadb.MatrixRTCCall, reason string, err error) error { + _ = wa.Main.DB.MatrixRTCCall.MarkEnded(ctx, call.UserLoginID, call.WACallID, "ended", reason, err.Error(), time.Now()) + if wa.VOIP != nil { + wa.VOIP.HandleMatrixRTCCallEvent(ctx, voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindRTCDecline, + RoomID: call.RoomID, + }, call.WACallID) + } + return err +} + +func matrixRTCLiveKitAuthRequest(call *wadb.MatrixRTCCall, openIDToken voip.MatrixOpenIDToken) voip.LiveKitAuthRequest { + if call == nil { + return voip.LiveKitAuthRequest{OpenIDToken: openIDToken} + } + return voip.LiveKitAuthRequest{ + RoomID: call.RoomID.String(), + SlotID: voip.MatrixRTCDefaultSlotID, + OpenIDToken: openIDToken, + Member: matrixRTCLiveKitAuthMember(call), + } +} + +func matrixRTCLegacyLiveKitAuthRequest(call *wadb.MatrixRTCCall, openIDToken voip.MatrixOpenIDToken) voip.LegacyLiveKitAuthRequest { + if call == nil { + return voip.LegacyLiveKitAuthRequest{OpenIDToken: openIDToken} + } + return voip.LegacyLiveKitAuthRequest{ + Room: call.RoomID.String(), + OpenIDToken: openIDToken, + DeviceID: call.MatrixSessionID, + } +} + +func matrixRTCLiveKitAuthMember(call *wadb.MatrixRTCCall) *voip.LiveKitAuthMember { + if call == nil { + return nil + } + return &voip.LiveKitAuthMember{ + ID: voip.MatrixRTCMemberID(call.MatrixParticipantMXID, call.MatrixSessionID), + ClaimedDeviceID: call.MatrixSessionID, + ClaimedUserID: call.MatrixParticipantMXID.String(), + } +} + +func (wa *WhatsAppClient) matrixRTCLiveKitFocusForTrigger(ctx context.Context, trigger voip.MatrixRTCEvent) (*voip.Focus, error) { + for _, focus := range trigger.FociPreferred { + if focus.Type == "livekit" && focus.LiveKitServiceURL != "" { + focusCopy := focus + return &focusCopy, nil + } + } + return voip.DiscoverLiveKitFocus(ctx, nil, wa.Main.Bridge.Matrix.ServerName(), wa.Main.Config.VOIP.MatrixRTC.LiveKitServiceURL) +} + +func (wa *WhatsAppClient) matrixRTCParticipantIntent(ctx context.Context, peer types.JID) (bridgev2.MatrixAPI, error) { + mode := strings.ToLower(wa.Main.Config.VOIP.MatrixRTC.ParticipantMode) + if mode == "" || mode == "whatsapp_ghost" { + if ghostID := waid.MakeUserID(peer); ghostID != "" { + ghost, err := wa.Main.Bridge.GetGhostByID(ctx, ghostID) + if err != nil { + return nil, err + } + if ghost != nil && ghost.Intent != nil { + return ghost.Intent, nil + } + } + } + return wa.Main.Bridge.Bot, nil +} + +func (wa *WhatsAppClient) matrixRTCIntentForMXID(ctx context.Context, mxid id.UserID) bridgev2.MatrixAPI { + if mxid == "" || mxid == wa.Main.Bridge.Bot.GetMXID() { + return wa.Main.Bridge.Bot + } + if ghost, err := wa.Main.Bridge.GetGhostByMXID(ctx, mxid); err == nil && ghost != nil && ghost.Intent != nil { + return ghost.Intent + } + return wa.Main.Bridge.Bot +} + +func matrixRTCPortalSupportsWhatsAppCalls(peer types.JID) bool { + switch peer.Server { + case types.DefaultUserServer, types.HiddenUserServer, types.GroupServer: + return true + default: + return false + } +} + +func matrixRTCCallIntent(call *meowcaller.Call) string { + if call != nil && call.IsVideo() { + return "video" + } + return "audio" +} + +func matrixRTCOutboundMediaKind(trigger voip.MatrixRTCEvent) (mediaKind string, downgraded bool) { + switch trigger.Intent { + case "", "audio": + return "audio", false + case "video": + return "video", false + default: + return "", false + } +} + +func matrixRTCTriggerParticipantID(trigger voip.MatrixRTCEvent) string { + if trigger.Sender == "" { + return "" + } + deviceID := trigger.SessionID + if deviceID == "" { + deviceID = trigger.DeviceID + } + return voip.MatrixRTCMemberID(trigger.Sender, deviceID) +} + +func matrixRTCTriggerStateKey(trigger voip.MatrixRTCEvent) string { + if trigger.StateKey != "" { + return trigger.StateKey + } + deviceID := trigger.SessionID + if deviceID == "" { + deviceID = trigger.DeviceID + } + if trigger.Sender == "" { + return deviceID + } + return voip.MatrixRTCStateKey(trigger.Sender, deviceID) +} + +func matrixRTCTriggerStickyKey(trigger voip.MatrixRTCEvent) string { + if stickyKey, ok := trigger.Raw["sticky_key"].(string); ok && stickyKey != "" { + return stickyKey + } + if stickyKey, ok := trigger.Raw["msc4354_sticky_key"].(string); ok && stickyKey != "" { + return stickyKey + } + return matrixRTCTriggerParticipantID(trigger) +} + +func sendMatrixRTCMessage(ctx context.Context, intent bridgev2.MatrixAPI, roomID id.RoomID, eventType event.Type, raw map[string]any, sticky time.Duration) (*mautrix.RespSendEvent, error) { + if asIntent, ok := intent.(*mxbridge.ASIntent); ok { + return asIntent.Matrix.SendMessageEvent(ctx, roomID, eventType, &event.Content{Raw: raw}, mautrix.ReqSendEvent{ + UnstableStickyDuration: sticky, + DontEncrypt: true, + }) + } + return intent.SendMessage(ctx, roomID, eventType, &event.Content{Raw: raw}, nil) +} + +func requestMatrixOpenIDToken(ctx context.Context, intent bridgev2.MatrixAPI) (voip.MatrixOpenIDToken, error) { + asIntent, ok := intent.(*mxbridge.ASIntent) + if !ok { + return voip.MatrixOpenIDToken{}, fmt.Errorf("matrix intent %T does not support OpenID token requests", intent) + } + if asIntent.Matrix == nil || asIntent.Matrix.Client == nil { + return voip.MatrixOpenIDToken{}, fmt.Errorf("matrix intent %T has no Matrix client", intent) + } + resp, err := asIntent.Matrix.Client.RequestOpenIDToken(ctx) + if err != nil { + return voip.MatrixOpenIDToken{}, err + } + return voip.MatrixOpenIDToken{ + AccessToken: resp.AccessToken, + TokenType: resp.TokenType, + MatrixServerName: resp.MatrixServerName, + ExpiresIn: resp.ExpiresIn, + }, nil +} + +func matrixRTCCompatAllowsModern(mode string) bool { + switch strings.ToLower(mode) { + case "legacy", "legacy_only", "msc3401", "org.matrix.msc3401.call.member": + return false + default: + return true + } +} + +func matrixRTCCompatAllowsLegacy(mode string) bool { + switch strings.ToLower(mode) { + case "modern", "modern_only", "msc4143", "org.matrix.msc4143.rtc.member", "none", "off", "false", "disabled": + return false + default: + return true + } +} diff --git a/pkg/connector/matrixrtc_reactions.go b/pkg/connector/matrixrtc_reactions.go new file mode 100644 index 0000000..973ab53 --- /dev/null +++ b/pkg/connector/matrixrtc_reactions.go @@ -0,0 +1,254 @@ +package connector + +import ( + "context" + "strings" + + "github.com/purpshell/meowcaller" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/event" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func isMatrixRTCCallControlEvent(evt voip.MatrixRTCEvent) bool { + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction, voip.MatrixRTCEventKindHandRaise, voip.MatrixRTCEventKindRedaction: + return true + default: + return false + } +} + +func matrixRTCControlEventMatchesCall(evt voip.MatrixRTCEvent, call *wadb.MatrixRTCCall) bool { + if call == nil || !matrixRTCEventSenderMatchesPublisher(evt.Sender.String(), call.SelectedPublisherID) { + return false + } + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction, voip.MatrixRTCEventKindHandRaise: + return evt.RelatesToEventID != "" && evt.RelatesToEventID == call.SelectedMembershipEventID + case voip.MatrixRTCEventKindRedaction: + return evt.Redacts != "" && evt.Redacts == call.SelectedHandRaiseEventID + default: + return false + } +} + +func matrixRTCEventSenderMatchesPublisher(sender, publisherID string) bool { + if sender == "" || publisherID == "" { + return false + } + return publisherID == sender || strings.HasPrefix(publisherID, sender+":") +} + +func (wa *WhatsAppConnector) handleMatrixRTCCallControlEvent( + ctx context.Context, + evt voip.MatrixRTCEvent, + activeCalls []*wadb.MatrixRTCCall, + log zerolog.Logger, +) { + for _, activeCall := range activeCalls { + if !matrixRTCControlEventMatchesCall(evt, activeCall) { + continue + } + login, err := wa.Bridge.GetExistingUserLoginByID(ctx, activeCall.UserLoginID) + if err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to resolve login for MatrixRTC call control") + continue + } + if login == nil { + continue + } + client, ok := login.Client.(*WhatsAppClient) + if !ok || client == nil || client.VOIP == nil { + continue + } + + switch evt.Kind { + case voip.MatrixRTCEventKindCallReaction: + if evt.RelationType != event.RelReference { + continue + } + emoji, supported := voip.NormalizeWhatsAppCallReaction(evt.ReactionEmoji) + if !supported { + log.Debug().Str("emoji", evt.ReactionEmoji).Msg("Ignoring unsupported MatrixRTC call reaction") + continue + } + if err = client.VOIP.SendReaction(activeCall.WACallID, emoji); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Str("emoji", emoji).Msg("Failed to send MatrixRTC reaction to WhatsApp") + } + case voip.MatrixRTCEventKindHandRaise: + if evt.RelationType != event.RelAnnotation || evt.RelationKey != "🖐️" || evt.EventID == "" { + continue + } + if activeCall.SelectedHandRaiseEventID != "" { + continue + } + if err = client.VOIP.SetHandRaised(activeCall.WACallID, true); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to raise hand in WhatsApp call") + continue + } + activeCall.SelectedHandRaiseEventID = evt.EventID + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to persist MatrixRTC hand raise") + _ = client.VOIP.SetHandRaised(activeCall.WACallID, false) + } + case voip.MatrixRTCEventKindRedaction: + if err = client.VOIP.SetHandRaised(activeCall.WACallID, false); err != nil { + log.Warn().Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to lower hand in WhatsApp call") + continue + } + activeCall.SelectedHandRaiseEventID = "" + if err = wa.DB.MatrixRTCCall.Put(ctx, activeCall); err != nil { + log.Err(err).Str("wa_call_id", activeCall.WACallID).Msg("Failed to persist MatrixRTC hand lowering") + _ = client.VOIP.SetHandRaised(activeCall.WACallID, true) + } + } + } +} + +func (wa *WhatsAppClient) handleWhatsAppCallReaction(ctx context.Context, callID string, reaction meowcaller.CallReaction) { + if reaction.Removed || wa.isOwnWhatsAppCallParticipant(reaction.Sender) { + return + } + emoji, supported := voip.NormalizeWhatsAppCallReaction(reaction.Emoji) + if !supported { + return + } + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil || call == nil || !call.EndedTS.IsZero() || call.BridgeMembershipEventID == "" { + if err != nil { + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to load MatrixRTC call for WhatsApp reaction") + } + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + _, err = intent.SendMessage(ctx, call.RoomID, voip.ElementCallReactionEventType(), &event.Content{ + Raw: voip.BuildElementCallReactionContent(call.BridgeMembershipEventID, emoji), + }, nil) + if err != nil { + wa.UserLogin.Log.Warn().Err(err).Str("call_id", callID).Str("emoji", emoji).Msg("Failed to bridge WhatsApp call reaction to MatrixRTC") + } +} + +func (wa *WhatsAppClient) handleWhatsAppHandRaise(ctx context.Context, callID string, state meowcaller.HandRaiseState) { + if wa.isOwnWhatsAppCallParticipant(state.Participant) { + return + } + wa.voipHandBridgeLock.Lock() + defer wa.voipHandBridgeLock.Unlock() + raised, changed := wa.updateWhatsAppRemoteHandRaise(callID, state) + if !changed { + return + } + rollback := func() { + state.Raised = !state.Raised + wa.updateWhatsAppRemoteHandRaise(callID, state) + } + call, err := wa.Main.DB.MatrixRTCCall.Get(ctx, wa.UserLogin.ID, callID) + if err != nil || call == nil || !call.EndedTS.IsZero() || call.BridgeMembershipEventID == "" { + rollback() + if err != nil { + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to load MatrixRTC call for WhatsApp hand state") + } + return + } + intent := wa.matrixRTCIntentForMXID(ctx, call.MatrixParticipantMXID) + if raised { + if call.BridgeHandRaiseEventID != "" { + return + } + resp, sendErr := intent.SendMessage(ctx, call.RoomID, event.EventReaction, &event.Content{ + Raw: voip.BuildElementCallHandRaiseContent(call.BridgeMembershipEventID), + }, nil) + if sendErr != nil { + rollback() + wa.UserLogin.Log.Warn().Err(sendErr).Str("call_id", callID).Msg("Failed to bridge WhatsApp hand raise to MatrixRTC") + return + } + if resp != nil { + call.BridgeHandRaiseEventID = resp.EventID + } + if call.BridgeHandRaiseEventID == "" { + rollback() + return + } + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + rollback() + _, _ = intent.SendMessage(ctx, call.RoomID, event.EventRedaction, &event.Content{ + Parsed: &event.RedactionEventContent{Redacts: call.BridgeHandRaiseEventID}, + }, nil) + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to persist bridged WhatsApp hand raise") + } + return + } else { + if call.BridgeHandRaiseEventID == "" { + return + } + handRaiseEventID := call.BridgeHandRaiseEventID + call.BridgeHandRaiseEventID = "" + if err = wa.Main.DB.MatrixRTCCall.Put(ctx, call); err != nil { + rollback() + wa.UserLogin.Log.Err(err).Str("call_id", callID).Msg("Failed to persist bridged WhatsApp hand lowering") + return + } + _, sendErr := intent.SendMessage(ctx, call.RoomID, event.EventRedaction, &event.Content{ + Parsed: &event.RedactionEventContent{Redacts: handRaiseEventID}, + }, nil) + if sendErr != nil { + rollback() + call.BridgeHandRaiseEventID = handRaiseEventID + _ = wa.Main.DB.MatrixRTCCall.Put(ctx, call) + wa.UserLogin.Log.Warn().Err(sendErr).Str("call_id", callID).Msg("Failed to bridge WhatsApp hand lowering to MatrixRTC") + return + } + } +} + +func (wa *WhatsAppClient) updateWhatsAppRemoteHandRaise(callID string, state meowcaller.HandRaiseState) (raised, changed bool) { + if callID == "" || state.Participant.IsEmpty() { + return false, false + } + participant := state.Participant.ToNonAD() + wa.voipHandRaiseLock.Lock() + defer wa.voipHandRaiseLock.Unlock() + if wa.voipHandRaises == nil { + wa.voipHandRaises = make(map[string]map[types.JID]bool) + } + hands := wa.voipHandRaises[callID] + if hands == nil { + hands = make(map[types.JID]bool) + wa.voipHandRaises[callID] = hands + } + wasRaised := len(hands) > 0 + if state.Raised { + hands[participant] = true + } else { + delete(hands, participant) + } + raised = len(hands) > 0 + if !raised { + delete(wa.voipHandRaises, callID) + } + return raised, wasRaised != raised +} + +func (wa *WhatsAppClient) clearWhatsAppRemoteHandRaises(callID string) { + wa.voipHandBridgeLock.Lock() + defer wa.voipHandBridgeLock.Unlock() + wa.voipHandRaiseLock.Lock() + delete(wa.voipHandRaises, callID) + wa.voipHandRaiseLock.Unlock() +} + +func (wa *WhatsAppClient) isOwnWhatsAppCallParticipant(participant types.JID) bool { + if participant.IsEmpty() || wa == nil || wa.Client == nil || wa.Client.Store == nil { + return false + } + device := wa.Client.Store + participant = participant.ToNonAD() + return participant == device.GetLID().ToNonAD() || + participant == device.GetJID().ToNonAD() +} diff --git a/pkg/connector/matrixrtc_test.go b/pkg/connector/matrixrtc_test.go new file mode 100644 index 0000000..bb49ba6 --- /dev/null +++ b/pkg/connector/matrixrtc_test.go @@ -0,0 +1,485 @@ +package connector + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/store" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + "go.mau.fi/mautrix-whatsapp/pkg/connector/wadb" +) + +func TestShouldStartOutboundMatrixRTCCall(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if !shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned false for active state membership") + } +} + +func TestMatrixRTCSenderMustOwnPortalLogin(t *testing.T) { + login := &bridgev2.UserLogin{ + User: &bridgev2.User{ + User: &database.User{MXID: "@alice:example.com"}, + }, + } + if !matrixRTCSenderOwnsLogin("@alice:example.com", login) { + t.Fatal("matrixRTCSenderOwnsLogin rejected the login owner") + } + if matrixRTCSenderOwnsLogin("@mallory:example.com", login) { + t.Fatal("matrixRTCSenderOwnsLogin accepted another Matrix user") + } + if matrixRTCSenderOwnsLogin("@alice:example.com", nil) { + t.Fatal("matrixRTCSenderOwnsLogin accepted a missing login") + } +} + +func TestOwnWhatsAppCallParticipantMatchesPhoneAndLID(t *testing.T) { + phone := types.NewJID("15550000001", types.DefaultUserServer) + lid := types.NewJID("111", types.HiddenUserServer) + client := &WhatsAppClient{ + Client: &whatsmeow.Client{ + Store: &store.Device{ID: &phone, LID: lid}, + }, + } + if !client.isOwnWhatsAppCallParticipant(phone) { + t.Fatal("phone JID was not recognized as the local WhatsApp call participant") + } + if !client.isOwnWhatsAppCallParticipant(lid) { + t.Fatal("LID was not recognized as the local WhatsApp call participant") + } + if client.isOwnWhatsAppCallParticipant(types.NewJID("222", types.HiddenUserServer)) { + t.Fatal("remote LID was recognized as the local WhatsApp call participant") + } +} + +func TestIncomingGroupCallUsesGroupPortalAndConsumesCorrelation(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + group := types.NewJID("120363000000000000", types.GroupServer) + fallback := types.NewJID("15550000001", types.DefaultUserServer) + + client.trackIncomingCallGroup("CALL", group) + if got := client.incomingCallPortalPeer("CALL", fallback); got != group { + t.Fatalf("incoming portal peer = %s, want group %s", got, group) + } + if got := client.incomingCallPortalPeer("CALL", fallback); got != fallback { + t.Fatalf("consumed incoming portal peer = %s, want fallback %s", got, fallback) + } +} + +func TestIncomingCallEventTracksOfferGroupBeforeManagedCall(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + group := types.NewJID("120363000000000000", types.GroupServer) + client.trackIncomingCallEvent(&events.CallOffer{ + BasicCallMeta: types.BasicCallMeta{ + CallID: "CALL", + GroupJID: group, + }, + }) + if got := client.incomingCallPortalPeer("CALL", types.EmptyJID); got != group { + t.Fatalf("incoming portal peer = %s, want group %s", got, group) + } +} + +func TestIncomingCallGroupIgnoresNonGroupJID(t *testing.T) { + client := &WhatsAppClient{incomingCallGroups: make(map[string]incomingCallGroup)} + direct := types.NewJID("15550000001", types.DefaultUserServer) + client.trackIncomingCallGroup("CALL", direct) + if got := client.incomingCallPortalPeer("CALL", direct); got != direct { + t.Fatalf("incoming portal peer = %s, want direct fallback %s", got, direct) + } + if len(client.incomingCallGroups) != 0 { + t.Fatalf("tracked non-group calls = %d, want 0", len(client.incomingCallGroups)) + } +} + +func TestIncomingCallGroupIgnoresExpiredCorrelation(t *testing.T) { + group := types.NewJID("120363000000000000", types.GroupServer) + fallback := types.NewJID("15550000001", types.DefaultUserServer) + client := &WhatsAppClient{incomingCallGroups: map[string]incomingCallGroup{ + "CALL": { + JID: group, + ExpiresAt: time.Now().Add(-time.Second), + }, + }} + if got := client.incomingCallPortalPeer("CALL", fallback); got != fallback { + t.Fatalf("expired incoming portal peer = %s, want fallback %s", got, fallback) + } +} + +func TestFormatWaitingRoomNotice(t *testing.T) { + waiting := formatWaitingRoomNotice(meowcaller.WaitingRoomState{ + Enabled: true, + InWaitingRoom: true, + }) + if !strings.Contains(waiting, "Waiting for approval") { + t.Fatalf("waiting-room self notice = %q", waiting) + } + + participants := formatWaitingRoomNotice(meowcaller.WaitingRoomState{ + Enabled: true, + Users: []meowcaller.WaitingRoomUser{ + { + JID: types.NewJID("222", types.HiddenUserServer), + PN: types.NewJID("15550000002", types.DefaultUserServer), + }, + {JID: types.NewJID("111", types.HiddenUserServer)}, + }, + }) + if !strings.Contains(participants, "2 participant(s)") || + !strings.Contains(participants, "111@lid, 15550000002@s.whatsapp.net") { + t.Fatalf("waiting-room participant notice = %q", participants) + } +} + +func TestShouldStartOutboundMatrixRTCCallRejectsMessageMembership(t *testing.T) { + evt := matrixRTCMemberEvent(event.MessageEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned true for message membership") + } +} + +func TestShouldStartOutboundMatrixRTCCallRejectsActivePreviousState(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + evt.Unsigned.PrevContent = &event.Content{Raw: matrixRTCMemberContent()} + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldStartOutboundMatrixRTCCall(evt, parsed, "auto") { + t.Fatalf("shouldStartOutboundMatrixRTCCall returned true for an active-to-active state update") + } +} + +func TestMatrixRTCTriggerStateKeyUsesEventStateKey(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if stateKey := matrixRTCTriggerStateKey(parsed); stateKey != "@alice:example.com_DEVICE" { + t.Fatalf("state key = %q, want event state key", stateKey) + } +} + +func TestMatrixRTCTriggerStateKeyFallsBackToSenderAndSession(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + SessionID: "SESSION", + } + if stateKey := matrixRTCTriggerStateKey(parsed); stateKey != "@alice:example.com_SESSION" { + t.Fatalf("state key = %q, want sender/session-derived key", stateKey) + } +} + +func TestMatrixRTCTriggerStickyKeyPrefersContentStickyKey(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + SessionID: "SESSION", + Raw: map[string]any{ + "sticky_key": "sticky", + }, + } + if stickyKey := matrixRTCTriggerStickyKey(parsed); stickyKey != "sticky" { + t.Fatalf("sticky key = %q, want content sticky key", stickyKey) + } +} + +func TestMatrixRTCTriggerStickyKeyFallsBackToParticipantID(t *testing.T) { + parsed := voip.MatrixRTCEvent{ + Sender: "@alice:example.com", + DeviceID: "DEVICE", + Raw: map[string]any{}, + } + if stickyKey := matrixRTCTriggerStickyKey(parsed); stickyKey != "@alice:example.com:DEVICE" { + t.Fatalf("sticky key = %q, want participant id", stickyKey) + } +} + +func TestMatrixRTCOutboundMediaKindDefaultsToAudio(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{}) + if mediaKind != "audio" { + t.Fatalf("mediaKind = %q, want audio", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindKeepsAudio(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "audio"}) + if mediaKind != "audio" { + t.Fatalf("mediaKind = %q, want audio", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindKeepsVideo(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "video"}) + if mediaKind != "video" { + t.Fatalf("mediaKind = %q, want video", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestMatrixRTCOutboundMediaKindRejectsUnknown(t *testing.T) { + mediaKind, downgraded := matrixRTCOutboundMediaKind(voip.MatrixRTCEvent{Intent: "screen"}) + if mediaKind != "" { + t.Fatalf("mediaKind = %q, want empty", mediaKind) + } + if downgraded { + t.Fatalf("downgraded = true, want false") + } +} + +func TestShouldEndMatrixRTCCallFromLegacyMembershipLeave(t *testing.T) { + stateKey := "_@alice:example.com_DEVICE_m.call" + evt := &event.Event{ + Type: voip.GroupCallMemberEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{}}, + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if !shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned false for selected participant leave") + } +} + +func TestShouldEndMatrixRTCCallFromMembershipKeepsActiveJoin(t *testing.T) { + evt := matrixRTCMemberEvent(event.StateEventType) + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned true for active join") + } +} + +func TestShouldEndMatrixRTCCallFromMembershipRejectsOtherParticipant(t *testing.T) { + stateKey := "_@alice:example.com_OTHER_m.call" + evt := &event.Event{ + Type: voip.GroupCallMemberEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{}}, + } + parsed, ok := voip.ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership") + } + if shouldEndMatrixRTCCallFromMembership(parsed, "@alice:example.com:DEVICE") { + t.Fatalf("shouldEndMatrixRTCCallFromMembership returned true for another participant") + } +} + +func TestMatrixRTCPortalSupportsWhatsAppGroupCalls(t *testing.T) { + peer := types.NewJID("120363000000000000", types.GroupServer) + if !matrixRTCPortalSupportsWhatsAppCalls(peer) { + t.Fatal("matrixRTCPortalSupportsWhatsAppCalls rejected a WhatsApp group portal") + } +} + +func TestMatrixRTCControlEventMatchesSelectedMembership(t *testing.T) { + call := &wadb.MatrixRTCCall{ + SelectedPublisherID: "@alice:example.com:DEVICE", + SelectedMembershipEventID: "$membership", + } + reaction := voip.MatrixRTCEvent{ + Kind: voip.MatrixRTCEventKindCallReaction, + Sender: "@alice:example.com", + RelatesToEventID: "$membership", + } + if !matrixRTCControlEventMatchesCall(reaction, call) { + t.Fatal("reaction did not match its selected MatrixRTC membership") + } + reaction.Sender = "@mallory:example.com" + if matrixRTCControlEventMatchesCall(reaction, call) { + t.Fatal("reaction from another Matrix user matched the selected membership") + } +} + +func TestWhatsAppRemoteHandRaisesAreAggregated(t *testing.T) { + wa := &WhatsAppClient{} + alice := types.NewJID("111", types.DefaultUserServer) + bob := types.NewJID("222", types.DefaultUserServer) + + raised, changed := wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: alice, Raised: true}) + if !raised || !changed { + t.Fatalf("first raise = %t, %t, want true, true", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: bob, Raised: true}) + if !raised || changed { + t.Fatalf("second raise = %t, %t, want true, false", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: alice, Raised: false}) + if !raised || changed { + t.Fatalf("first lower = %t, %t, want true, false", raised, changed) + } + raised, changed = wa.updateWhatsAppRemoteHandRaise("call", meowcaller.HandRaiseState{Participant: bob, Raised: false}) + if raised || !changed { + t.Fatalf("last lower = %t, %t, want false, true", raised, changed) + } +} + +func TestReserveMatrixRTCOutboundStartSuppressesDuplicates(t *testing.T) { + wa := &WhatsAppConnector{} + if !wa.reserveMatrixRTCOutboundStart("!room:example.com") { + t.Fatalf("first reserve returned false") + } + if wa.reserveMatrixRTCOutboundStart("!room:example.com") { + t.Fatalf("second reserve returned true, want false") + } + if !wa.reserveMatrixRTCOutboundStart("!other:example.com") { + t.Fatalf("reserve for another room returned false") + } +} + +func TestMatrixRTCFinalEndReasonPreservesActivationFailure(t *testing.T) { + reason, lastErr := matrixRTCFinalEndReason(&wadb.MatrixRTCCall{ + EndedTS: time.Unix(123, 0), + EndReason: "livekit_bridge_failed", + LastError: "could not connect after timeout", + }, "rejected") + if reason != "livekit_bridge_failed" { + t.Fatalf("reason = %q, want livekit_bridge_failed", reason) + } + if lastErr != "could not connect after timeout" { + t.Fatalf("lastErr = %q, want timeout error", lastErr) + } +} + +func TestMatrixRTCFinalEndReasonUsesWhatsAppReasonForFreshEnd(t *testing.T) { + reason, lastErr := matrixRTCFinalEndReason(&wadb.MatrixRTCCall{}, "rejected") + if reason != "rejected" { + t.Fatalf("reason = %q, want rejected", reason) + } + if lastErr != "" { + t.Fatalf("lastErr = %q, want empty", lastErr) + } +} + +func TestMatrixRTCLiveKitAuthRequestUsesStrictJWTFields(t *testing.T) { + req := matrixRTCLiveKitAuthRequest(&wadb.MatrixRTCCall{ + RoomID: "!room:example.com", + MatrixParticipantMXID: "@whatsapp_123:example.com", + MatrixSessionID: "WA123", + }, voip.MatrixOpenIDToken{AccessToken: "openid"}) + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + for _, field := range []string{ + `"device_id"`, + `"session_id"`, + `"participant_id"`, + `"focus_type"`, + `"extra"`, + } { + if bytes.Contains(body, []byte(field)) { + t.Fatalf("request body contains strict-JWT-incompatible field %s: %s", field, string(body)) + } + } + if req.RoomID != "!room:example.com" || req.SlotID != voip.MatrixRTCDefaultSlotID { + t.Fatalf("unexpected room or slot in request: %+v", req) + } + if req.Member == nil || req.Member.ID != "@whatsapp_123:example.com:WA123" { + t.Fatalf("unexpected member in request: %+v", req.Member) + } +} + +func TestMatrixRTCLegacyLiveKitAuthRequestUsesRoomAndDevice(t *testing.T) { + req := matrixRTCLegacyLiveKitAuthRequest(&wadb.MatrixRTCCall{ + RoomID: "!room:example.com", + MatrixSessionID: "WA123", + }, voip.MatrixOpenIDToken{AccessToken: "openid"}) + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + if !bytes.Contains(body, []byte(`"room":"!room:example.com"`)) { + t.Fatalf("request body missing legacy room: %s", string(body)) + } + if !bytes.Contains(body, []byte(`"device_id":"WA123"`)) { + t.Fatalf("request body missing legacy device_id: %s", string(body)) + } + if bytes.Contains(body, []byte(`"member"`)) || bytes.Contains(body, []byte(`"slot_id"`)) { + t.Fatalf("legacy request body contains modern fields: %s", string(body)) + } +} + +func TestMatrixRTCCompatModeConfigValues(t *testing.T) { + tests := []struct { + mode string + wantModern bool + wantLegacy bool + }{ + {mode: "auto", wantModern: true, wantLegacy: true}, + {mode: "msc4143", wantModern: true, wantLegacy: false}, + {mode: "msc3401", wantModern: false, wantLegacy: true}, + } + for _, tt := range tests { + t.Run(tt.mode, func(t *testing.T) { + if got := matrixRTCCompatAllowsModern(tt.mode); got != tt.wantModern { + t.Fatalf("matrixRTCCompatAllowsModern(%q) = %v, want %v", tt.mode, got, tt.wantModern) + } + if got := matrixRTCCompatAllowsLegacy(tt.mode); got != tt.wantLegacy { + t.Fatalf("matrixRTCCompatAllowsLegacy(%q) = %v, want %v", tt.mode, got, tt.wantLegacy) + } + }) + } +} + +func matrixRTCMemberEvent(class event.TypeClass) *event.Event { + stateKey := "@alice:example.com_DEVICE" + return &event.Event{ + Type: voip.RTCMembershipEventType(class), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: matrixRTCMemberContent()}, + } +} + +func matrixRTCMemberContent() map[string]any { + return voip.BuildRTCMembershipContent(voip.MatrixRTCSession{ + UserID: "@alice:example.com", + DeviceID: "DEVICE", + Intent: "audio", + Focus: voip.Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + }) +} diff --git a/pkg/connector/mediarequest.go b/pkg/connector/mediarequest.go index 3806196..2686250 100644 --- a/pkg/connector/mediarequest.go +++ b/pkg/connector/mediarequest.go @@ -155,8 +155,8 @@ func (wa *WhatsAppClient) sendMediaRequestDirect(ctx context.Context, rawMsgID n return wa.Client.SendMediaRetryReceipt(ctx, &types.MessageInfo{ ID: msgID.ID, MessageSource: types.MessageSource{ - IsFromMe: wa.IsOwnJID(msgID.Sender), - IsGroup: msgID.Chat.Server != types.DefaultUserServer && msgID.Chat.Server != types.HiddenUserServer && msgID.Chat.Server != types.BotServer, + IsFromMe: msgID.Sender.User == wa.JID.User, + IsGroup: msgID.Chat.Server != types.DefaultUserServer && msgID.Chat.Server != types.BotServer, Sender: msgID.Sender, Chat: msgID.Chat, }, diff --git a/pkg/connector/startchat.go b/pkg/connector/startchat.go index c123311..fb2ce16 100644 --- a/pkg/connector/startchat.go +++ b/pkg/connector/startchat.go @@ -25,7 +25,6 @@ import ( "time" "github.com/rs/zerolog" - "go.mau.fi/util/exmaps" "go.mau.fi/util/exsync" "go.mau.fi/util/ptr" "go.mau.fi/whatsmeow" @@ -120,24 +119,16 @@ func (wa *WhatsAppConnector) ValidateUserID(id networkid.UserID) bool { } } -func (wa *WhatsAppClient) startChatPNToLID(ctx context.Context, jid types.JID) (types.JID, error) { - if jid.Server == types.DefaultUserServer { - lid, err := wa.GetStore().LIDs.GetLIDForPN(ctx, jid) +func (wa *WhatsAppClient) startChatLIDToPN(ctx context.Context, jid types.JID) (types.JID, error) { + if jid.Server == types.HiddenUserServer { + pn, err := wa.GetStore().LIDs.GetPNForLID(ctx, jid) if err != nil { - return jid, fmt.Errorf("failed to get lid for phone number: %w", err) - } else if lid.IsEmpty() { - resp, err := wa.Client.GetUserInfo(ctx, []types.JID{jid}) - if err != nil { - return jid, fmt.Errorf("failed to get user info for phone number: %w", err) - } else if info, ok := resp[jid]; !ok { - return jid, fmt.Errorf("server didn't return user info for phone number") - } else if info.LID.IsEmpty() { - return jid, fmt.Errorf("server didn't return lid for phone number") - } else { - return info.LID, nil - } + return jid, fmt.Errorf("failed to get phone number for lid: %w", err) + } else if pn.IsEmpty() { + // Don't allow starting chats with LIDs for now + return jid, fmt.Errorf("phone number not found") } - return lid, nil + return pn, nil } return jid, nil } @@ -156,7 +147,7 @@ func (wa *WhatsAppClient) makeCreateChatResponse(ctx context.Context, jid, origJ func (wa *WhatsAppClient) CreateChatWithGhost(ctx context.Context, ghost *bridgev2.Ghost) (*bridgev2.CreateChatResponse, error) { origJID := waid.ParseUserID(ghost.ID) - jid, err := wa.startChatPNToLID(ctx, origJID) + jid, err := wa.startChatLIDToPN(ctx, origJID) if err != nil { return nil, err } @@ -168,7 +159,7 @@ func (wa *WhatsAppClient) ResolveIdentifier(ctx context.Context, identifier stri if err != nil { return nil, err } - jid, err := wa.startChatPNToLID(ctx, origJID) + jid, err := wa.startChatLIDToPN(ctx, origJID) if err != nil { return nil, err } @@ -208,7 +199,6 @@ func (wa *WhatsAppClient) getContactList(ctx context.Context, filter string, onl return nil, err } resp := make([]*bridgev2.ResolveIdentifierResponse, 0, len(contacts)) - addedIDs := make(exmaps.Set[types.JID]) for jid, contactInfo := range contacts { if onlyContacts && (contactInfo.FirstName == "" && contactInfo.FullName == "") { continue @@ -216,43 +206,31 @@ func (wa *WhatsAppClient) getContactList(ctx context.Context, filter string, onl if !matchesQuery(contactInfo.PushName, filter) && !matchesQuery(contactInfo.FullName, filter) && !matchesQuery(jid.User, filter) { continue } - var lid types.JID - if jid.Server == types.HiddenUserServer { - lid = jid - } else if jid.Server == types.DefaultUserServer { - lid, err = wa.GetStore().LIDs.GetLIDForPN(ctx, jid) - if err != nil { - return nil, fmt.Errorf("failed to get lid for phone number %s: %w", jid, err) - } else if !lid.IsEmpty() { - jid = lid - } - } - if !addedIDs.Add(jid) { - continue - } - var chatResp *bridgev2.CreateChatResponse - if !lid.IsEmpty() { - chatResp = &bridgev2.CreateChatResponse{PortalKey: wa.makeWAPortalKey(lid)} - } ghost, _ := wa.Main.Bridge.GetGhostByID(ctx, waid.MakeUserID(jid)) resp = append(resp, &bridgev2.ResolveIdentifierResponse{ Ghost: ghost, UserID: waid.MakeUserID(jid), UserInfo: wa.contactToUserInfo(ctx, jid, contactInfo, false), - Chat: chatResp, + Chat: &bridgev2.CreateChatResponse{PortalKey: wa.makeWAPortalKey(jid)}, }) } return resp, nil } func (wa *WhatsAppClient) CreateGroup(ctx context.Context, params *bridgev2.GroupCreateParams) (*bridgev2.CreateChatResponse, error) { + createKey := wa.Client.GenerateMessageID() + if params.RoomID != "" { + wa.createDedup.Add(createKey) + } req := whatsmeow.ReqCreateGroup{ Name: ptr.Val(params.Name).Name, Participants: make([]types.JID, len(params.Participants)), + CreateKey: createKey, } for i, participant := range params.Participants { jid := waid.ParseUserID(participant) - jid, err := wa.startChatPNToLID(ctx, jid) + // Normalize to PN if it's a LID + jid, err := wa.startChatLIDToPN(ctx, jid) if err != nil { return nil, fmt.Errorf("failed to normalize participant %s: %w", participant, err) } diff --git a/pkg/connector/voip/audio.go b/pkg/connector/voip/audio.go new file mode 100644 index 0000000..07e9877 --- /dev/null +++ b/pkg/connector/voip/audio.go @@ -0,0 +1,151 @@ +package voip + +import ( + "errors" + "fmt" + "io" + "math" + "sync" + + lkmedia "github.com/livekit/media-sdk" + "github.com/purpshell/meowcaller" +) + +var ErrAudioSourceClosed = errors.New("voip: audio source closed") + +func Float32FrameToPCM16(frame []float32) lkmedia.PCM16Sample { + sample := make(lkmedia.PCM16Sample, len(frame)) + for i, value := range frame { + switch { + case value > 1: + value = 1 + case value < -1: + value = -1 + } + if value == 1 { + sample[i] = math.MaxInt16 + } else { + sample[i] = int16(value * 32768) + } + } + return sample +} + +func PCM16ToFloat32Frame(sample lkmedia.PCM16Sample) []float32 { + frame := make([]float32, len(sample)) + for i, value := range sample { + frame[i] = float32(value) / 32768 + } + return frame +} + +type LiveKitPCMWriter struct { + mu sync.RWMutex + track interface { + WriteSample(lkmedia.PCM16Sample) error + } + closed bool +} + +func NewLiveKitPCMWriter(track interface { + WriteSample(lkmedia.PCM16Sample) error +}) *LiveKitPCMWriter { + return &LiveKitPCMWriter{track: track} +} + +func (w *LiveKitPCMWriter) WriteFrame(frame []float32) error { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return ErrAudioSourceClosed + } + if w.track == nil { + return nil + } + return w.track.WriteSample(Float32FrameToPCM16(frame)) +} + +func (w *LiveKitPCMWriter) Close() error { + w.mu.Lock() + w.closed = true + w.track = nil + w.mu.Unlock() + return nil +} + +type MeowcallerAudioSource struct { + mu sync.Mutex + cond *sync.Cond + queue []float32 + closed bool + maxSize int +} + +func NewMeowcallerAudioSource(maxFrames int) *MeowcallerAudioSource { + if maxFrames <= 0 { + maxFrames = 8 + } + src := &MeowcallerAudioSource{ + maxSize: maxFrames * meowcaller.FrameSamples, + } + src.cond = sync.NewCond(&src.mu) + return src +} + +func (src *MeowcallerAudioSource) WriteSample(sample lkmedia.PCM16Sample) error { + src.mu.Lock() + defer src.mu.Unlock() + if src.closed { + return ErrAudioSourceClosed + } + frame := PCM16ToFloat32Frame(sample) + src.queue = append(src.queue, frame...) + if len(src.queue) > src.maxSize { + copy(src.queue, src.queue[len(src.queue)-src.maxSize:]) + src.queue = src.queue[:src.maxSize] + } + src.cond.Signal() + return nil +} + +func (src *MeowcallerAudioSource) SampleRate() int { + return meowcaller.SampleRate +} + +func (src *MeowcallerAudioSource) String() string { + return fmt.Sprintf("MeowcallerAudioSource(%d)", meowcaller.SampleRate) +} + +func (src *MeowcallerAudioSource) ReadFrame() ([]float32, error) { + src.mu.Lock() + defer src.mu.Unlock() + for len(src.queue) < meowcaller.FrameSamples && !src.closed { + src.cond.Wait() + } + if len(src.queue) == 0 && src.closed { + return nil, io.EOF + } + frame := make([]float32, meowcaller.FrameSamples) + n := copy(frame, src.queue) + if n == len(src.queue) { + src.queue = src.queue[:0] + } else { + copy(src.queue, src.queue[n:]) + src.queue = src.queue[:len(src.queue)-n] + } + return frame, nil +} + +func (src *MeowcallerAudioSource) Close() error { + src.mu.Lock() + src.closed = true + src.queue = nil + src.cond.Broadcast() + src.mu.Unlock() + return nil +} + +var ( + _ meowcaller.AudioSink = (*LiveKitPCMWriter)(nil) + _ meowcaller.AudioSource = (*MeowcallerAudioSource)(nil) +) diff --git a/pkg/connector/voip/audio_test.go b/pkg/connector/voip/audio_test.go new file mode 100644 index 0000000..6cd44b0 --- /dev/null +++ b/pkg/connector/voip/audio_test.go @@ -0,0 +1,49 @@ +package voip + +import ( + "io" + "math" + "testing" + + lkmedia "github.com/livekit/media-sdk" + "github.com/purpshell/meowcaller" +) + +func TestFloat32FrameToPCM16ClipsAndScales(t *testing.T) { + frame := []float32{-2, -1, -0.5, 0, 0.5, 1, 2} + sample := Float32FrameToPCM16(frame) + expected := lkmedia.PCM16Sample{math.MinInt16, math.MinInt16, -16384, 0, 16384, math.MaxInt16, math.MaxInt16} + for i, value := range expected { + if sample[i] != value { + t.Fatalf("sample[%d] = %d, want %d", i, sample[i], value) + } + } +} + +func TestMeowcallerAudioSourceFramesAndEOF(t *testing.T) { + src := NewMeowcallerAudioSource(1) + sample := make(lkmedia.PCM16Sample, meowcaller.FrameSamples) + for i := range sample { + sample[i] = int16(i) + } + if err := src.WriteSample(sample); err != nil { + t.Fatalf("WriteSample returned error: %v", err) + } + frame, err := src.ReadFrame() + if err != nil { + t.Fatalf("ReadFrame returned error: %v", err) + } + if len(frame) != meowcaller.FrameSamples { + t.Fatalf("frame length = %d, want %d", len(frame), meowcaller.FrameSamples) + } + if frame[1] != float32(1)/32768 { + t.Fatalf("frame[1] = %f, want %f", frame[1], float32(1)/32768) + } + if err = src.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + _, err = src.ReadFrame() + if err != io.EOF { + t.Fatalf("ReadFrame after close returned %v, want io.EOF", err) + } +} diff --git a/pkg/connector/voip/config.go b/pkg/connector/voip/config.go new file mode 100644 index 0000000..0325bd1 --- /dev/null +++ b/pkg/connector/voip/config.go @@ -0,0 +1,54 @@ +package voip + +import "time" + +type Config struct { + Enabled bool + IncomingPolicy string + MaxActiveCallsPerLogin int + MatrixRTC MatrixRTCConfig + LiveKit LiveKitConfig + Audio AudioConfig + Video VideoConfig + Diagnostics DiagnosticsConfig +} + +type MatrixRTCConfig struct { + LiveKitServiceURL string + RequireLiveKitFocus bool + MembershipEventCompat string + NotificationEventCompat string + UseDelayedEvents bool + ParticipantMode string + FallbackParticipantMXID string +} + +type LiveKitConfig struct { + ConnectTimeout time.Duration + PublishSilenceBeforeWhatsAppAnswer bool + AutoSubscribe bool + AudioUplinkPolicy string + SelectedParticipantTimeout time.Duration +} + +type AudioConfig struct { + Enabled bool + JitterBuffer time.Duration + OpusBackend string + SilenceOnUnderrun bool + MaxMixParticipants int +} + +type VideoConfig struct { + Enabled bool + SelectedSourcePolicy string + MaxWidth int + MaxHeight int + MaxFPS int +} + +type DiagnosticsConfig struct { + HealthcheckFocusOnStartup bool + EnableMeowcallerDiagnostics bool + MediaTraceDir string +} diff --git a/pkg/connector/voip/errors.go b/pkg/connector/voip/errors.go new file mode 100644 index 0000000..a6a97e5 --- /dev/null +++ b/pkg/connector/voip/errors.go @@ -0,0 +1,6 @@ +package voip + +import "errors" + +var ErrNotEnabled = errors.New("voip: MatrixRTC LiveKit bridge is not enabled") +var ErrCallNotFound = errors.New("voip: call not found") diff --git a/pkg/connector/voip/focus.go b/pkg/connector/voip/focus.go new file mode 100644 index 0000000..e1490c1 --- /dev/null +++ b/pkg/connector/voip/focus.go @@ -0,0 +1,241 @@ +package voip + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +const FocusWellKnownKey = "org.matrix.msc4143.rtc_foci" + +var ErrNoLiveKitFocus = errors.New("voip: no LiveKit MatrixRTC focus found") + +type Focus struct { + Type string `json:"type"` + LiveKitServiceURL string `json:"livekit_service_url"` +} + +type WellKnownClient struct { + RTCFoci []Focus `json:"org.matrix.msc4143.rtc_foci"` +} + +func DiscoverLiveKitFocus(ctx context.Context, httpClient *http.Client, serverName, overrideURL string) (*Focus, error) { + if overrideURL != "" { + if err := validateHTTPSURL(overrideURL); err != nil { + return nil, fmt.Errorf("invalid configured livekit service URL: %w", err) + } + return &Focus{Type: "livekit", LiveKitServiceURL: overrideURL}, nil + } + if serverName == "" { + return nil, fmt.Errorf("matrix server name is required") + } + if strings.Contains(serverName, "://") { + return nil, fmt.Errorf("matrix server name must not include a scheme") + } + if httpClient == nil { + httpClient = http.DefaultClient + } + wellKnownURL := "https://" + serverName + "/.well-known/matrix/client" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, wellKnownURL, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch Matrix client well-known: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("Matrix client well-known returned HTTP %d", resp.StatusCode) + } + var wellKnown WellKnownClient + if err = json.NewDecoder(resp.Body).Decode(&wellKnown); err != nil { + return nil, fmt.Errorf("failed to decode Matrix client well-known: %w", err) + } + for _, focus := range wellKnown.RTCFoci { + if focus.Type != "livekit" || focus.LiveKitServiceURL == "" { + continue + } + if err = validateHTTPSURL(focus.LiveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit focus URL in well-known: %w", err) + } + return &focus, nil + } + return nil, ErrNoLiveKitFocus +} + +type MatrixOpenIDToken struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + MatrixServerName string `json:"matrix_server_name"` + ExpiresIn int `json:"expires_in"` +} + +type LiveKitAuthRequest struct { + RoomID string `json:"room_id,omitempty"` + SlotID string `json:"slot_id,omitempty"` + OpenIDToken MatrixOpenIDToken `json:"openid_token"` + Member *LiveKitAuthMember `json:"member,omitempty"` + DeviceID string `json:"device_id,omitempty"` + SessionID string `json:"session_id,omitempty"` + ParticipantID string `json:"participant_id,omitempty"` + FocusType string `json:"focus_type,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +type LegacyLiveKitAuthRequest struct { + Room string `json:"room"` + OpenIDToken MatrixOpenIDToken `json:"openid_token"` + DeviceID string `json:"device_id"` +} + +type LiveKitAuthMember struct { + ID string `json:"id,omitempty"` + ClaimedDeviceID string `json:"claimed_device_id,omitempty"` + ClaimedUserID string `json:"claimed_user_id,omitempty"` +} + +type LiveKitAuthResponse struct { + URL string `json:"url,omitempty"` + Token string `json:"token,omitempty"` + JWTToken string `json:"jwt,omitempty"` + RoomName string `json:"room,omitempty"` + + ServerURL string `json:"server_url,omitempty"` + LiveKitURL string `json:"livekit_url,omitempty"` + AccessToken string `json:"access_token,omitempty"` +} + +func (resp LiveKitAuthResponse) ConnectionURL() string { + for _, candidate := range []string{resp.URL, resp.ServerURL, resp.LiveKitURL} { + if candidate != "" { + return candidate + } + } + return "" +} + +func (resp LiveKitAuthResponse) JWT() string { + for _, candidate := range []string{resp.Token, resp.JWTToken, resp.AccessToken} { + if candidate != "" { + return candidate + } + } + return "" +} + +func RequestLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, authReq LiveKitAuthRequest) (*LiveKitAuthResponse, error) { + if err := validateHTTPSURL(liveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit service URL: %w", err) + } + if httpClient == nil { + httpClient = http.DefaultClient + } + body, err := json.Marshal(authReq) + if err != nil { + return nil, err + } + var lastErr error + for _, endpoint := range liveKitAuthEndpoints(liveKitServiceURL) { + resp, err := postLiveKitAuth(ctx, httpClient, endpoint, body) + if err != nil { + lastErr = err + continue + } + if resp.ConnectionURL() == "" || resp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + return resp, nil + } + if lastErr != nil { + return nil, lastErr + } + return nil, fmt.Errorf("livekit auth did not try any endpoints") +} + +func RequestLegacyLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, authReq LegacyLiveKitAuthRequest) (*LiveKitAuthResponse, error) { + if err := validateHTTPSURL(liveKitServiceURL); err != nil { + return nil, fmt.Errorf("invalid livekit service URL: %w", err) + } + if httpClient == nil { + httpClient = http.DefaultClient + } + body, err := json.Marshal(authReq) + if err != nil { + return nil, err + } + resp, err := postLiveKitAuth(ctx, httpClient, legacyLiveKitAuthEndpoint(liveKitServiceURL), body) + if err != nil { + return nil, err + } + if resp.ConnectionURL() == "" || resp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + return resp, nil +} + +func liveKitAuthEndpoints(rawURL string) []string { + trimmed := strings.TrimRight(rawURL, "/") + if strings.HasSuffix(trimmed, "/get_token") || strings.HasSuffix(trimmed, "/sfu/get") { + return []string{trimmed} + } + return []string{ + trimmed + "/get_token", + trimmed + "/sfu/get", + trimmed, + } +} + +func legacyLiveKitAuthEndpoint(rawURL string) string { + trimmed := strings.TrimRight(rawURL, "/") + if strings.HasSuffix(trimmed, "/sfu/get") { + return trimmed + } + if strings.HasSuffix(trimmed, "/get_token") { + return strings.TrimSuffix(trimmed, "/get_token") + "/sfu/get" + } + return trimmed + "/sfu/get" +} + +func postLiveKitAuth(ctx context.Context, httpClient *http.Client, liveKitServiceURL string, body []byte) (*LiveKitAuthResponse, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, liveKitServiceURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to request livekit token: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("livekit auth returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + var authResp LiveKitAuthResponse + if err = json.NewDecoder(resp.Body).Decode(&authResp); err != nil { + return nil, fmt.Errorf("failed to decode livekit auth response: %w", err) + } + return &authResp, nil +} + +func validateHTTPSURL(rawURL string) error { + parsed, err := url.Parse(rawURL) + if err != nil { + return err + } + if parsed.Scheme != "https" && parsed.Scheme != "wss" { + return fmt.Errorf("URL must use https or wss") + } + if parsed.Host == "" { + return fmt.Errorf("URL must include a host") + } + return nil +} diff --git a/pkg/connector/voip/focus_test.go b/pkg/connector/voip/focus_test.go new file mode 100644 index 0000000..e41b1d8 --- /dev/null +++ b/pkg/connector/voip/focus_test.go @@ -0,0 +1,131 @@ +package voip + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestDiscoverLiveKitFocusOverride(t *testing.T) { + focus, err := DiscoverLiveKitFocus(context.Background(), nil, "", "https://rtc.example.com/livekit/jwt") + if err != nil { + t.Fatalf("DiscoverLiveKitFocus returned error: %v", err) + } + if focus.Type != "livekit" || focus.LiveKitServiceURL != "https://rtc.example.com/livekit/jwt" { + t.Fatalf("unexpected focus: %+v", focus) + } +} + +func TestRequestLiveKitAuthAcceptsResponseAliases(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/get_token" { + t.Fatalf("path = %q, want /get_token", r.URL.Path) + } + var req LiveKitAuthRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + if req.OpenIDToken.AccessToken != "openid" { + t.Fatalf("OpenID token = %q, want openid", req.OpenIDToken.AccessToken) + } + _ = json.NewEncoder(w).Encode(LiveKitAuthResponse{ + ServerURL: "wss://livekit.example.com", + JWTToken: "jwt", + }) + })) + defer server.Close() + + resp, err := RequestLiveKitAuth(context.Background(), server.Client(), server.URL, LiveKitAuthRequest{ + OpenIDToken: MatrixOpenIDToken{AccessToken: "openid"}, + }) + if err != nil { + t.Fatalf("RequestLiveKitAuth returned error: %v", err) + } + if resp.ConnectionURL() != "wss://livekit.example.com" || resp.JWT() != "jwt" { + t.Fatalf("unexpected response aliases: %+v", resp) + } +} + +func TestRequestLegacyLiveKitAuthUsesSFUEndpoint(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/sfu/get" { + t.Fatalf("path = %q, want /sfu/get", r.URL.Path) + } + var req LegacyLiveKitAuthRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + if req.Room != "!room:example.com" || req.DeviceID != "DEVICE" { + t.Fatalf("unexpected legacy request: %+v", req) + } + _ = json.NewEncoder(w).Encode(LiveKitAuthResponse{ + URL: "wss://livekit.example.com", + Token: "jwt", + }) + })) + defer server.Close() + + resp, err := RequestLegacyLiveKitAuth(context.Background(), server.Client(), server.URL, LegacyLiveKitAuthRequest{ + Room: "!room:example.com", + DeviceID: "DEVICE", + OpenIDToken: MatrixOpenIDToken{AccessToken: "openid", MatrixServerName: "example.com"}, + }) + if err != nil { + t.Fatalf("RequestLegacyLiveKitAuth returned error: %v", err) + } + if resp.ConnectionURL() != "wss://livekit.example.com" || resp.JWT() != "jwt" { + t.Fatalf("unexpected response: %+v", resp) + } +} + +func TestLiveKitAuthEndpoints(t *testing.T) { + tests := []struct { + name string + url string + want []string + }{ + { + name: "base", + url: "https://rtc.example.com/livekit/jwt", + want: []string{ + "https://rtc.example.com/livekit/jwt/get_token", + "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt", + }, + }, + { + name: "explicit", + url: "https://rtc.example.com/livekit/jwt/sfu/get", + want: []string{"https://rtc.example.com/livekit/jwt/sfu/get"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := liveKitAuthEndpoints(test.url) + if len(got) != len(test.want) { + t.Fatalf("got %d endpoints, want %d: %+v", len(got), len(test.want), got) + } + for i := range got { + if got[i] != test.want[i] { + t.Fatalf("endpoint %d = %q, want %q", i, got[i], test.want[i]) + } + } + }) + } +} + +func TestLegacyLiveKitAuthEndpoint(t *testing.T) { + tests := map[string]string{ + "https://rtc.example.com/livekit/jwt": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/get_token": "https://rtc.example.com/livekit/jwt/sfu/get", + "https://rtc.example.com/livekit/jwt/sfu/get": "https://rtc.example.com/livekit/jwt/sfu/get", + } + for input, want := range tests { + if got := legacyLiveKitAuthEndpoint(input); got != want { + t.Fatalf("legacy endpoint for %q = %q, want %q", input, got, want) + } + } +} diff --git a/pkg/connector/voip/group_media.go b/pkg/connector/voip/group_media.go new file mode 100644 index 0000000..9f87d58 --- /dev/null +++ b/pkg/connector/voip/group_media.go @@ -0,0 +1,295 @@ +package voip + +import ( + "errors" + "sync" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" +) + +type whatsAppVideoSink interface { + WriteVideo([]byte) error + SetOrientation(int) +} + +type whatsAppVideoRouter struct { + mu sync.Mutex + + camera whatsAppVideoSink + screen whatsAppVideoSink + setCameraMute func(bool) + setScreenMute func(bool) + + group bool + connected map[string]struct{} + aliases map[string]string + screenSharers map[string]struct{} + selectedCamera string + selectedScreen string +} + +func newWhatsAppVideoRouter( + camera, screen whatsAppVideoSink, + setCameraMute, setScreenMute func(bool), +) *whatsAppVideoRouter { + return &whatsAppVideoRouter{ + camera: camera, + screen: screen, + setCameraMute: setCameraMute, + setScreenMute: setScreenMute, + connected: make(map[string]struct{}), + aliases: make(map[string]string), + screenSharers: make(map[string]struct{}), + } +} + +func (r *whatsAppVideoRouter) SetGroupState(state meowcaller.GroupCallState) { + if r == nil { + return + } + connected := make(map[string]struct{}) + aliases := make(map[string]string) + for _, participant := range state.Participants { + if participant.State != "connected" { + continue + } + canonical := videoParticipantIdentity(participant.JID) + if canonical == "" { + canonical = videoParticipantIdentity(participant.PN) + } + if canonical == "" && len(participant.Devices) > 0 { + canonical = videoParticipantIdentity(participant.Devices[0].JID) + } + if canonical == "" { + continue + } + connected[canonical] = struct{}{} + addVideoParticipantAlias(aliases, participant.JID, canonical) + addVideoParticipantAlias(aliases, participant.PN, canonical) + for _, device := range participant.Devices { + addVideoParticipantAlias(aliases, device.JID, canonical) + } + } + + r.mu.Lock() + r.group = true + r.connected = connected + r.aliases = aliases + cameraRemoved := r.selectedCamera != "" + if cameraRemoved { + _, cameraRemoved = connected[r.selectedCamera] + cameraRemoved = !cameraRemoved + } + if cameraRemoved { + r.selectedCamera = "" + } + screenRemoved := r.selectedScreen != "" + if screenRemoved { + _, screenRemoved = connected[r.selectedScreen] + screenRemoved = !screenRemoved + } + if screenRemoved { + delete(r.screenSharers, r.selectedScreen) + r.selectedScreen = "" + } + setCameraMute := r.setCameraMute + setScreenMute := r.setScreenMute + r.mu.Unlock() + + if cameraRemoved && setCameraMute != nil { + setCameraMute(true) + } + if screenRemoved && setScreenMute != nil { + setScreenMute(true) + } +} + +func (r *whatsAppVideoRouter) SelectCamera(participant types.JID) error { + if r == nil { + return errors.New("WhatsApp video router is not available") + } + identity := videoParticipantIdentity(participant) + r.mu.Lock() + if !r.group { + r.mu.Unlock() + return errors.New("the active call has no WhatsApp group video roster") + } + canonical := r.aliases[identity] + if canonical == "" { + canonical = identity + } + if _, ok := r.connected[canonical]; !ok { + r.mu.Unlock() + return errors.New("the selected WhatsApp participant is not connected") + } + r.selectedCamera = canonical + setCameraMute := r.setCameraMute + r.mu.Unlock() + if setCameraMute != nil { + setCameraMute(true) + } + return nil +} + +func (r *whatsAppVideoRouter) SetScreenShare(state meowcaller.ScreenShareState) { + if r == nil || state.Participant.IsEmpty() { + return + } + participant := videoParticipantIdentity(state.Participant) + r.mu.Lock() + if canonical := r.aliases[participant]; canonical != "" { + participant = canonical + } + if state.Active { + r.screenSharers[participant] = struct{}{} + if r.selectedScreen == "" { + r.selectedScreen = participant + } + } else { + delete(r.screenSharers, participant) + if r.selectedScreen == participant { + r.selectedScreen = "" + } + } + selected := r.selectedScreen + setScreenMute := r.setScreenMute + r.mu.Unlock() + + if setScreenMute != nil { + setScreenMute(selected == "") + } +} + +func (r *whatsAppVideoRouter) WriteParticipantFrame(frame meowcaller.ParticipantVideoFrame) { + if r == nil || len(frame.AccessUnit) == 0 { + return + } + identity := participantVideoFrameIdentity(frame) + r.mu.Lock() + if canonical := r.aliases[identity]; canonical != "" { + identity = canonical + } + _, sharing := r.screenSharers[identity] + if sharing { + if r.selectedScreen == "" { + r.selectedScreen = identity + } + if r.selectedScreen != identity { + r.mu.Unlock() + return + } + sink := r.screen + setMuted := r.setScreenMute + r.mu.Unlock() + if setMuted != nil { + setMuted(false) + } + writeWhatsAppVideoFrame(sink, frame) + return + } + + if r.group { + if r.selectedCamera == "" { + r.selectedCamera = identity + } + if r.selectedCamera != identity { + r.mu.Unlock() + return + } + } + sink := r.camera + setMuted := r.setCameraMute + r.mu.Unlock() + if setMuted != nil { + setMuted(false) + } + writeWhatsAppVideoFrame(sink, frame) +} + +func writeWhatsAppVideoFrame(sink whatsAppVideoSink, frame meowcaller.ParticipantVideoFrame) { + if sink == nil { + return + } + sink.SetOrientation(frame.Orientation) + _ = sink.WriteVideo(frame.AccessUnit) +} + +func participantVideoFrameIdentity(frame meowcaller.ParticipantVideoFrame) string { + if !frame.Sender.IsEmpty() { + return videoParticipantIdentity(frame.Sender) + } + if !frame.Device.IsEmpty() { + return videoParticipantIdentity(frame.Device) + } + return frame.ParticipantID +} + +func addVideoParticipantAlias(target map[string]string, jid types.JID, canonical string) { + if identity := videoParticipantIdentity(jid); identity != "" { + target[identity] = canonical + } +} + +func videoParticipantIdentity(jid types.JID) string { + if jid.IsEmpty() { + return "" + } + return jid.ToNonAD().String() +} + +type matrixVideoSourceRouter struct { + mu sync.RWMutex + screenSharing bool + write func(LiveKitVideoFrame) error +} + +func newMatrixVideoSourceRouter(write func(LiveKitVideoFrame) error) *matrixVideoSourceRouter { + return &matrixVideoSourceRouter{write: write} +} + +func (r *matrixVideoSourceRouter) SetScreenSharing(active bool) { + if r == nil { + return + } + r.mu.Lock() + r.screenSharing = active + r.mu.Unlock() +} + +func (r *matrixVideoSourceRouter) ScreenSharing() bool { + if r == nil { + return false + } + r.mu.RLock() + defer r.mu.RUnlock() + return r.screenSharing +} + +func (r *matrixVideoSourceRouter) WriteCamera(frame LiveKitVideoFrame) error { + if r == nil { + return nil + } + r.mu.RLock() + screenSharing := r.screenSharing + write := r.write + r.mu.RUnlock() + if screenSharing || write == nil { + return nil + } + return write(frame) +} + +func (r *matrixVideoSourceRouter) WriteScreen(frame LiveKitVideoFrame) error { + if r == nil { + return nil + } + r.mu.RLock() + screenSharing := r.screenSharing + write := r.write + r.mu.RUnlock() + if !screenSharing || write == nil { + return nil + } + return write(frame) +} diff --git a/pkg/connector/voip/group_media_test.go b/pkg/connector/voip/group_media_test.go new file mode 100644 index 0000000..e29b1ac --- /dev/null +++ b/pkg/connector/voip/group_media_test.go @@ -0,0 +1,185 @@ +package voip + +import ( + "testing" + + "github.com/purpshell/meowcaller" + "go.mau.fi/whatsmeow/types" +) + +type recordingGroupVideoSink struct { + frames [][]byte + orientations []int + muted []bool +} + +func (s *recordingGroupVideoSink) WriteVideo(frame []byte) error { + s.frames = append(s.frames, append([]byte(nil), frame...)) + return nil +} + +func (s *recordingGroupVideoSink) SetOrientation(orientation int) { + s.orientations = append(s.orientations, orientation) +} + +func (s *recordingGroupVideoSink) setMuted(muted bool) { + s.muted = append(s.muted, muted) +} + +func TestWhatsAppVideoRouterKeepsOneStableGroupCamera(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + alice := types.NewJID("111", types.DefaultUserServer) + bob := types.NewJID("222", types.DefaultUserServer) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + {JID: alice, State: "connected"}, + {JID: bob, State: "connected"}, + }, + }) + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + Orientation: 1, + AccessUnit: []byte{0x01}, + }) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: bob.String(), + Sender: bob, + Orientation: 2, + AccessUnit: []byte{0x02}, + }) + + if len(camera.frames) != 1 || camera.frames[0][0] != 0x01 { + t.Fatalf("camera frames = %v, want only the first connected participant", camera.frames) + } + if len(camera.orientations) != 1 || camera.orientations[0] != 1 { + t.Fatalf("camera orientations = %v, want [1]", camera.orientations) + } +} + +func TestWhatsAppVideoRouterSelectsCameraByPhoneNumberAlias(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + aliceLID := types.NewJID("111", types.HiddenUserServer) + alicePN := types.NewJID("15550000001", types.DefaultUserServer) + bobLID := types.NewJID("222", types.HiddenUserServer) + bobPN := types.NewJID("15550000002", types.DefaultUserServer) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + {JID: aliceLID, PN: alicePN, State: "connected"}, + {JID: bobLID, PN: bobPN, State: "connected"}, + }, + }) + if err := router.SelectCamera(bobPN); err != nil { + t.Fatalf("SelectCamera returned error: %v", err) + } + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + Sender: aliceLID, + AccessUnit: []byte{0x01}, + }) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + Sender: bobLID, + AccessUnit: []byte{0x02}, + }) + + if len(camera.frames) != 1 || camera.frames[0][0] != 0x02 { + t.Fatalf("camera frames = %v, want only the selected participant", camera.frames) + } +} + +func TestWhatsAppVideoRouterRejectsDisconnectedCameraSelection(t *testing.T) { + router := newWhatsAppVideoRouter( + &recordingGroupVideoSink{}, + &recordingGroupVideoSink{}, + nil, + nil, + ) + router.SetGroupState(meowcaller.GroupCallState{ + Participants: []meowcaller.GroupCallParticipant{ + { + JID: types.NewJID("111", types.HiddenUserServer), + State: "connected", + }, + }, + }) + if err := router.SelectCamera(types.NewJID("222", types.HiddenUserServer)); err == nil { + t.Fatal("SelectCamera accepted a disconnected participant") + } +} + +func TestWhatsAppVideoRouterSeparatesScreenShareFromCamera(t *testing.T) { + camera := &recordingGroupVideoSink{} + screen := &recordingGroupVideoSink{} + router := newWhatsAppVideoRouter(camera, screen, camera.setMuted, screen.setMuted) + alice := types.NewJID("111", types.DefaultUserServer) + router.SetScreenShare(meowcaller.ScreenShareState{Participant: alice, Active: true}) + + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + Orientation: 3, + AccessUnit: []byte{0x03}, + }) + + if len(camera.frames) != 0 { + t.Fatalf("camera received screen-share frames: %v", camera.frames) + } + if len(screen.frames) != 1 || screen.frames[0][0] != 0x03 { + t.Fatalf("screen frames = %v, want one screen-share frame", screen.frames) + } + if len(screen.orientations) != 1 || screen.orientations[0] != 3 { + t.Fatalf("screen orientations = %v, want [3]", screen.orientations) + } + if len(screen.muted) == 0 || screen.muted[len(screen.muted)-1] { + t.Fatalf("screen mute transitions = %v, want unmuted", screen.muted) + } + + router.SetScreenShare(meowcaller.ScreenShareState{Participant: alice, Active: false}) + router.WriteParticipantFrame(meowcaller.ParticipantVideoFrame{ + ParticipantID: alice.String(), + Sender: alice, + AccessUnit: []byte{0x04}, + }) + if len(camera.frames) != 1 || camera.frames[0][0] != 0x04 { + t.Fatalf("camera frames after screen-share stop = %v, want camera frame", camera.frames) + } + if !screen.muted[len(screen.muted)-1] { + t.Fatalf("screen mute transitions = %v, want muted after stop", screen.muted) + } +} + +func TestMatrixVideoRouterForwardsOnlyTheActiveSource(t *testing.T) { + var got []byte + router := newMatrixVideoSourceRouter(func(frame LiveKitVideoFrame) error { + got = append(got, frame.AccessUnit...) + return nil + }) + + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x01}}); err != nil { + t.Fatal(err) + } + router.SetScreenSharing(true) + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x02}}); err != nil { + t.Fatal(err) + } + if err := router.WriteScreen(LiveKitVideoFrame{AccessUnit: []byte{0x03}}); err != nil { + t.Fatal(err) + } + router.SetScreenSharing(false) + if err := router.WriteScreen(LiveKitVideoFrame{AccessUnit: []byte{0x04}}); err != nil { + t.Fatal(err) + } + if err := router.WriteCamera(LiveKitVideoFrame{AccessUnit: []byte{0x05}}); err != nil { + t.Fatal(err) + } + + want := []byte{0x01, 0x03, 0x05} + if string(got) != string(want) { + t.Fatalf("forwarded frames = %v, want %v", got, want) + } +} diff --git a/pkg/connector/voip/livekit.go b/pkg/connector/voip/livekit.go new file mode 100644 index 0000000..48511c8 --- /dev/null +++ b/pkg/connector/voip/livekit.go @@ -0,0 +1,720 @@ +package voip + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + lkpcm "github.com/livekit/media-sdk" + livekitproto "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + lksdk "github.com/livekit/server-sdk-go/v2" + lkmedia "github.com/livekit/server-sdk-go/v2/pkg/media" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media/samplebuilder" + "github.com/rs/zerolog" +) + +type LiveKitParticipant struct { + cfg LiveKitConfig + videoCfg VideoConfig + log zerolog.Logger + room *lksdk.Room + audio *lkmedia.PCMLocalTrack + audioPub *lksdk.LocalTrackPublication + audioSrc *MeowcallerAudioSource + video *lksdk.LocalTrack + videoPub *lksdk.LocalTrackPublication + screen *lksdk.LocalTrack + screenPub *lksdk.LocalTrackPublication + + mu sync.Mutex + remoteAudio []*lkmedia.PCMRemoteTrack + remoteMediaCancel context.CancelFunc + remoteVideoPLI lksdk.PLIWriter + remoteVideoSSRC webrtc.SSRC + remoteScreenPLI lksdk.PLIWriter + remoteScreenSSRC webrtc.SSRC + remoteScreenActive bool + remoteVideoKeyframePending [2]bool + remoteVideoKeyframeAwaited [2]bool + disconnected bool + selectedRemoteParticipant string + remoteAudioMuteStateChange func(muted bool) + remoteVideoFrame func(frame LiveKitVideoFrame) error + remoteVideoMuteStateChange func(muted bool) + remoteScreenFrame func(frame LiveKitVideoFrame) error + remoteScreenMuteStateChange func(muted bool) +} + +type liveKitVideoSource uint8 + +const ( + liveKitVideoSourceCamera liveKitVideoSource = iota + liveKitVideoSourceScreenShare +) + +func ConnectLiveKitParticipant(ctx context.Context, authResp *LiveKitAuthResponse, cfg LiveKitConfig, videoCfg VideoConfig, log zerolog.Logger) (*LiveKitParticipant, error) { + if authResp == nil { + return nil, fmt.Errorf("livekit auth response is nil") + } + if authResp.ConnectionURL() == "" || authResp.JWT() == "" { + return nil, fmt.Errorf("livekit auth response did not include both URL and token") + } + remoteMediaCtx, remoteMediaCancel := context.WithCancel(context.Background()) + participant := &LiveKitParticipant{ + cfg: cfg, + videoCfg: videoCfg, + log: log, + audioSrc: NewMeowcallerAudioSource(12), + remoteMediaCancel: remoteMediaCancel, + } + callback := &lksdk.RoomCallback{ + ParticipantCallback: lksdk.ParticipantCallback{ + OnTrackSubscribed: func(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + participant.onTrackSubscribed(remoteMediaCtx, track, publication, rp) + }, + OnTrackUnsubscribed: participant.onTrackUnsubscribed, + OnTrackMuted: participant.onTrackMuted, + OnTrackUnmuted: participant.onTrackUnmuted, + }, + OnDisconnected: func() { + participant.closeRemoteTracks() + }, + OnDisconnectedWithReason: func(reason lksdk.DisconnectionReason) { + log.Info().Str("reason", string(reason)).Msg("Disconnected from LiveKit") + participant.closeRemoteTracks() + }, + } + opts := []lksdk.ConnectOption{ + lksdk.WithAutoSubscribe(cfg.AutoSubscribe), + } + if cfg.ConnectTimeout > 0 { + opts = append(opts, lksdk.WithConnectTimeout(cfg.ConnectTimeout)) + } + room, err := connectLiveKit(ctx, authResp.ConnectionURL(), authResp.JWT(), callback, opts...) + if err != nil { + remoteMediaCancel() + return nil, err + } + participant.room = room + return participant, nil +} + +func connectLiveKit(ctx context.Context, url, token string, callback *lksdk.RoomCallback, opts ...lksdk.ConnectOption) (*lksdk.Room, error) { + room := lksdk.NewRoom(callback) + if err := room.JoinWithContextAndToken(ctx, url, token, opts...); err != nil { + return nil, err + } + return room, nil +} + +func (p *LiveKitParticipant) SetRemoteAudioMuteHandler(selectedParticipant string, handler func(muted bool)) { + p.mu.Lock() + p.selectedRemoteParticipant = selectedParticipant + p.remoteAudioMuteStateChange = handler + p.mu.Unlock() +} + +func (p *LiveKitParticipant) SetRemoteVideoHandlers(selectedParticipant string, frameHandler func(frame LiveKitVideoFrame) error, muteHandler func(muted bool)) { + p.mu.Lock() + p.selectedRemoteParticipant = selectedParticipant + p.remoteVideoFrame = frameHandler + p.remoteVideoMuteStateChange = muteHandler + p.mu.Unlock() +} + +func (p *LiveKitParticipant) SetRemoteScreenShareHandlers(frameHandler func(frame LiveKitVideoFrame) error, muteHandler func(muted bool)) { + p.mu.Lock() + p.remoteScreenFrame = frameHandler + p.remoteScreenMuteStateChange = muteHandler + p.mu.Unlock() +} + +func (p *LiveKitParticipant) requestRemoteVideoKeyframe() bool { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return false + } + pli := p.remoteVideoPLI + ssrc := p.remoteVideoSSRC + source := liveKitVideoSourceCamera + if p.remoteScreenActive && p.remoteScreenPLI != nil && p.remoteScreenSSRC != 0 { + pli = p.remoteScreenPLI + ssrc = p.remoteScreenSSRC + source = liveKitVideoSourceScreenShare + } else if p.remoteScreenActive { + source = liveKitVideoSourceScreenShare + pli = nil + ssrc = 0 + } + if pli == nil || ssrc == 0 { + p.remoteVideoKeyframePending[source] = true + p.mu.Unlock() + return false + } + p.remoteVideoKeyframePending[source] = false + p.mu.Unlock() + p.sendRemoteVideoPLI(source, pli, ssrc) + return true +} + +func (p *LiveKitParticipant) setRemoteVideoPLI(pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.setRemoteVideoPLIForSource(liveKitVideoSourceCamera, pli, ssrc) +} + +func (p *LiveKitParticipant) setRemoteVideoPLIForSource(source liveKitVideoSource, pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return + } + if source == liveKitVideoSourceScreenShare { + p.remoteScreenPLI = pli + p.remoteScreenSSRC = ssrc + } else { + p.remoteVideoPLI = pli + p.remoteVideoSSRC = ssrc + } + pending := p.remoteVideoKeyframePending[source] && pli != nil && ssrc != 0 + if pending { + p.remoteVideoKeyframePending[source] = false + } + p.mu.Unlock() + if pending { + p.sendRemoteVideoPLI(source, pli, ssrc) + } +} + +func (p *LiveKitParticipant) sendRemoteVideoPLI(source liveKitVideoSource, pli lksdk.PLIWriter, ssrc webrtc.SSRC) { + p.mu.Lock() + if !p.disconnected { + p.remoteVideoKeyframeAwaited[source] = true + } + p.mu.Unlock() + pli(ssrc) + p.log.Info().Uint32("ssrc", uint32(ssrc)).Msg("Requested LiveKit H.264 keyframe for WhatsApp peer") +} + +func (p *LiveKitParticipant) PublishAudioTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.audio != nil { + return nil + } + track, err := lkmedia.NewPCMLocalTrack(meowcallerSampleRate, 1, logger.GetLogger()) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-audio" + } + pub, err := p.room.LocalParticipant.PublishTrack(track, &lksdk.TrackPublicationOptions{Name: name}) + if err != nil { + track.Close() + return err + } + p.audio = track + p.audioPub = pub + return nil +} + +func (p *LiveKitParticipant) PublishVideoTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.video != nil { + return nil + } + track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-video" + } + pub, err := p.room.LocalParticipant.PublishTrack( + track, + videoTrackPublicationOptions(name, livekitproto.TrackSource_CAMERA, p.videoCfg), + ) + if err != nil { + _ = track.Close() + return err + } + p.video = track + p.videoPub = pub + return nil +} + +func (p *LiveKitParticipant) PublishScreenShareTrack(name string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.room == nil { + return fmt.Errorf("livekit room is not connected") + } + if p.screen != nil { + return nil + } + track, err := lksdk.NewLocalTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: liveKitH264ClockRate}) + if err != nil { + return err + } + if name == "" { + name = "whatsapp-screen" + } + pub, err := p.room.LocalParticipant.PublishTrack( + track, + videoTrackPublicationOptions(name, livekitproto.TrackSource_SCREEN_SHARE, p.videoCfg), + ) + if err != nil { + _ = track.Close() + return err + } + pub.SetMuted(true) + p.screen = track + p.screenPub = pub + return nil +} + +func videoTrackPublicationOptions(name string, source livekitproto.TrackSource, cfg VideoConfig) *lksdk.TrackPublicationOptions { + return &lksdk.TrackPublicationOptions{ + Name: name, + Source: source, + VideoWidth: cfg.MaxWidth, + VideoHeight: cfg.MaxHeight, + } +} + +func (p *LiveKitParticipant) SetWhatsAppAudioMuted(muted bool) { + p.mu.Lock() + pub := p.audioPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp audio mute state") +} + +func (p *LiveKitParticipant) SetWhatsAppVideoMuted(muted bool) { + p.mu.Lock() + pub := p.videoPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp video mute state") +} + +func (p *LiveKitParticipant) SetWhatsAppScreenShareMuted(muted bool) { + p.mu.Lock() + pub := p.screenPub + p.mu.Unlock() + if pub == nil { + return + } + pub.SetMuted(muted) + p.log.Debug().Bool("muted", muted).Msg("Set LiveKit WhatsApp screen-share mute state") +} + +func (p *LiveKitParticipant) SetWhatsAppVideoOrientation(orientation int) { + p.mu.Lock() + video := p.video + p.mu.Unlock() + if video == nil { + return + } + if setLiveKitVideoOrientation(video, orientation) { + p.log.Debug().Int("orientation", orientation&0x03).Msg("Set LiveKit WhatsApp video orientation") + } +} + +func (p *LiveKitParticipant) WhatsAppSink() *LiveKitPCMWriter { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitPCMWriter(p.audio) +} + +func (p *LiveKitParticipant) WhatsAppVideoSink() *LiveKitH264Writer { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitH264Writer(p.video, videoFrameDuration(p.videoCfg)) +} + +func (p *LiveKitParticipant) WhatsAppScreenShareSink() *LiveKitH264Writer { + p.mu.Lock() + defer p.mu.Unlock() + return NewLiveKitH264Writer(p.screen, videoFrameDuration(p.videoCfg)) +} + +func (p *LiveKitParticipant) MatrixAudioSource() *MeowcallerAudioSource { + return p.audioSrc +} + +func (p *LiveKitParticipant) WriteWhatsAppFrame(frame []float32) error { + p.mu.Lock() + audio := p.audio + p.mu.Unlock() + if audio == nil { + return nil + } + return audio.WriteSample(Float32FrameToPCM16(frame)) +} + +func (p *LiveKitParticipant) Close() { + p.mu.Lock() + if p.disconnected { + p.mu.Unlock() + return + } + p.disconnected = true + room := p.room + audio := p.audio + audioPub := p.audioPub + video := p.video + videoPub := p.videoPub + screen := p.screen + screenPub := p.screenPub + p.room = nil + p.audio = nil + p.audioPub = nil + p.video = nil + p.videoPub = nil + p.screen = nil + p.screenPub = nil + p.mu.Unlock() + p.closeRemoteTracks() + if audioPub != nil { + audioPub.SetMuted(true) + } + if videoPub != nil { + videoPub.SetMuted(true) + } + if screenPub != nil { + screenPub.SetMuted(true) + } + if audio != nil { + audio.ClearQueue() + _ = audio.Close() + } + if video != nil { + _ = video.Close() + } + if screen != nil { + _ = screen.Close() + } + if room != nil { + room.Disconnect() + } + _ = p.audioSrc.Close() +} + +func (p *LiveKitParticipant) onTrackSubscribed(ctx context.Context, track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + switch track.Kind() { + case webrtc.RTPCodecTypeAudio: + p.onAudioTrackSubscribed(track, publication, rp) + case webrtc.RTPCodecTypeVideo: + p.onVideoTrackSubscribed(ctx, track, publication, rp) + } +} + +func (p *LiveKitParticipant) onAudioTrackSubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if track.Codec().MimeType != webrtc.MimeTypeOpus { + p.log.Warn(). + Str("codec", track.Codec().MimeType). + Str("participant", string(rp.Identity())). + Msg("Ignoring non-Opus LiveKit audio track") + return + } + remote, err := lkmedia.NewPCMRemoteTrack( + track, + p.audioSrc, + lkmedia.WithTargetSampleRate(meowcallerSampleRate), + lkmedia.WithTargetChannels(1), + lkmedia.WithLogger(logger.GetLogger()), + ) + if err != nil { + p.log.Warn(). + Err(err). + Str("participant", string(rp.Identity())). + Msg("Failed to subscribe LiveKit audio track") + return + } + p.mu.Lock() + p.remoteAudio = append(p.remoteAudio, remote) + p.mu.Unlock() + p.handleRemoteAudioMuteState(publication, rp, publication.IsMuted()) + _ = publication +} + +func (p *LiveKitParticipant) onVideoTrackSubscribed(ctx context.Context, track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if !p.videoCfg.Enabled { + return + } + if !remoteParticipantSelected(p.selectedParticipant(), string(rp.Identity())) { + p.log.Debug(). + Str("participant", string(rp.Identity())). + Str("selected_participant", p.selectedParticipant()). + Msg("Ignoring LiveKit video track from non-selected participant") + return + } + if !strings.EqualFold(track.Codec().MimeType, webrtc.MimeTypeH264) { + p.log.Warn(). + Str("codec", track.Codec().MimeType). + Str("participant", string(rp.Identity())). + Msg("Ignoring unsupported LiveKit video track; only H.264 passthrough is implemented") + p.handleRemoteVideoMuteState(publication, rp, true) + return + } + source := liveKitVideoSourceRole(publication.Source()) + p.setRemoteVideoPLIForSource(source, rp.WritePLI, track.SSRC()) + p.handleRemoteVideoMuteState(publication, rp, publication.IsMuted()) + go p.forwardRemoteH264Track(ctx, track, rp, source) +} + +func (p *LiveKitParticipant) onTrackUnsubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if track.Kind() == webrtc.RTPCodecTypeVideo { + source := liveKitVideoSourceRole(publication.Source()) + p.mu.Lock() + if source == liveKitVideoSourceScreenShare && p.remoteScreenSSRC == track.SSRC() { + p.remoteScreenPLI = nil + p.remoteScreenSSRC = 0 + p.remoteScreenActive = false + p.remoteVideoKeyframePending[liveKitVideoSourceScreenShare] = true + } else if source == liveKitVideoSourceCamera && p.remoteVideoSSRC == track.SSRC() { + p.remoteVideoPLI = nil + p.remoteVideoSSRC = 0 + p.remoteVideoKeyframePending[liveKitVideoSourceCamera] = true + } + p.mu.Unlock() + p.handleRemoteVideoMuteState(publication, rp, true) + } +} + +func (p *LiveKitParticipant) onTrackMuted(pub lksdk.TrackPublication, participant lksdk.Participant) { + p.handleRemoteAudioMuteState(pub, participant, true) + p.handleRemoteVideoMuteState(pub, participant, true) +} + +func (p *LiveKitParticipant) onTrackUnmuted(pub lksdk.TrackPublication, participant lksdk.Participant) { + p.handleRemoteAudioMuteState(pub, participant, false) + p.handleRemoteVideoMuteState(pub, participant, false) +} + +func (p *LiveKitParticipant) handleRemoteAudioMuteState(pub lksdk.TrackPublication, participant lksdk.Participant, muted bool) { + if pub == nil || participant == nil || pub.Kind() != lksdk.TrackKindAudio { + return + } + if _, ok := participant.(*lksdk.RemoteParticipant); !ok { + return + } + identity := participant.Identity() + p.mu.Lock() + selected := p.selectedRemoteParticipant + handler := p.remoteAudioMuteStateChange + p.mu.Unlock() + if selected != "" && identity != selected { + p.log.Debug(). + Str("participant", identity). + Str("selected_participant", selected). + Bool("muted", muted). + Msg("Ignoring LiveKit mute state from non-selected participant") + return + } + p.log.Debug(). + Str("participant", identity). + Str("track_id", pub.SID()). + Bool("muted", muted). + Msg("Observed LiveKit remote audio mute state") + if handler != nil { + handler(muted) + } +} + +func (p *LiveKitParticipant) handleRemoteVideoMuteState(pub lksdk.TrackPublication, participant lksdk.Participant, muted bool) { + if pub == nil || participant == nil || pub.Kind() != lksdk.TrackKindVideo { + return + } + if _, ok := participant.(*lksdk.RemoteParticipant); !ok { + return + } + identity := participant.Identity() + p.mu.Lock() + selected := p.selectedRemoteParticipant + p.mu.Unlock() + if !remoteParticipantSelected(selected, identity) { + p.log.Debug(). + Str("participant", identity). + Str("selected_participant", selected). + Bool("muted", muted). + Msg("Ignoring LiveKit video mute state from non-selected participant") + return + } + source := liveKitVideoSourceRole(pub.Source()) + p.mu.Lock() + handler := p.remoteVideoMuteStateChange + if source == liveKitVideoSourceScreenShare { + p.remoteScreenActive = !muted + handler = p.remoteScreenMuteStateChange + } + p.mu.Unlock() + p.log.Debug(). + Str("participant", identity). + Str("track_id", pub.SID()). + Bool("muted", muted). + Msg("Observed LiveKit remote video mute state") + if handler != nil { + handler(muted) + } +} + +func (p *LiveKitParticipant) forwardRemoteH264Track(ctx context.Context, track *webrtc.TrackRemote, rp *lksdk.RemoteParticipant, source liveKitVideoSource) { + builder := samplebuilder.New( + liveKitH264MaxLatePackets, + &codecs.H264Packet{}, + track.Codec().ClockRate, + ) + var parameterSets h264ParameterSetRepeater + loggedIDR := false + p.log.Info(). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Str("source", source.String()). + Str("fmtp", track.Codec().SDPFmtpLine). + Msg("Started forwarding LiveKit H.264 video to WhatsApp") + for { + if ctx.Err() != nil { + return + } + packet, _, err := track.ReadRTP() + if err != nil { + if !errors.Is(err, io.EOF) && ctx.Err() == nil { + p.log.Debug(). + Err(err). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Msg("Stopped reading LiveKit H.264 video track") + } + return + } + builder.Push(packet) + for sample := builder.Pop(); sample != nil; sample = builder.Pop() { + if len(sample.Data) == 0 { + continue + } + accessUnit, repeatedParameterSets := parameterSets.Normalize(sample.Data) + nalTypes, profileLevelID, hasIDR, hasSPS, hasPPS := h264AccessUnitMetadata(accessUnit) + p.mu.Lock() + afterPLI := hasIDR && p.remoteVideoKeyframeAwaited[source] + if afterPLI { + p.remoteVideoKeyframeAwaited[source] = false + } + p.mu.Unlock() + if hasIDR && (!loggedIDR || afterPLI || repeatedParameterSets) { + p.log.Info(). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Ints("nal_types", nalTypes). + Str("profile_level_id", profileLevelID). + Int("bytes", len(accessUnit)). + Bool("has_sps", hasSPS). + Bool("has_pps", hasPPS). + Bool("after_pli", afterPLI). + Bool("repeated_parameter_sets", repeatedParameterSets). + Msg("Forwarding decoder-safe LiveKit H.264 keyframe to WhatsApp") + loggedIDR = true + } + p.mu.Lock() + handler := p.remoteVideoFrame + if source == liveKitVideoSourceScreenShare { + handler = p.remoteScreenFrame + } + p.mu.Unlock() + if handler == nil { + continue + } + if err = handler(LiveKitVideoFrame{ + AccessUnit: accessUnit, + Duration: sample.Duration, + }); err != nil { + p.log.Warn(). + Err(err). + Str("participant", string(rp.Identity())). + Str("track_id", track.ID()). + Str("source", source.String()). + Msg("Failed to forward LiveKit H.264 frame to WhatsApp") + } + } + } +} + +func liveKitVideoSourceRole(source livekitproto.TrackSource) liveKitVideoSource { + if source == livekitproto.TrackSource_SCREEN_SHARE { + return liveKitVideoSourceScreenShare + } + return liveKitVideoSourceCamera +} + +func (s liveKitVideoSource) String() string { + if s == liveKitVideoSourceScreenShare { + return "screen_share" + } + return "camera" +} + +func (p *LiveKitParticipant) selectedParticipant() string { + p.mu.Lock() + defer p.mu.Unlock() + return p.selectedRemoteParticipant +} + +func remoteParticipantSelected(selected, identity string) bool { + return selected == "" || identity == selected +} + +func (p *LiveKitParticipant) closeRemoteTracks() { + p.mu.Lock() + tracks := p.remoteAudio + p.remoteAudio = nil + p.remoteVideoPLI = nil + p.remoteVideoSSRC = 0 + p.remoteScreenPLI = nil + p.remoteScreenSSRC = 0 + p.remoteScreenActive = false + p.remoteVideoKeyframePending = [2]bool{} + p.remoteVideoKeyframeAwaited = [2]bool{} + cancel := p.remoteMediaCancel + p.remoteMediaCancel = nil + p.mu.Unlock() + if cancel != nil { + cancel() + } + for _, track := range tracks { + track.Close() + } +} + +const meowcallerSampleRate = 16000 +const liveKitH264ClockRate = 90000 +const liveKitH264MaxLatePackets = 1000 + +func videoFrameDuration(cfg VideoConfig) time.Duration { + if cfg.MaxFPS <= 0 { + return time.Second / 30 + } + return time.Second / time.Duration(cfg.MaxFPS) +} + +var _ lkpcm.PCM16Writer = (*MeowcallerAudioSource)(nil) diff --git a/pkg/connector/voip/livekit_test.go b/pkg/connector/voip/livekit_test.go new file mode 100644 index 0000000..3a425b7 --- /dev/null +++ b/pkg/connector/voip/livekit_test.go @@ -0,0 +1,30 @@ +package voip + +import ( + "testing" + + livekitproto "github.com/livekit/protocol/livekit" +) + +func TestVideoTrackPublicationOptionsPreserveScreenShareSource(t *testing.T) { + cfg := VideoConfig{MaxWidth: 1280, MaxHeight: 720} + opts := videoTrackPublicationOptions("whatsapp-screen", livekitproto.TrackSource_SCREEN_SHARE, cfg) + if opts.Name != "whatsapp-screen" { + t.Fatalf("track name = %q, want whatsapp-screen", opts.Name) + } + if opts.Source != livekitproto.TrackSource_SCREEN_SHARE { + t.Fatalf("track source = %s, want SCREEN_SHARE", opts.Source) + } + if opts.VideoWidth != 1280 || opts.VideoHeight != 720 { + t.Fatalf("track dimensions = %dx%d, want 1280x720", opts.VideoWidth, opts.VideoHeight) + } +} + +func TestLiveKitVideoSourceClassifiesScreenShareIndependently(t *testing.T) { + if liveKitVideoSourceRole(livekitproto.TrackSource_CAMERA) != liveKitVideoSourceCamera { + t.Fatal("camera publication was not classified as camera") + } + if liveKitVideoSourceRole(livekitproto.TrackSource_SCREEN_SHARE) != liveKitVideoSourceScreenShare { + t.Fatal("screen-share publication was not classified as screen share") + } +} diff --git a/pkg/connector/voip/manager.go b/pkg/connector/voip/manager.go new file mode 100644 index 0000000..a5b96fd --- /dev/null +++ b/pkg/connector/voip/manager.go @@ -0,0 +1,1054 @@ +package voip + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/purpshell/meowcaller" + "github.com/purpshell/meowcaller/diag" + "github.com/purpshell/meowcaller/signaling" + "github.com/rs/zerolog" + "go.mau.fi/whatsmeow" + waBinary "go.mau.fi/whatsmeow/binary" + "go.mau.fi/whatsmeow/types" +) + +const ( + localUnmuteState = "0" + localMuteState = "1" +) + +type matrixVideoAction uint8 + +const ( + matrixVideoNone matrixVideoAction = iota + matrixVideoStop + matrixVideoStart +) + +var localMuteRetryIntervals = []time.Duration{ + 0, + 2 * time.Second, + 2 * time.Second, + 2 * time.Second, +} + +type Manager struct { + cfg Config + waClient *whatsmeow.Client + client *meowcaller.Client + log zerolog.Logger + + mu sync.Mutex + calls map[string]*meowcaller.Call + callCreators map[string]types.JID + livekit map[string]*LiveKitParticipant + livekitConnecting map[string]struct{} + matrixAudioMuted map[string]bool + matrixVideoMuted map[string]bool + whatsAppMuted map[string]bool + whatsAppVideoMuted map[string]bool + videoKeyframePending map[string]bool + videoRouters map[string]*whatsAppVideoRouter + incomingCallNotify func(*meowcaller.Call) + callEndNotify func(callID, reason string) + callReactionNotify func(callID string, reaction meowcaller.CallReaction) + handRaiseNotify func(callID string, state meowcaller.HandRaiseState) + waitingRoomNotify func(callID string, state meowcaller.WaitingRoomState) +} + +func NewManager(waClient *whatsmeow.Client, cfg Config, log zerolog.Logger) *Manager { + manager := &Manager{ + cfg: cfg, + waClient: waClient, + log: log, + calls: make(map[string]*meowcaller.Call), + callCreators: make(map[string]types.JID), + livekit: make(map[string]*LiveKitParticipant), + livekitConnecting: make(map[string]struct{}), + matrixAudioMuted: make(map[string]bool), + matrixVideoMuted: make(map[string]bool), + whatsAppMuted: make(map[string]bool), + whatsAppVideoMuted: make(map[string]bool), + videoKeyframePending: make(map[string]bool), + videoRouters: make(map[string]*whatsAppVideoRouter), + } + if !cfg.Enabled || waClient == nil { + return manager + } + opts := []meowcaller.Option{meowcaller.WithLogger(log)} + if cfg.Diagnostics.EnableMeowcallerDiagnostics { + rec, err := diag.NewRecorder(cfg.Diagnostics.MediaTraceDir) + if err != nil { + log.Warn(). + Err(err). + Str("media_trace_dir", cfg.Diagnostics.MediaTraceDir). + Msg("Failed to enable meowcaller media diagnostics") + } else { + opts = append(opts, meowcaller.WithDiagnostics(rec)) + log.Warn(). + Str("media_trace_dir", cfg.Diagnostics.MediaTraceDir). + Msg("Enabled unsafe meowcaller media diagnostics") + } + } + manager.client = meowcaller.NewClient(waClient, opts...) + manager.client.OnIncomingCall(manager.handleIncomingCall) + return manager +} + +func (m *Manager) Enabled() bool { + return m != nil && m.cfg.Enabled && m.client != nil +} + +func (m *Manager) Client() *meowcaller.Client { + if m == nil { + return nil + } + return m.client +} + +func (m *Manager) SetIncomingCallHandler(handler func(*meowcaller.Call)) { + if m == nil { + return + } + m.mu.Lock() + m.incomingCallNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetCallEndHandler(handler func(callID, reason string)) { + if m == nil { + return + } + m.mu.Lock() + m.callEndNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetCallReactionHandler(handler func(callID string, reaction meowcaller.CallReaction)) { + if m == nil { + return + } + m.mu.Lock() + m.callReactionNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetHandRaiseHandler(handler func(callID string, state meowcaller.HandRaiseState)) { + if m == nil { + return + } + m.mu.Lock() + m.handRaiseNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SetWaitingRoomHandler(handler func(callID string, state meowcaller.WaitingRoomState)) { + if m == nil { + return + } + m.mu.Lock() + m.waitingRoomNotify = handler + m.mu.Unlock() +} + +func (m *Manager) SendReaction(callID, emoji string) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return ErrCallNotFound + } + normalized, ok := NormalizeWhatsAppCallReaction(emoji) + if !ok { + return fmt.Errorf("unsupported WhatsApp call reaction %q", emoji) + } + return call.SendReaction(normalized) +} + +func (m *Manager) SetHandRaised(callID string, raised bool) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return ErrCallNotFound + } + return call.SetHandRaised(raised) +} + +func (m *Manager) GroupState(callID string) (meowcaller.GroupCallState, bool, error) { + call, err := m.activeCall(callID) + if err != nil { + return meowcaller.GroupCallState{}, false, err + } + state, ok := call.GroupState() + return state, ok, nil +} + +func (m *Manager) AddParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.AddParticipant(ctx, target) +} + +func (m *Manager) RingParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.RingParticipant(ctx, target) +} + +func (m *Manager) SelectVideoParticipant(callID, target string) error { + if _, err := m.activeCall(callID); err != nil { + return err + } + jid, err := parseVideoParticipantTarget(target) + if err != nil { + return err + } + m.mu.Lock() + router := m.videoRouters[callID] + m.mu.Unlock() + if router == nil { + return fmt.Errorf("WhatsApp group video is not connected to LiveKit") + } + return router.SelectCamera(jid) +} + +func (m *Manager) CreateCallLink(ctx context.Context, video bool) (meowcaller.CallLink, error) { + if !m.Enabled() { + return meowcaller.CallLink{}, ErrNotEnabled + } + return m.client.CreateCallLink(ctx, meowcaller.CallLinkOptions{Video: video}) +} + +func (m *Manager) PreviewCallLink(ctx context.Context, tokenOrURL string, video bool) (meowcaller.CallLinkPreview, error) { + if !m.Enabled() { + return meowcaller.CallLinkPreview{}, ErrNotEnabled + } + return m.client.PreviewCallLink(ctx, tokenOrURL, meowcaller.CallLinkOptions{Video: video}) +} + +func (m *Manager) JoinCallLink(ctx context.Context, tokenOrURL string, video bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + call, err := m.client.JoinCallLink(ctx, tokenOrURL, meowcaller.CallLinkOptions{Video: video}) + if err != nil { + return nil, err + } + if call == nil { + return nil, fmt.Errorf("meowcaller returned no call for call-link join") + } + m.trackCall(call, call.Peer()) + return call, nil +} + +func (m *Manager) WaitingRoomState(callID string) (meowcaller.WaitingRoomState, bool, error) { + call, err := m.activeCall(callID) + if err != nil { + return meowcaller.WaitingRoomState{}, false, err + } + state, ok := call.WaitingRoomState() + return state, ok, nil +} + +func (m *Manager) SetApprovalRequired(ctx context.Context, callID string, enabled bool) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.SetApprovalRequired(ctx, enabled) +} + +func (m *Manager) AdmitParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.AdmitParticipant(ctx, target) +} + +func (m *Manager) DenyParticipant(ctx context.Context, callID, target string) error { + call, err := m.activeCall(callID) + if err != nil { + return err + } + return call.DenyParticipant(ctx, target) +} + +func (m *Manager) activeCall(callID string) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + m.mu.Lock() + call := m.calls[callID] + m.mu.Unlock() + if call == nil || call.State() == meowcaller.CallPhaseEnded { + return nil, ErrCallNotFound + } + return call, nil +} + +func (m *Manager) Dial(ctx context.Context, target string, video ...bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + opts := meowcaller.CallOptions{} + if len(video) > 0 { + opts.Video = video[0] + } + call, err := m.client.CallWithOptions(ctx, target, opts) + if err != nil { + return nil, err + } + m.trackCall(call, m.ownCallCreator()) + return call, nil +} + +func (m *Manager) DialGroupByID(ctx context.Context, groupID string, video ...bool) (*meowcaller.Call, error) { + if !m.Enabled() { + return nil, ErrNotEnabled + } + opts := meowcaller.GroupCallOptions{} + if len(video) > 0 { + opts.Video = video[0] + } + call, err := m.client.GroupCallByIDWithOptions(ctx, groupID, opts) + if err != nil { + return nil, err + } + m.trackCall(call, m.ownCallCreator()) + return call, nil +} + +func (m *Manager) AbortAll() { + if m == nil { + return + } + m.mu.Lock() + calls := make([]*meowcaller.Call, 0, len(m.calls)) + for _, call := range m.calls { + calls = append(calls, call) + } + m.calls = make(map[string]*meowcaller.Call) + m.callCreators = make(map[string]types.JID) + m.matrixAudioMuted = make(map[string]bool) + m.matrixVideoMuted = make(map[string]bool) + m.whatsAppMuted = make(map[string]bool) + m.whatsAppVideoMuted = make(map[string]bool) + m.videoKeyframePending = make(map[string]bool) + m.videoRouters = make(map[string]*whatsAppVideoRouter) + participants := make([]*LiveKitParticipant, 0, len(m.livekit)) + for _, participant := range m.livekit { + participants = append(participants, participant) + } + m.livekit = make(map[string]*LiveKitParticipant) + m.livekitConnecting = make(map[string]struct{}) + m.mu.Unlock() + for _, participant := range participants { + participant.Close() + } + for _, call := range calls { + if err := call.Hangup(); err != nil { + m.log.Debug().Err(err).Str("call_id", call.ID()).Msg("Failed to hang up VOIP call during abort") + } + } +} + +func (m *Manager) BridgeCallToLiveKit(ctx context.Context, waCallID string, authResp *LiveKitAuthResponse, selectedRemoteParticipantID string) error { + if !m.Enabled() { + return ErrNotEnabled + } + m.mu.Lock() + call := m.calls[waCallID] + existing := m.livekit[waCallID] + _, connecting := m.livekitConnecting[waCallID] + if call != nil && existing == nil && !connecting { + m.livekitConnecting[waCallID] = struct{}{} + } + m.mu.Unlock() + if call == nil { + return ErrCallNotFound + } + if existing != nil || connecting { + return nil + } + participant, err := ConnectLiveKitParticipant(ctx, authResp, m.cfg.LiveKit, m.cfg.Video, m.log.With().Str("call_id", waCallID).Str("component", "livekit").Logger()) + if err != nil { + m.clearLiveKitConnecting(waCallID) + return err + } + participant.SetRemoteAudioMuteHandler(selectedRemoteParticipantID, func(muted bool) { + m.handleMatrixAudioMuteState(call, muted) + }) + videoEnabled := m.cfg.Video.Enabled + var videoRouter *whatsAppVideoRouter + if videoEnabled { + var videoBuffer whatsAppVideoStartupBuffer + var videoBufferLock sync.Mutex + sendMatrixVideoFrame := func(frame LiveKitVideoFrame) error { + if call.State() == meowcaller.CallPhaseEnded { + return nil + } + videoBufferLock.Lock() + bufferedBefore := videoBuffer.Len() + flushed, err := videoBuffer.Send(frame, func(frame LiveKitVideoFrame) error { + return call.SendVideoWithDuration(frame.AccessUnit, frame.Duration) + }) + bufferedAfter := videoBuffer.Len() + videoBufferLock.Unlock() + if err != nil { + m.log.Debug(). + Err(err). + Str("call_id", call.ID()). + Int("buffered_frames", bufferedAfter). + Msg("Buffered LiveKit H.264 frame until WhatsApp video media is ready") + return nil + } + if bufferedBefore > 0 && bufferedAfter == 0 { + m.log.Info(). + Str("call_id", call.ID()). + Int("flushed_frames", flushed). + Msg("Flushed buffered LiveKit H.264 video to WhatsApp") + } + return nil + } + sourceRouter := newMatrixVideoSourceRouter(sendMatrixVideoFrame) + participant.SetRemoteVideoHandlers(selectedRemoteParticipantID, sourceRouter.WriteCamera, func(muted bool) { + m.handleMatrixCameraMuteState(call, sourceRouter, muted) + }) + participant.SetRemoteScreenShareHandlers(sourceRouter.WriteScreen, func(muted bool) { + m.handleMatrixScreenShareMuteState(call, participant, sourceRouter, muted) + }) + } + if err = participant.PublishAudioTrack("whatsapp-audio"); err != nil { + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + call.Receive(participant.WhatsAppSink()) + call.Play(participant.MatrixAudioSource()) + if videoEnabled { + if err = participant.PublishVideoTrack("whatsapp-video"); err != nil { + call.Receive(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + if err = participant.PublishScreenShareTrack("whatsapp-screen"); err != nil { + call.Receive(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + videoRouter = newWhatsAppVideoRouter( + participant.WhatsAppVideoSink(), + participant.WhatsAppScreenShareSink(), + participant.SetWhatsAppVideoMuted, + participant.SetWhatsAppScreenShareMuted, + ) + call.ReceiveVideo(nil) + call.OnParticipantVideoFrame(videoRouter.WriteParticipantFrame) + call.OnGroupState(videoRouter.SetGroupState) + call.OnScreenShare(videoRouter.SetScreenShare) + for _, state := range call.ScreenShares() { + videoRouter.SetScreenShare(state) + } + } + answeredIncoming := call.State() == meowcaller.CallPhaseRinging + if call.State() == meowcaller.CallPhaseRinging { + if err = call.Answer(); err != nil { + call.Receive(nil) + call.ReceiveVideo(nil) + call.Subscribe(nil) + participant.Close() + m.clearLiveKitConnecting(waCallID) + return err + } + } + m.mu.Lock() + if m.calls[waCallID] != call || call.State() == meowcaller.CallPhaseEnded { + delete(m.livekitConnecting, waCallID) + delete(m.videoKeyframePending, waCallID) + m.mu.Unlock() + call.Receive(nil) + call.ReceiveVideo(nil) + call.Subscribe(nil) + participant.Close() + return ErrCallNotFound + } + whatsAppMuted, knownWhatsAppMute := m.whatsAppMuted[waCallID] + whatsAppVideoMuted, knownWhatsAppVideoMute := m.whatsAppVideoMuted[waCallID] + keyframePending := m.videoKeyframePending[waCallID] + delete(m.videoKeyframePending, waCallID) + delete(m.livekitConnecting, waCallID) + m.livekit[waCallID] = participant + if videoRouter != nil { + m.videoRouters[waCallID] = videoRouter + } + m.mu.Unlock() + if videoEnabled && keyframePending { + participant.requestRemoteVideoKeyframe() + } + if knownWhatsAppMute { + participant.SetWhatsAppAudioMuted(whatsAppMuted) + } + if videoEnabled { + if !knownWhatsAppVideoMute { + whatsAppVideoMuted = !call.IsReceivingVideo() + } + participant.SetWhatsAppVideoMuted(whatsAppVideoMuted) + } + if answeredIncoming { + m.log.Debug().Str("call_id", waCallID).Msg("Answered incoming WhatsApp call before sending local unmute") + } + go m.sendLocalMuteStateRetries(call) + return nil +} + +func (m *Manager) sendLocalMuteStateRetries(call *meowcaller.Call) { + if m == nil || m.waClient == nil || call == nil { + return + } + for attempt, interval := range localMuteRetryIntervals { + time.Sleep(interval) + if call.State() == meowcaller.CallPhaseEnded { + return + } + muted := m.currentMatrixAudioMuted(call.ID()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalMuteState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localMuteStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Failed to send WhatsApp local mute state") + continue + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Int("attempt", attempt+1). + Msg("Sent WhatsApp local mute state") + } +} + +func (m *Manager) handleMatrixAudioMuteState(call *meowcaller.Call, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + m.mu.Lock() + previous, known := m.matrixAudioMuted[call.ID()] + m.matrixAudioMuted[call.ID()] = muted + m.mu.Unlock() + if known && previous == muted { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := m.sendLocalMuteState(ctx, call.ID(), call.Peer(), m.callCreatorFor(call), localMuteStateFor(muted)) + cancel() + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Failed to send WhatsApp local mute state from LiveKit") + return + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Sent WhatsApp local mute state from LiveKit") +} + +func (m *Manager) handleMatrixVideoMuteState(call *meowcaller.Call, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + m.mu.Lock() + previous, known := m.matrixVideoMuted[call.ID()] + m.matrixVideoMuted[call.ID()] = muted + m.mu.Unlock() + if known && previous == muted { + return + } + err := applyMatrixVideoState(call, muted) + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Failed to send WhatsApp local video state from LiveKit") + return + } + m.log.Debug(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("muted", muted). + Msg("Sent WhatsApp local video state from LiveKit") +} + +func (m *Manager) handleMatrixCameraMuteState(call *meowcaller.Call, sourceRouter *matrixVideoSourceRouter, muted bool) { + if m == nil || call == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + if sourceRouter != nil && sourceRouter.ScreenSharing() { + m.mu.Lock() + m.matrixVideoMuted[call.ID()] = muted + m.mu.Unlock() + return + } + m.handleMatrixVideoMuteState(call, muted) +} + +func (m *Manager) handleMatrixScreenShareMuteState( + call *meowcaller.Call, + participant *LiveKitParticipant, + sourceRouter *matrixVideoSourceRouter, + muted bool, +) { + if m == nil || call == nil || sourceRouter == nil || call.State() == meowcaller.CallPhaseEnded { + return + } + active := !muted + if sourceRouter.ScreenSharing() == active { + return + } + sourceRouter.SetScreenSharing(active) + + var err error + startedVideo := false + if active { + if !call.IsSendingVideo() { + err = call.StartVideo() + startedVideo = err == nil + } + if err == nil { + err = call.StartScreenShare(nil) + if err != nil && startedVideo { + _ = call.StopVideo() + } + } + } else { + err = call.StopScreenShare() + if err == nil && m.currentMatrixVideoMuted(call.ID()) && call.IsSendingVideo() { + err = call.StopVideo() + } + } + if err != nil { + sourceRouter.SetScreenSharing(!active) + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Bool("active", active). + Msg("Failed to update WhatsApp screen-share state from LiveKit") + return + } + if participant != nil { + participant.requestRemoteVideoKeyframe() + } + m.log.Info(). + Str("call_id", call.ID()). + Bool("active", active). + Msg("Updated WhatsApp screen-share state from LiveKit") +} + +func (m *Manager) handleWhatsAppAudioMuteState(callID string, muted bool) { + if m == nil || callID == "" { + return + } + m.mu.Lock() + m.whatsAppMuted[callID] = muted + participant := m.livekit[callID] + m.mu.Unlock() + if participant != nil { + participant.SetWhatsAppAudioMuted(muted) + } + m.log.Debug(). + Str("call_id", callID). + Bool("muted", muted). + Msg("Observed WhatsApp remote audio mute state") +} + +func (m *Manager) handleWhatsAppVideoState(callID string, state meowcaller.VideoState) { + if m == nil || callID == "" { + return + } + muted, muteChanged := remoteVideoMuteForState(state) + m.mu.Lock() + if muteChanged { + m.whatsAppVideoMuted[callID] = muted + } + call := m.calls[callID] + participant := m.livekit[callID] + m.mu.Unlock() + if state.Upgrade && call != nil { + if err := call.AcceptVideo(); err != nil { + m.log.Warn().Err(err).Str("call_id", callID).Int("raw_state", state.Raw).Msg("Failed to accept WhatsApp peer video upgrade") + } else { + m.log.Info().Str("call_id", callID).Int("raw_state", state.Raw).Msg("Accepted WhatsApp peer video upgrade") + } + } + if participant != nil { + if muteChanged { + participant.SetWhatsAppVideoMuted(muted) + } + participant.SetWhatsAppVideoOrientation(state.Orientation) + } + m.log.Debug(). + Str("call_id", callID). + Bool("muted", muted). + Bool("mute_changed", muteChanged). + Bool("active", state.Active). + Bool("upgrade", state.Upgrade). + Int("orientation", state.Orientation). + Int("raw_state", state.Raw). + Msg("Observed WhatsApp remote video state") +} + +func remoteVideoMuteForState(state meowcaller.VideoState) (muted, changed bool) { + switch state.Raw { + case signaling.VideoStateEnabled: + return false, true + case signaling.VideoStateDisabled, signaling.VideoStateStopped: + return true, true + default: + return false, false + } +} + +func (m *Manager) currentMatrixAudioMuted(callID string) bool { + if m == nil { + return false + } + m.mu.Lock() + muted := m.matrixAudioMuted[callID] + m.mu.Unlock() + return muted +} + +func (m *Manager) currentMatrixVideoMuted(callID string) bool { + if m == nil { + return false + } + m.mu.Lock() + muted := m.matrixVideoMuted[callID] + m.mu.Unlock() + return muted +} + +func localMuteStateFor(muted bool) string { + if muted { + return localMuteState + } + return localUnmuteState +} + +func matrixVideoActionFor(muted, sending, _ bool) matrixVideoAction { + if muted { + if sending { + return matrixVideoStop + } + return matrixVideoNone + } + if sending { + return matrixVideoNone + } + return matrixVideoStart +} + +func applyMatrixVideoState(call *meowcaller.Call, muted bool) error { + switch matrixVideoActionFor(muted, call.IsSendingVideo(), call.IsReceivingVideo()) { + case matrixVideoStop: + return call.StopVideo() + case matrixVideoStart: + return call.StartVideo() + default: + return nil + } +} + +func (m *Manager) sendLocalMuteState(ctx context.Context, callID string, peer, callCreator types.JID, muteState string) error { + if m == nil || m.waClient == nil { + return fmt.Errorf("whatsapp client is not available") + } + if callID == "" { + return fmt.Errorf("call ID is empty") + } + if peer.IsEmpty() { + return fmt.Errorf("peer JID is empty") + } + if callCreator.IsEmpty() { + return fmt.Errorf("call creator JID is empty") + } + node := buildLocalMuteV2Node(callID, peer, callCreator, string(m.waClient.GenerateMessageID()), muteState) + //lint:ignore SA1019 low-level call signaling is not exposed by whatsmeow's public API + if err := m.waClient.DangerousInternals().SendNode(ctx, node); err != nil { + return fmt.Errorf("send mute_v2: %w", err) + } + return nil +} + +func buildLocalMuteV2Node(callID string, peer, callCreator types.JID, wrapperID, muteState string) waBinary.Node { + node := signaling.BuildMuteV2(callID, peer, callCreator, muteState) + if wrapperID != "" { + node.Attrs["id"] = wrapperID + } + return node +} + +func (m *Manager) HandleMatrixRTCEvent(ctx context.Context, evt MatrixRTCEvent) int { + return m.HandleMatrixRTCCallEvent(ctx, evt, "") +} + +func (m *Manager) HandleMatrixRTCCallEvent(_ context.Context, evt MatrixRTCEvent, waCallID string) int { + if !m.Enabled() { + return 0 + } + switch evt.Kind { + case MatrixRTCEventKindRTCDecline: + ended := m.endCallsFromMatrixRTC(waCallID) + if ended == 0 { + m.log.Debug(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Msg("Received MatrixRTC decline with no active WhatsApp VOIP calls") + } else { + m.log.Info(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Int("ended_call_count", ended). + Msg("Ended WhatsApp VOIP calls after MatrixRTC decline") + } + return ended + case MatrixRTCEventKindRTCMembership, MatrixRTCEventKindGroupCallMember, MatrixRTCEventKindRTCNotification, MatrixRTCEventKindLegacyCallNotify, MatrixRTCEventKindGroupCall: + m.log.Debug(). + Stringer("matrix_room_id", evt.RoomID). + Stringer("matrix_sender", evt.Sender). + Str("matrix_call_id", evt.CallID). + Str("wa_call_id", waCallID). + Str("matrixrtc_kind", string(evt.Kind)). + Msg("Observed MatrixRTC event") + } + return 0 +} + +func (m *Manager) handleIncomingCall(call *meowcaller.Call) { + m.trackCall(call, call.Peer()) + m.log.Info(). + Str("call_id", call.ID()). + Stringer("peer_jid", call.Peer()). + Bool("video", call.IsVideo()). + Msg("Received incoming WhatsApp call for MatrixRTC bridge") + if m.cfg.IncomingPolicy == "notice" { + if err := call.Reject(); err != nil { + m.log.Warn().Err(err).Str("call_id", call.ID()).Msg("Failed to reject VOIP call handled as notice") + } + return + } + m.mu.Lock() + handler := m.incomingCallNotify + m.mu.Unlock() + if handler != nil { + go handler(call) + } +} + +func (m *Manager) trackCall(call *meowcaller.Call, callCreator types.JID) { + if call == nil { + return + } + if callCreator.IsEmpty() { + callCreator = call.Peer() + } + m.mu.Lock() + m.calls[call.ID()] = call + m.callCreators[call.ID()] = callCreator + m.mu.Unlock() + call.OnEnd(func(reason string) { + m.mu.Lock() + delete(m.calls, call.ID()) + delete(m.callCreators, call.ID()) + delete(m.matrixAudioMuted, call.ID()) + delete(m.matrixVideoMuted, call.ID()) + delete(m.whatsAppMuted, call.ID()) + delete(m.whatsAppVideoMuted, call.ID()) + delete(m.videoKeyframePending, call.ID()) + delete(m.videoRouters, call.ID()) + participant := m.livekit[call.ID()] + delete(m.livekit, call.ID()) + delete(m.livekitConnecting, call.ID()) + handler := m.callEndNotify + m.mu.Unlock() + if participant != nil { + participant.Close() + } + m.log.Info().Str("call_id", call.ID()).Str("reason", reason).Msg("WhatsApp VOIP call ended") + if handler != nil { + go handler(call.ID(), reason) + } + }) + call.OnStateChange(func(phase meowcaller.CallPhase) { + m.log.Debug().Str("call_id", call.ID()).Int("phase", int(phase)).Msg("WhatsApp VOIP call state changed") + }) + call.OnPeerAccept(func() { + if call.IsVideo() { + m.requestLiveKitVideoKeyframe(call.ID()) + } + }) + call.OnVideoKeyframeRequest(func() { + m.requestLiveKitVideoKeyframe(call.ID()) + }) + call.OnMuteState(func(muted bool) { + m.handleWhatsAppAudioMuteState(call.ID(), muted) + }) + call.OnVideoState(func(state meowcaller.VideoState) { + m.handleWhatsAppVideoState(call.ID(), state) + }) + call.OnReaction(func(reaction meowcaller.CallReaction) { + m.mu.Lock() + handler := m.callReactionNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), reaction) + } + }) + call.OnHandRaise(func(state meowcaller.HandRaiseState) { + m.mu.Lock() + handler := m.handRaiseNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), state) + } + }) + call.OnWaitingRoomState(func(state meowcaller.WaitingRoomState) { + m.mu.Lock() + handler := m.waitingRoomNotify + m.mu.Unlock() + if handler != nil { + handler(call.ID(), state) + } + }) +} + +func parseVideoParticipantTarget(target string) (types.JID, error) { + target = strings.TrimSpace(target) + if target == "" { + return types.EmptyJID, fmt.Errorf("WhatsApp video participant is empty") + } + if !strings.ContainsRune(target, '@') { + return types.NewJID(strings.TrimPrefix(target, "+"), types.DefaultUserServer), nil + } + jid, err := types.ParseJID(target) + if err != nil { + return types.EmptyJID, fmt.Errorf("parse WhatsApp video participant: %w", err) + } + return jid.ToNonAD(), nil +} + +func (m *Manager) requestLiveKitVideoKeyframe(callID string) { + m.mu.Lock() + call := m.calls[callID] + participant := m.livekit[callID] + if call != nil && participant == nil { + m.videoKeyframePending[callID] = true + } + m.mu.Unlock() + if call != nil && participant != nil { + participant.requestRemoteVideoKeyframe() + } +} + +func (m *Manager) callCreatorFor(call *meowcaller.Call) types.JID { + if m == nil || call == nil { + return types.EmptyJID + } + m.mu.Lock() + callCreator := m.callCreators[call.ID()] + m.mu.Unlock() + if !callCreator.IsEmpty() { + return callCreator + } + if call.State() == meowcaller.CallPhaseCalling { + return m.ownCallCreator() + } + return call.Peer() +} + +func (m *Manager) ownCallCreator() types.JID { + if m == nil || m.waClient == nil || m.waClient.Store == nil { + return types.EmptyJID + } + return m.waClient.Store.GetLID() +} + +func (m *Manager) endCallsFromMatrixRTC(waCallID string) int { + m.mu.Lock() + calls := make([]*meowcaller.Call, 0, len(m.calls)) + if waCallID != "" { + if call := m.calls[waCallID]; call != nil { + calls = append(calls, call) + } + } else { + for _, call := range m.calls { + calls = append(calls, call) + } + } + m.mu.Unlock() + + var ended int + for _, call := range calls { + if call.State() == meowcaller.CallPhaseEnded { + continue + } + var err error + if call.State() == meowcaller.CallPhaseRinging { + err = call.Reject() + } else { + err = call.Hangup() + } + if err != nil { + m.log.Warn(). + Err(err). + Str("call_id", call.ID()). + Int("phase", int(call.State())). + Msg("Failed to end WhatsApp VOIP call after MatrixRTC event") + continue + } + ended++ + } + return ended +} + +func (m *Manager) clearLiveKitConnecting(waCallID string) { + m.mu.Lock() + delete(m.livekitConnecting, waCallID) + m.mu.Unlock() +} diff --git a/pkg/connector/voip/manager_test.go b/pkg/connector/voip/manager_test.go new file mode 100644 index 0000000..2f5e6be --- /dev/null +++ b/pkg/connector/voip/manager_test.go @@ -0,0 +1,156 @@ +package voip + +import ( + "errors" + "testing" + "time" + + "github.com/purpshell/meowcaller" + "github.com/purpshell/meowcaller/signaling" + "go.mau.fi/whatsmeow/types" +) + +func TestBuildLocalMuteV2Node(t *testing.T) { + peer := types.NewJID("12345", types.HiddenUserServer) + callCreator := types.NewJID("67890", types.HiddenUserServer) + node := buildLocalMuteV2Node("call-id", peer, callCreator, "wrapper-id", localUnmuteState) + if node.Tag != "call" { + t.Fatalf("node tag = %q, want call", node.Tag) + } + if got := node.AttrGetter().JID("to"); got != peer { + t.Fatalf("to = %s, want %s", got, peer) + } + if got := node.AttrGetter().String("id"); got != "wrapper-id" { + t.Fatalf("wrapper id = %q, want wrapper-id", got) + } + children := node.GetChildren() + if len(children) != 1 { + t.Fatalf("children = %d, want 1", len(children)) + } + mute := children[0] + if mute.Tag != "mute_v2" { + t.Fatalf("child tag = %q, want mute_v2", mute.Tag) + } + attrs := mute.AttrGetter() + if got := attrs.String("call-id"); got != "call-id" { + t.Fatalf("call-id = %q, want call-id", got) + } + if got := attrs.JID("call-creator"); got != callCreator { + t.Fatalf("call-creator = %s, want %s", got, callCreator) + } + if got := attrs.String("mute-state"); got != localUnmuteState { + t.Fatalf("mute-state = %q, want %q", got, localUnmuteState) + } +} + +func TestLocalMuteStateFor(t *testing.T) { + if got := localMuteStateFor(false); got != localUnmuteState { + t.Fatalf("unmuted state = %q, want %q", got, localUnmuteState) + } + if got := localMuteStateFor(true); got != localMuteState { + t.Fatalf("muted state = %q, want %q", got, localMuteState) + } +} + +func TestMatrixVideoActionFor(t *testing.T) { + tests := []struct { + muted, sending, receiving bool + want matrixVideoAction + }{ + {muted: true, sending: true, receiving: true, want: matrixVideoStop}, + {muted: true, sending: false, receiving: true, want: matrixVideoNone}, + {muted: false, sending: true, receiving: false, want: matrixVideoNone}, + {muted: false, sending: true, receiving: true, want: matrixVideoNone}, + {muted: false, sending: false, receiving: true, want: matrixVideoStart}, + {muted: false, sending: false, receiving: false, want: matrixVideoStart}, + } + for _, tc := range tests { + if got := matrixVideoActionFor(tc.muted, tc.sending, tc.receiving); got != tc.want { + t.Errorf("muted:%v sending:%v receiving:%v => %d, want %d", + tc.muted, tc.sending, tc.receiving, got, tc.want) + } + } +} + +func TestRemoteVideoMuteForStateOnlyChangesPeerOwnedFlow(t *testing.T) { + tests := []struct { + state int + muted bool + changed bool + }{ + {signaling.VideoStateEnabled, false, true}, + {signaling.VideoStateDisabled, true, true}, + {signaling.VideoStateStopped, true, true}, + {signaling.VideoStateUpgradeRequestV2, false, false}, + {signaling.VideoStateUpgradeAccept, false, false}, + {signaling.VideoStateUpgradeReject, false, false}, + {signaling.VideoStateUpgradeCancel, false, false}, + } + for _, tc := range tests { + muted, changed := remoteVideoMuteForState(meowcaller.VideoState{Raw: tc.state}) + if muted != tc.muted || changed != tc.changed { + t.Errorf("state %d => muted:%v changed:%v, want muted:%v changed:%v", + tc.state, muted, changed, tc.muted, tc.changed) + } + } +} + +func TestWhatsAppVideoStartupBufferRetriesEarlyFrames(t *testing.T) { + notReady := errors.New("meowcaller: call has no active video media") + var attempts int + var sent [][]byte + var durations []time.Duration + buffer := whatsAppVideoStartupBuffer{} + + flushed, err := buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{1}, Duration: 33 * time.Millisecond}, func(frame LiveKitVideoFrame) error { + attempts++ + return notReady + }) + if err != notReady { + t.Fatalf("first send error = %v, want notReady", err) + } + if flushed != 0 || buffer.Len() != 1 { + t.Fatalf("flushed=%d buffered=%d, want flushed=0 buffered=1", flushed, buffer.Len()) + } + + flushed, err = buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{2}, Duration: 17 * time.Millisecond}, func(frame LiveKitVideoFrame) error { + attempts++ + sent = append(sent, append([]byte(nil), frame.AccessUnit...)) + durations = append(durations, frame.Duration) + return nil + }) + if err != nil { + t.Fatalf("second send returned error: %v", err) + } + if flushed != 2 || buffer.Len() != 0 { + t.Fatalf("flushed=%d buffered=%d, want flushed=2 buffered=0", flushed, buffer.Len()) + } + if attempts != 3 { + t.Fatalf("attempts=%d, want 3", attempts) + } + if len(sent) != 2 || sent[0][0] != 1 || sent[1][0] != 2 { + t.Fatalf("sent frames = %#v, want [1] then [2]", sent) + } + if len(durations) != 2 || durations[0] != 33*time.Millisecond || durations[1] != 17*time.Millisecond { + t.Fatalf("sent durations = %v, want 33ms then 17ms", durations) + } + if got := buffer.ready; !got { + t.Fatalf("buffer ready = %v, want true", got) + } +} + +func TestWhatsAppVideoStartupBufferCapsPendingFrames(t *testing.T) { + notReady := errors.New("not ready") + buffer := whatsAppVideoStartupBuffer{} + for i := 0; i < maxPendingWhatsAppVideoFrames+3; i++ { + _, _ = buffer.Send(LiveKitVideoFrame{AccessUnit: []byte{byte(i)}}, func(LiveKitVideoFrame) error { + return notReady + }) + } + if buffer.Len() != maxPendingWhatsAppVideoFrames { + t.Fatalf("buffered frames = %d, want %d", buffer.Len(), maxPendingWhatsAppVideoFrames) + } + if got := buffer.frames[0].AccessUnit[0]; got != 3 { + t.Fatalf("oldest retained frame = %d, want 3", got) + } +} diff --git a/pkg/connector/voip/matrixrtc.go b/pkg/connector/voip/matrixrtc.go new file mode 100644 index 0000000..1f6b154 --- /dev/null +++ b/pkg/connector/voip/matrixrtc.go @@ -0,0 +1,555 @@ +package voip + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "time" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +const ( + EventTypeGroupCall = "org.matrix.msc3401.call" + EventTypeGroupCallMember = "org.matrix.msc3401.call.member" + EventTypeRTCMembership = "org.matrix.msc4143.rtc.member" + EventTypeRTCNotification = "org.matrix.msc4075.rtc.notification" + EventTypeCallNotify = "org.matrix.msc4075.call.notify" + EventTypeRTCDecline = "org.matrix.msc4310.rtc.decline" + EventTypeElementCallReaction = "io.element.call.reaction" + + MatrixRTCApplicationCall = "m.call" + MatrixRTCDefaultSlotID = "m.call#ROOM" + MatrixRTCMembershipV1 = "org.matrix.msc4143.rtc.member" +) + +var supportedMatrixRTCEventTypes = []event.Type{ + {Type: EventTypeGroupCall, Class: event.StateEventType}, + {Type: EventTypeGroupCallMember, Class: event.StateEventType}, + {Type: EventTypeRTCMembership, Class: event.StateEventType}, + {Type: EventTypeRTCMembership, Class: event.MessageEventType}, + {Type: EventTypeRTCNotification, Class: event.MessageEventType}, + {Type: EventTypeCallNotify, Class: event.MessageEventType}, + {Type: EventTypeRTCDecline, Class: event.MessageEventType}, + ElementCallReactionEventType(), + event.EventReaction, + event.EventRedaction, +} + +type MatrixRTCEventKind string + +const ( + MatrixRTCEventKindUnknown MatrixRTCEventKind = "" + MatrixRTCEventKindGroupCall MatrixRTCEventKind = "group_call" + MatrixRTCEventKindGroupCallMember MatrixRTCEventKind = "group_call_member" + MatrixRTCEventKindRTCMembership MatrixRTCEventKind = "rtc_membership" + MatrixRTCEventKindRTCNotification MatrixRTCEventKind = "rtc_notification" + MatrixRTCEventKindLegacyCallNotify MatrixRTCEventKind = "legacy_call_notify" + MatrixRTCEventKindRTCDecline MatrixRTCEventKind = "rtc_decline" + MatrixRTCEventKindCallReaction MatrixRTCEventKind = "call_reaction" + MatrixRTCEventKindHandRaise MatrixRTCEventKind = "hand_raise" + MatrixRTCEventKindRedaction MatrixRTCEventKind = "redaction" +) + +type MatrixRTCEvent struct { + Type event.Type + Kind MatrixRTCEventKind + RoomID id.RoomID + Sender id.UserID + StateKey string + CallID string + DeviceID string + SessionID string + Intent string + LifetimeMS int + FociPreferred []Focus + Raw map[string]any + EventID id.EventID + RelatesToEventID id.EventID + RelationType event.RelationType + RelationKey string + ReactionEmoji string + ReactionName string + Redacts id.EventID +} + +type MatrixRTCSession struct { + UserID id.UserID + DeviceID string + MemberID string + CallID string + Intent string + Focus Focus + Created time.Time + Expires time.Duration + StickyKey string + NotificationEventID id.EventID + MembershipEventID id.EventID +} + +func SupportedMatrixRTCEventTypes() []event.Type { + return append([]event.Type(nil), supportedMatrixRTCEventTypes...) +} + +func ClassifyMatrixRTCEventType(evtType event.Type) MatrixRTCEventKind { + switch evtType.Type { + case EventTypeGroupCall: + return MatrixRTCEventKindGroupCall + case EventTypeGroupCallMember: + return MatrixRTCEventKindGroupCallMember + case EventTypeRTCMembership: + return MatrixRTCEventKindRTCMembership + case EventTypeRTCNotification: + return MatrixRTCEventKindRTCNotification + case EventTypeCallNotify: + return MatrixRTCEventKindLegacyCallNotify + case EventTypeRTCDecline: + return MatrixRTCEventKindRTCDecline + case EventTypeElementCallReaction: + return MatrixRTCEventKindCallReaction + case event.EventReaction.Type: + return MatrixRTCEventKindHandRaise + case event.EventRedaction.Type: + return MatrixRTCEventKindRedaction + default: + return MatrixRTCEventKindUnknown + } +} + +func ParseMatrixRTCEvent(evt *event.Event) (MatrixRTCEvent, bool) { + if evt == nil { + return MatrixRTCEvent{}, false + } + kind := ClassifyMatrixRTCEventType(evt.Type) + if kind == MatrixRTCEventKindUnknown { + return MatrixRTCEvent{}, false + } + raw := rawMatrixRTCContent(evt) + parsed := MatrixRTCEvent{ + Type: evt.Type, + Kind: kind, + RoomID: evt.RoomID, + Sender: evt.Sender, + EventID: evt.ID, + Raw: raw, + } + if evt.StateKey != nil { + parsed.StateKey = *evt.StateKey + } + fillMatrixRTCFields(&parsed, raw) + if evt.Redacts != "" { + parsed.Redacts = evt.Redacts + } + return parsed, true +} + +func rawMatrixRTCContent(evt *event.Event) map[string]any { + if evt.Content.Raw != nil { + return evt.Content.Raw + } + if len(evt.Content.VeryRaw) > 0 { + var raw map[string]any + if err := json.Unmarshal(evt.Content.VeryRaw, &raw); err == nil && raw != nil { + return raw + } + } + if evt.Content.Parsed != nil { + data, err := json.Marshal(evt.Content.Parsed) + if err == nil { + var raw map[string]any + if err = json.Unmarshal(data, &raw); err == nil && raw != nil { + return raw + } + } + } + return map[string]any{} +} + +func fillMatrixRTCFields(parsed *MatrixRTCEvent, raw map[string]any) { + parsed.CallID = firstString(raw, "call_id", "m.call_id", "callId", "callID") + parsed.DeviceID = firstString(raw, "device_id", "m.device_id", "deviceId", "deviceID") + parsed.SessionID = firstString(raw, "session_id", "m.session_id", "sessionId", "sessionID") + parsed.Intent = firstString(raw, "intent", "m.call.intent", "call_intent") + parsed.LifetimeMS = firstInt(raw, "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms") + if relatesTo, ok := raw["m.relates_to"].(map[string]any); ok { + parsed.RelatesToEventID = id.EventID(firstString(relatesTo, "event_id")) + parsed.RelationType = event.RelationType(firstString(relatesTo, "rel_type")) + parsed.RelationKey = firstString(relatesTo, "key") + } + parsed.ReactionEmoji = firstString(raw, "emoji") + parsed.ReactionName = firstString(raw, "name") + if parsed.Redacts == "" { + parsed.Redacts = id.EventID(firstString(raw, "redacts")) + } + forEachObject(raw["application"], func(application map[string]any) { + if parsed.Intent == "" { + parsed.Intent = firstString(application, "intent", "m.call.intent", "call_intent") + } + }) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["rtc_transports"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["foci_preferred"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(raw["m.foci_preferred"])...) + + forEachObject(raw["memberships"], func(membership map[string]any) { + if parsed.CallID == "" { + parsed.CallID = firstString(membership, "call_id", "m.call_id", "callId", "callID") + } + if parsed.DeviceID == "" { + parsed.DeviceID = firstString(membership, "device_id", "m.device_id", "deviceId", "deviceID") + } + if parsed.SessionID == "" { + parsed.SessionID = firstString(membership, "session_id", "m.session_id", "sessionId", "sessionID") + } + if parsed.Intent == "" { + parsed.Intent = firstString(membership, "intent", "m.call.intent", "call_intent") + } + if parsed.LifetimeMS == 0 { + parsed.LifetimeMS = firstInt(membership, "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms") + } + forEachObject(membership["application"], func(application map[string]any) { + if parsed.Intent == "" { + parsed.Intent = firstString(application, "intent", "m.call.intent", "call_intent") + } + }) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["rtc_transports"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["foci_preferred"])...) + parsed.FociPreferred = append(parsed.FociPreferred, parseFoci(membership["m.foci_preferred"])...) + }) + + if parsed.DeviceID == "" { + parsed.DeviceID = parsed.StateKey + } +} + +func firstString(raw map[string]any, keys ...string) string { + for _, key := range keys { + if value, ok := raw[key]; ok { + if str, ok := value.(string); ok { + return str + } + } + } + return "" +} + +func firstInt(raw map[string]any, keys ...string) int { + for _, key := range keys { + value, ok := raw[key] + if !ok { + continue + } + switch typed := value.(type) { + case int: + return typed + case int64: + return int(typed) + case float64: + return int(typed) + case json.Number: + if integer, err := typed.Int64(); err == nil { + return int(integer) + } + } + } + return 0 +} + +func MatrixRTCEventHasJoinContent(evt MatrixRTCEvent) bool { + switch evt.Kind { + case MatrixRTCEventKindRTCMembership, MatrixRTCEventKindGroupCallMember: + return matrixRTCContentHasJoinData(evt.Raw) + default: + return false + } +} + +func matrixRTCContentHasJoinData(raw map[string]any) bool { + if len(raw) == 0 { + return false + } + if matrixRTCModernContentHasJoinData(raw) || matrixRTCLegacyContentHasJoinData(raw) { + return true + } + hasJoin := false + forEachObject(raw["memberships"], func(membership map[string]any) { + if matrixRTCMembershipArrayItemHasJoinData(membership) { + hasJoin = true + } + }) + return hasJoin +} + +func matrixRTCModernContentHasJoinData(raw map[string]any) bool { + if slotID := firstString(raw, "slot_id"); slotID != "" && slotID != MatrixRTCDefaultSlotID { + return false + } + return matrixRTCApplicationIsCall(raw["application"]) && + matrixRTCContentHasMember(raw) && + len(parseFoci(raw["rtc_transports"])) > 0 +} + +func matrixRTCLegacyContentHasJoinData(raw map[string]any) bool { + if !matrixRTCApplicationIsCall(raw["application"]) || + !matrixRTCContentHasIdentifier(raw) || + !matrixRTCContentHasPositiveLifetime(raw) { + return false + } + return len(parseFoci(raw["foci_preferred"])) > 0 || + len(parseFoci(raw["m.foci_preferred"])) > 0 +} + +func matrixRTCMembershipArrayItemHasJoinData(raw map[string]any) bool { + if application, ok := raw["application"]; ok && !matrixRTCApplicationIsCall(application) { + return false + } + if !matrixRTCContentHasIdentifier(raw) || !matrixRTCContentHasPositiveLifetime(raw) { + return false + } + return len(parseFoci(raw["rtc_transports"])) > 0 || + len(parseFoci(raw["foci_preferred"])) > 0 || + len(parseFoci(raw["m.foci_preferred"])) > 0 +} + +func matrixRTCApplicationIsCall(value any) bool { + switch typed := value.(type) { + case string: + return typed == MatrixRTCApplicationCall + case map[string]any: + return firstString(typed, "type", "application") == MatrixRTCApplicationCall + case []any: + for _, item := range typed { + if matrixRTCApplicationIsCall(item) { + return true + } + } + case []map[string]any: + for _, item := range typed { + if matrixRTCApplicationIsCall(item) { + return true + } + } + } + return false +} + +func matrixRTCContentHasMember(raw map[string]any) bool { + hasMember := false + forEachObject(raw["member"], func(member map[string]any) { + if firstString(member, "user_id", "device_id", "id") != "" { + hasMember = true + } + }) + return hasMember +} + +func matrixRTCContentHasIdentifier(raw map[string]any) bool { + return matrixRTCContentHasMember(raw) || + firstString(raw, "membershipID", "membership_id", "device_id", "m.device_id", "deviceId", "deviceID", "session_id", "m.session_id", "sessionId", "sessionID") != "" +} + +func matrixRTCContentHasPositiveLifetime(raw map[string]any) bool { + for _, key := range []string{"expires", "lifetime", "lifetime_ms", "m.lifetime", "m.lifetime_ms"} { + if _, ok := raw[key]; ok { + return firstInt(raw, key) > 0 + } + } + return true +} + +func forEachObject(value any, fn func(map[string]any)) { + switch typed := value.(type) { + case []any: + for _, item := range typed { + if object, ok := item.(map[string]any); ok { + fn(object) + } + } + case []map[string]any: + for _, item := range typed { + fn(item) + } + case map[string]any: + fn(typed) + } +} + +func parseFoci(value any) []Focus { + var output []Focus + forEachObject(value, func(rawFocus map[string]any) { + if firstString(rawFocus, "type") != "livekit" { + return + } + serviceURL := firstString(rawFocus, "livekit_service_url", "livekit_service_url_prefix", "service_url") + if serviceURL == "" { + return + } + output = append(output, Focus{ + Type: "livekit", + LiveKitServiceURL: serviceURL, + }) + }) + return output +} + +func MatrixRTCDeviceID(loginID, waCallID string) string { + sum := sha256.Sum256([]byte(loginID + "\x00" + waCallID)) + return "WA" + hex.EncodeToString(sum[:8]) +} + +func MatrixRTCMemberID(userID id.UserID, deviceID string) string { + if deviceID == "" { + return userID.String() + } + return userID.String() + ":" + deviceID +} + +func MatrixRTCStateKey(userID id.UserID, deviceID string) string { + if deviceID == "" { + return userID.String() + } + return userID.String() + "_" + deviceID +} + +func RTCMembershipEventType(class event.TypeClass) event.Type { + return event.Type{Type: EventTypeRTCMembership, Class: class} +} + +func GroupCallMemberEventType() event.Type { + return event.Type{Type: EventTypeGroupCallMember, Class: event.StateEventType} +} + +func RTCNotificationEventType() event.Type { + return event.Type{Type: EventTypeRTCNotification, Class: event.MessageEventType} +} + +func LegacyCallNotifyEventType() event.Type { + return event.Type{Type: EventTypeCallNotify, Class: event.MessageEventType} +} + +func BuildRTCMembershipContent(session MatrixRTCSession) map[string]any { + deviceID := session.DeviceID + memberID := session.MemberID + if memberID == "" { + memberID = MatrixRTCMemberID(session.UserID, deviceID) + } + stickyKey := session.StickyKey + if stickyKey == "" { + stickyKey = memberID + } + intent := session.Intent + if intent == "" { + intent = "audio" + } + application := map[string]any{ + "type": MatrixRTCApplicationCall, + "m.call.intent": intent, + } + content := map[string]any{ + "slot_id": MatrixRTCDefaultSlotID, + "member": map[string]any{ + "user_id": session.UserID.String(), + "device_id": deviceID, + "id": memberID, + }, + "application": application, + "rtc_transports": []map[string]any{liveKitTransport(session.Focus)}, + "versions": []string{MatrixRTCMembershipV1}, + "sticky_key": stickyKey, + "msc4354_sticky_key": stickyKey, + } + if session.NotificationEventID != "" { + content["m.relates_to"] = map[string]any{ + "rel_type": "m.reference", + "event_id": session.NotificationEventID.String(), + } + } + return content +} + +func BuildLegacyCallMemberContent(session MatrixRTCSession) map[string]any { + deviceID := session.DeviceID + memberID := session.MemberID + if memberID == "" { + memberID = MatrixRTCMemberID(session.UserID, deviceID) + } + intent := session.Intent + if intent == "" { + intent = "audio" + } + created := session.Created + if created.IsZero() { + created = time.Now() + } + expires := session.Expires + if expires <= 0 { + expires = 4 * time.Hour + } + return map[string]any{ + "application": MatrixRTCApplicationCall, + "call_id": "", + "device_id": deviceID, + "focus_active": map[string]any{ + "type": "livekit", + "focus_selection": "multi_sfu", + }, + "foci_preferred": []map[string]any{liveKitTransport(session.Focus)}, + "created_ts": created.UnixMilli(), + "scope": "m.room", + "expires": expires.Milliseconds(), + "m.call.intent": intent, + "membershipID": memberID, + } +} + +func BuildRTCNotificationContent(now time.Time, lifetime time.Duration, intent string) map[string]any { + if now.IsZero() { + now = time.Now() + } + if lifetime <= 0 || lifetime > 90*time.Second { + lifetime = 90 * time.Second + } + if intent == "" { + intent = "audio" + } + return map[string]any{ + "notification_type": "ring", + "sender_ts": now.UnixMilli(), + "lifetime": lifetime.Milliseconds(), + "m.call.intent": intent, + "m.mentions": map[string]any{}, + } +} + +func BuildLegacyCallNotifyContent(callID, intent string) map[string]any { + if intent == "" { + intent = "audio" + } + return map[string]any{ + "application": MatrixRTCApplicationCall, + "notify_type": "ring", + "call_id": callID, + "m.call.intent": intent, + "m.mentions": map[string]any{}, + } +} + +func EmptyMatrixRTCContent(stickyKey string) map[string]any { + if stickyKey == "" { + return map[string]any{} + } + return map[string]any{ + "sticky_key": stickyKey, + "msc4354_sticky_key": stickyKey, + } +} + +func liveKitTransport(focus Focus) map[string]any { + transport := map[string]any{ + "type": "livekit", + } + if focus.LiveKitServiceURL != "" { + transport["livekit_service_url"] = focus.LiveKitServiceURL + } + return transport +} diff --git a/pkg/connector/voip/matrixrtc_test.go b/pkg/connector/voip/matrixrtc_test.go new file mode 100644 index 0000000..1c605b0 --- /dev/null +++ b/pkg/connector/voip/matrixrtc_test.go @@ -0,0 +1,229 @@ +package voip + +import ( + "testing" + "time" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func TestSupportedMatrixRTCEventTypesHaveExplicitClasses(t *testing.T) { + types := SupportedMatrixRTCEventTypes() + if len(types) != 10 { + t.Fatalf("SupportedMatrixRTCEventTypes returned %d types, want 10", len(types)) + } + for _, evtType := range types { + switch evtType.Type { + case EventTypeGroupCall, EventTypeGroupCallMember: + if evtType.Class != event.StateEventType { + t.Fatalf("%s class = %v, want state", evtType.Type, evtType.Class) + } + case EventTypeRTCMembership: + if evtType.Class != event.StateEventType && evtType.Class != event.MessageEventType { + t.Fatalf("%s class = %v, want state or message", evtType.Type, evtType.Class) + } + case EventTypeRTCNotification, EventTypeCallNotify, EventTypeRTCDecline, + EventTypeElementCallReaction, event.EventReaction.Type, event.EventRedaction.Type: + if evtType.Class != event.MessageEventType { + t.Fatalf("%s class = %v, want message", evtType.Type, evtType.Class) + } + default: + t.Fatalf("unexpected MatrixRTC event type %s", evtType.Type) + } + } +} + +func TestParseElementCallReactionEvent(t *testing.T) { + evt := &event.Event{ + ID: id.EventID("$reaction"), + Type: ElementCallReactionEventType(), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + Content: event.Content{Raw: map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": "m.reference", + "event_id": "$membership", + }, + "emoji": "❤️", + "name": "generic", + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatal("ParseMatrixRTCEvent did not recognize Element Call reaction") + } + if parsed.Kind != MatrixRTCEventKindCallReaction || + parsed.EventID != "$reaction" || + parsed.RelatesToEventID != "$membership" || + parsed.ReactionEmoji != "❤️" { + t.Fatalf("unexpected parsed reaction: %+v", parsed) + } +} + +func TestSupportedWhatsAppCallReactions(t *testing.T) { + for _, emoji := range []string{"👍", "❤️", "😂", "😮", "😢", "🙏"} { + normalized, ok := NormalizeWhatsAppCallReaction(emoji) + if !ok || normalized != emoji { + t.Fatalf("NormalizeWhatsAppCallReaction(%q) = %q, %t", emoji, normalized, ok) + } + } + if normalized, ok := NormalizeWhatsAppCallReaction("🎉"); ok || normalized != "" { + t.Fatalf("unsupported reaction normalized to %q, %t", normalized, ok) + } +} + +func TestParseMatrixRTCDeclineEvent(t *testing.T) { + evt := &event.Event{ + Type: event.Type{Type: EventTypeRTCDecline, Class: event.MessageEventType}, + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + Content: event.Content{Raw: map[string]any{ + "call_id": "call-1", + "device_id": "DEVICE", + "session_id": "SESSION", + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize decline event") + } + if parsed.Kind != MatrixRTCEventKindRTCDecline { + t.Fatalf("kind = %q, want %q", parsed.Kind, MatrixRTCEventKindRTCDecline) + } + if parsed.CallID != "call-1" || parsed.DeviceID != "DEVICE" || parsed.SessionID != "SESSION" { + t.Fatalf("unexpected parsed event: %+v", parsed) + } +} + +func TestParseMatrixRTCMembershipEvent(t *testing.T) { + stateKey := "@alice:example.com" + evt := &event.Event{ + Type: event.Type{Type: EventTypeRTCMembership, Class: event.StateEventType}, + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@alice:example.com"), + StateKey: &stateKey, + Content: event.Content{Raw: map[string]any{ + "memberships": []any{map[string]any{ + "call_id": "call-2", + "device_id": "DEVICE", + "session_id": "SESSION", + "lifetime_ms": float64(60000), + "foci_preferred": []any{map[string]any{ + "type": "livekit", + "livekit_service_url": "https://rtc.example.com/jwt", + }}, + }}, + }}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership event") + } + if parsed.Kind != MatrixRTCEventKindRTCMembership || parsed.StateKey != stateKey { + t.Fatalf("unexpected parsed event metadata: %+v", parsed) + } + if parsed.CallID != "call-2" || parsed.DeviceID != "DEVICE" || parsed.SessionID != "SESSION" { + t.Fatalf("unexpected parsed event identifiers: %+v", parsed) + } + if parsed.LifetimeMS != 60000 { + t.Fatalf("LifetimeMS = %d, want 60000", parsed.LifetimeMS) + } + if len(parsed.FociPreferred) != 1 || parsed.FociPreferred[0].LiveKitServiceURL != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected foci: %+v", parsed.FociPreferred) + } +} + +func TestBuildRTCMembershipContent(t *testing.T) { + session := MatrixRTCSession{ + UserID: "@wa_123:example.com", + DeviceID: "WADEVICE", + Focus: Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + } + content := BuildRTCMembershipContent(session) + if content["slot_id"] != MatrixRTCDefaultSlotID { + t.Fatalf("slot_id = %q, want %q", content["slot_id"], MatrixRTCDefaultSlotID) + } + member := content["member"].(map[string]any) + if member["user_id"] != "@wa_123:example.com" || member["device_id"] != "WADEVICE" { + t.Fatalf("unexpected member: %+v", member) + } + transports := content["rtc_transports"].([]map[string]any) + if len(transports) != 1 || transports[0]["livekit_service_url"] != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected transports: %+v", transports) + } +} + +func TestParseBuiltRTCMembershipContent(t *testing.T) { + content := BuildRTCMembershipContent(MatrixRTCSession{ + UserID: "@wa_123:example.com", + DeviceID: "WADEVICE", + Intent: "audio", + Focus: Focus{ + Type: "livekit", + LiveKitServiceURL: "https://rtc.example.com/jwt", + }, + }) + evt := &event.Event{ + Type: RTCMembershipEventType(event.MessageEventType), + RoomID: id.RoomID("!room:example.com"), + Sender: id.UserID("@wa_123:example.com"), + Content: event.Content{Raw: content}, + } + parsed, ok := ParseMatrixRTCEvent(evt) + if !ok { + t.Fatalf("ParseMatrixRTCEvent did not recognize membership event") + } + if parsed.Intent != "audio" { + t.Fatalf("Intent = %q, want audio", parsed.Intent) + } + if len(parsed.FociPreferred) != 1 || parsed.FociPreferred[0].LiveKitServiceURL != "https://rtc.example.com/jwt" { + t.Fatalf("unexpected foci: %+v", parsed.FociPreferred) + } + if !MatrixRTCEventHasJoinContent(parsed) { + t.Fatalf("MatrixRTCEventHasJoinContent returned false for a built membership") + } +} + +func TestMatrixRTCEventHasJoinContentRejectsStickyCleanup(t *testing.T) { + evt := MatrixRTCEvent{ + Kind: MatrixRTCEventKindRTCMembership, + Raw: EmptyMatrixRTCContent("sticky"), + } + if MatrixRTCEventHasJoinContent(evt) { + t.Fatalf("MatrixRTCEventHasJoinContent returned true for sticky cleanup content") + } +} + +func TestMatrixRTCEventHasJoinContentRejectsMetadataWithoutFocus(t *testing.T) { + for _, raw := range []map[string]any{ + { + "application": map[string]any{"type": MatrixRTCApplicationCall}, + }, + { + "slot_id": MatrixRTCDefaultSlotID, + "application": map[string]any{"type": MatrixRTCApplicationCall}, + "member": map[string]any{"user_id": "@alice:example.com", "device_id": "DEVICE"}, + }, + } { + evt := MatrixRTCEvent{Kind: MatrixRTCEventKindRTCMembership, Raw: raw} + if MatrixRTCEventHasJoinContent(evt) { + t.Fatalf("MatrixRTCEventHasJoinContent returned true for metadata-only content: %+v", raw) + } + } +} + +func TestBuildRTCNotificationContentCapsLifetime(t *testing.T) { + content := BuildRTCNotificationContent(testTime, 5*time.Minute, "audio") + if content["notification_type"] != "ring" { + t.Fatalf("notification_type = %q, want ring", content["notification_type"]) + } + if content["lifetime"] != int64(90000) { + t.Fatalf("lifetime = %v, want 90000", content["lifetime"]) + } +} + +var testTime = time.Unix(123, 0) diff --git a/pkg/connector/voip/reactions.go b/pkg/connector/voip/reactions.go new file mode 100644 index 0000000..8e7afd7 --- /dev/null +++ b/pkg/connector/voip/reactions.go @@ -0,0 +1,54 @@ +package voip + +import ( + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +var supportedWhatsAppCallReactions = map[string]string{ + "👍": "thumbsup", + "❤️": "generic", + "😂": "generic", + "😮": "generic", + "😢": "generic", + "🙏": "generic", +} + +func ElementCallReactionEventType() event.Type { + return event.Type{Type: EventTypeElementCallReaction, Class: event.MessageEventType} +} + +func NormalizeWhatsAppCallReaction(emoji string) (string, bool) { + if _, ok := supportedWhatsAppCallReactions[emoji]; ok { + return emoji, true + } + return "", false +} + +func ElementCallReactionName(emoji string) string { + if name, ok := supportedWhatsAppCallReactions[emoji]; ok { + return name + } + return "generic" +} + +func BuildElementCallReactionContent(membershipEventID id.EventID, emoji string) map[string]any { + return map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": string(event.RelReference), + "event_id": membershipEventID, + }, + "emoji": emoji, + "name": ElementCallReactionName(emoji), + } +} + +func BuildElementCallHandRaiseContent(membershipEventID id.EventID) map[string]any { + return map[string]any{ + "m.relates_to": map[string]any{ + "rel_type": string(event.RelAnnotation), + "event_id": membershipEventID, + "key": "🖐️", + }, + } +} diff --git a/pkg/connector/voip/video.go b/pkg/connector/voip/video.go new file mode 100644 index 0000000..d97264b --- /dev/null +++ b/pkg/connector/voip/video.go @@ -0,0 +1,162 @@ +package voip + +import ( + "errors" + "fmt" + "sync" + "time" + + lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/purpshell/meowcaller" + wartp "github.com/purpshell/meowcaller/rtp" +) + +var ErrVideoSinkClosed = errors.New("voip: video sink closed") + +type LiveKitH264Writer struct { + mu sync.RWMutex + track interface { + WriteSample(media.Sample, *lksdk.SampleWriteOptions) error + } + duration time.Duration + closed bool +} + +type liveKitVideoOrientationSetter interface { + SetVideoOrientation(uint8) +} + +type h264ParameterSetRepeater struct { + sps []byte + pps []byte +} + +func (r *h264ParameterSetRepeater) Normalize(accessUnit []byte) ([]byte, bool) { + nalus := wartp.SplitAnnexB(accessUnit) + if len(nalus) == 0 { + return accessUnit, false + } + var currentSPS, currentPPS []byte + hasIDR := false + for _, nalu := range nalus { + if len(nalu) == 0 { + continue + } + switch nalu[0] & 0x1f { + case 5: + hasIDR = true + case 7: + currentSPS = nalu + r.sps = append(r.sps[:0], nalu...) + case 8: + currentPPS = nalu + r.pps = append(r.pps[:0], nalu...) + } + } + if !hasIDR || (currentSPS != nil && currentPPS != nil) { + return accessUnit, false + } + sps := currentSPS + if sps == nil { + sps = r.sps + } + pps := currentPPS + if pps == nil { + pps = r.pps + } + if len(sps) == 0 || len(pps) == 0 { + return accessUnit, false + } + + normalized := make([]byte, 0, len(accessUnit)+len(sps)+len(pps)+8) + normalized = appendAnnexBNAL(normalized, sps) + normalized = appendAnnexBNAL(normalized, pps) + for _, nalu := range nalus { + if len(nalu) == 0 || nalu[0]&0x1f == 7 || nalu[0]&0x1f == 8 { + continue + } + normalized = appendAnnexBNAL(normalized, nalu) + } + return normalized, true +} + +func appendAnnexBNAL(dst, nalu []byte) []byte { + dst = append(dst, 0, 0, 0, 1) + return append(dst, nalu...) +} + +func h264AccessUnitMetadata(accessUnit []byte) (nalTypes []int, profileLevelID string, hasIDR, hasSPS, hasPPS bool) { + for _, nalu := range wartp.SplitAnnexB(accessUnit) { + if len(nalu) == 0 { + continue + } + nalType := int(nalu[0] & 0x1f) + nalTypes = append(nalTypes, nalType) + switch nalType { + case 5: + hasIDR = true + case 7: + hasSPS = true + if len(nalu) >= 4 { + profileLevelID = fmt.Sprintf("%02x%02x%02x", nalu[1], nalu[2], nalu[3]) + } + case 8: + hasPPS = true + } + } + return +} + +func NewLiveKitH264Writer(track interface { + WriteSample(media.Sample, *lksdk.SampleWriteOptions) error +}, duration time.Duration) *LiveKitH264Writer { + if duration <= 0 { + duration = time.Second / 30 + } + return &LiveKitH264Writer{track: track, duration: duration} +} + +func (w *LiveKitH264Writer) WriteVideo(accessUnit []byte) error { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return ErrVideoSinkClosed + } + if w.track == nil || len(accessUnit) == 0 { + return nil + } + sample := media.Sample{ + Data: append([]byte(nil), accessUnit...), + Duration: w.duration, + } + return w.track.WriteSample(sample, nil) +} + +func (w *LiveKitH264Writer) SetOrientation(orientation int) { + w.mu.RLock() + defer w.mu.RUnlock() + if w.closed { + return + } + setLiveKitVideoOrientation(w.track, orientation) +} + +func setLiveKitVideoOrientation(track any, orientation int) bool { + setter, ok := track.(liveKitVideoOrientationSetter) + if !ok { + return false + } + setter.SetVideoOrientation(uint8(orientation) & 0x03) + return true +} + +func (w *LiveKitH264Writer) Close() error { + w.mu.Lock() + w.closed = true + w.track = nil + w.mu.Unlock() + return nil +} + +var _ meowcaller.VideoSink = (*LiveKitH264Writer)(nil) diff --git a/pkg/connector/voip/video_buffer.go b/pkg/connector/voip/video_buffer.go new file mode 100644 index 0000000..68d6773 --- /dev/null +++ b/pkg/connector/voip/video_buffer.go @@ -0,0 +1,56 @@ +package voip + +import "time" + +const maxPendingWhatsAppVideoFrames = 64 + +type LiveKitVideoFrame struct { + AccessUnit []byte + Duration time.Duration +} + +type whatsAppVideoStartupBuffer struct { + ready bool + frames []LiveKitVideoFrame +} + +func (b *whatsAppVideoStartupBuffer) Len() int { + if b == nil { + return 0 + } + return len(b.frames) +} + +func (b *whatsAppVideoStartupBuffer) Send(frame LiveKitVideoFrame, send func(LiveKitVideoFrame) error) (int, error) { + if b == nil || len(frame.AccessUnit) == 0 || send == nil { + return 0, nil + } + if b.ready { + return 0, send(frame) + } + b.enqueue(frame) + flushed := 0 + for len(b.frames) > 0 { + if err := send(b.frames[0]); err != nil { + return flushed, err + } + b.frames[0] = LiveKitVideoFrame{} + b.frames = b.frames[1:] + flushed++ + } + b.ready = true + return flushed, nil +} + +func (b *whatsAppVideoStartupBuffer) enqueue(frame LiveKitVideoFrame) { + queued := LiveKitVideoFrame{ + AccessUnit: append([]byte(nil), frame.AccessUnit...), + Duration: frame.Duration, + } + b.frames = append(b.frames, queued) + if len(b.frames) <= maxPendingWhatsAppVideoFrames { + return + } + copy(b.frames, b.frames[len(b.frames)-maxPendingWhatsAppVideoFrames:]) + b.frames = b.frames[:maxPendingWhatsAppVideoFrames] +} diff --git a/pkg/connector/voip/video_test.go b/pkg/connector/voip/video_test.go new file mode 100644 index 0000000..f229595 --- /dev/null +++ b/pkg/connector/voip/video_test.go @@ -0,0 +1,156 @@ +package voip + +import ( + "bytes" + "testing" + "time" + + lksdk "github.com/livekit/server-sdk-go/v2" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/purpshell/meowcaller" +) + +func annexBNAL(nalu ...byte) []byte { + return append([]byte{0, 0, 0, 1}, nalu...) +} + +func TestH264ParameterSetRepeaterAddsCachedHeadersToIDR(t *testing.T) { + repeater := h264ParameterSetRepeater{} + sps := annexBNAL(0x67, 0x42, 0xe0, 0x1f) + pps := annexBNAL(0x68, 0xce, 0x06, 0xe2) + repeater.Normalize(append(append([]byte{}, sps...), pps...)) + + idr := annexBNAL(0x65, 0x88, 0x84) + got, repeated := repeater.Normalize(idr) + want := append(append(append([]byte{}, sps...), pps...), idr...) + if !repeated { + t.Fatal("Normalize did not report repeated parameter sets") + } + if !bytes.Equal(got, want) { + t.Fatalf("normalized IDR = %x, want %x", got, want) + } +} + +func TestH264ParameterSetRepeaterPreservesCompleteIDR(t *testing.T) { + repeater := h264ParameterSetRepeater{} + au := append(append(annexBNAL(0x67, 0x42, 0xe0, 0x1f), annexBNAL(0x68, 0xce, 0x06, 0xe2)...), annexBNAL(0x65, 0x88, 0x84)...) + + got, repeated := repeater.Normalize(au) + if repeated { + t.Fatal("Normalize reported repeating already-present parameter sets") + } + if !bytes.Equal(got, au) { + t.Fatalf("complete IDR changed: got %x, want %x", got, au) + } +} + +func TestH264ParameterSetRepeaterUsesCurrentAndCachedHeadersInDecodeOrder(t *testing.T) { + repeater := h264ParameterSetRepeater{} + oldSPS := annexBNAL(0x67, 0x42, 0xe0, 0x1f) + pps := annexBNAL(0x68, 0xce, 0x06, 0xe2) + repeater.Normalize(append(append([]byte{}, oldSPS...), pps...)) + + newSPS := annexBNAL(0x67, 0x42, 0xe0, 0x20) + idr := annexBNAL(0x65, 0x99) + au := append(append([]byte{}, newSPS...), idr...) + got, repeated := repeater.Normalize(au) + want := append(append(append([]byte{}, newSPS...), pps...), idr...) + if !repeated { + t.Fatal("Normalize did not report filling the missing PPS") + } + if !bytes.Equal(got, want) { + t.Fatalf("normalized partial IDR = %x, want %x", got, want) + } +} + +type orientedSampleTrack struct { + orientation uint8 +} + +func (t *orientedSampleTrack) WriteSample(media.Sample, *lksdk.SampleWriteOptions) error { + return nil +} + +func (t *orientedSampleTrack) SetVideoOrientation(orientation uint8) { + t.orientation = orientation +} + +func TestLiveKitH264WriterSetsVideoOrientation(t *testing.T) { + track := &orientedSampleTrack{} + writer := NewLiveKitH264Writer(track, time.Second/30) + + writer.SetOrientation(5) + + if track.orientation != 1 { + t.Fatalf("track orientation = %d, want 1", track.orientation) + } +} + +func TestLiveKitParticipantRequestsRemoteVideoKeyframe(t *testing.T) { + const wantSSRC = webrtc.SSRC(0x12345678) + var gotSSRC webrtc.SSRC + participant := &LiveKitParticipant{} + + if participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned true before track subscription") + } + participant.setRemoteVideoPLI(func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, wantSSRC) + if gotSSRC != wantSSRC { + t.Fatalf("deferred PLI SSRC = %#x, want %#x", gotSSRC, wantSSRC) + } + + gotSSRC = 0 + if !participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned false with a subscribed track") + } + if gotSSRC != wantSSRC { + t.Fatalf("immediate PLI SSRC = %#x, want %#x", gotSSRC, wantSSRC) + } +} + +func TestLiveKitParticipantKeepsScreenShareKeyframeRequestSourceSpecific(t *testing.T) { + const ( + cameraSSRC = webrtc.SSRC(0x11111111) + screenSSRC = webrtc.SSRC(0x22222222) + ) + var gotSSRC webrtc.SSRC + participant := &LiveKitParticipant{remoteScreenActive: true} + + if participant.requestRemoteVideoKeyframe() { + t.Fatal("requestRemoteVideoKeyframe returned true before screen-share subscription") + } + participant.setRemoteVideoPLIForSource(liveKitVideoSourceCamera, func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, cameraSSRC) + if gotSSRC != 0 { + t.Fatalf("camera subscription consumed pending screen-share PLI with SSRC %#x", gotSSRC) + } + participant.setRemoteVideoPLIForSource(liveKitVideoSourceScreenShare, func(ssrc webrtc.SSRC) { + gotSSRC = ssrc + }, screenSSRC) + if gotSSRC != screenSSRC { + t.Fatalf("screen-share subscription PLI SSRC = %#x, want %#x", gotSSRC, screenSSRC) + } +} + +func TestManagerDefersVideoKeyframeOnlyForTrackedCall(t *testing.T) { + manager := &Manager{ + calls: make(map[string]*meowcaller.Call), + livekit: make(map[string]*LiveKitParticipant), + videoKeyframePending: make(map[string]bool), + } + + manager.requestLiveKitVideoKeyframe("ended") + if manager.videoKeyframePending["ended"] { + t.Fatal("keyframe request was retained for an untracked call") + } + + manager.calls["active"] = &meowcaller.Call{} + manager.requestLiveKitVideoKeyframe("active") + if !manager.videoKeyframePending["active"] { + t.Fatal("keyframe request was not retained for a tracked call") + } +} diff --git a/pkg/connector/voip_config.go b/pkg/connector/voip_config.go new file mode 100644 index 0000000..ce27737 --- /dev/null +++ b/pkg/connector/voip_config.go @@ -0,0 +1,46 @@ +package connector + +import "go.mau.fi/mautrix-whatsapp/pkg/connector/voip" + +func makeVOIPConfig(cfg VOIPConfig) voip.Config { + return voip.Config{ + Enabled: cfg.Enabled, + IncomingPolicy: cfg.IncomingPolicy, + MaxActiveCallsPerLogin: cfg.MaxActiveCallsPerLogin, + MatrixRTC: voip.MatrixRTCConfig{ + LiveKitServiceURL: cfg.MatrixRTC.LiveKitServiceURL, + RequireLiveKitFocus: cfg.MatrixRTC.RequireLiveKitFocus, + MembershipEventCompat: cfg.MatrixRTC.MembershipEventCompat, + NotificationEventCompat: cfg.MatrixRTC.NotificationEventCompat, + UseDelayedEvents: cfg.MatrixRTC.UseDelayedEvents, + ParticipantMode: cfg.MatrixRTC.ParticipantMode, + FallbackParticipantMXID: cfg.MatrixRTC.FallbackParticipantMXID, + }, + LiveKit: voip.LiveKitConfig{ + ConnectTimeout: cfg.LiveKit.ConnectTimeout, + PublishSilenceBeforeWhatsAppAnswer: cfg.LiveKit.PublishSilenceBeforeWhatsAppAnswer, + AutoSubscribe: cfg.LiveKit.AutoSubscribe, + AudioUplinkPolicy: cfg.LiveKit.AudioUplinkPolicy, + SelectedParticipantTimeout: cfg.LiveKit.SelectedParticipantTimeout, + }, + Audio: voip.AudioConfig{ + Enabled: cfg.Audio.Enabled, + JitterBuffer: cfg.Audio.JitterBuffer, + OpusBackend: cfg.Audio.OpusBackend, + SilenceOnUnderrun: cfg.Audio.SilenceOnUnderrun, + MaxMixParticipants: cfg.Audio.MaxMixParticipants, + }, + Video: voip.VideoConfig{ + Enabled: cfg.Video.Enabled, + SelectedSourcePolicy: cfg.Video.SelectedSourcePolicy, + MaxWidth: cfg.Video.MaxWidth, + MaxHeight: cfg.Video.MaxHeight, + MaxFPS: cfg.Video.MaxFPS, + }, + Diagnostics: voip.DiagnosticsConfig{ + HealthcheckFocusOnStartup: cfg.Diagnostics.HealthcheckFocusOnStartup, + EnableMeowcallerDiagnostics: cfg.Diagnostics.EnableMeowcallerDiagnostics, + MediaTraceDir: cfg.Diagnostics.MediaTraceDir, + }, + } +} diff --git a/pkg/connector/wadb/call.go b/pkg/connector/wadb/call.go new file mode 100644 index 0000000..6c25f3a --- /dev/null +++ b/pkg/connector/wadb/call.go @@ -0,0 +1,271 @@ +package wadb + +import ( + "context" + "database/sql" + "time" + + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/id" +) + +type MatrixRTCCallQuery struct { + BridgeID networkid.BridgeID + *dbutil.QueryHelper[*MatrixRTCCall] +} + +type MatrixRTCCall struct { + BridgeID networkid.BridgeID + UserLoginID networkid.UserLoginID + WACallID string + RoomID id.RoomID + PortalKey networkid.PortalKey + PeerJID types.JID + Direction string + MediaKind string + FocusType string + LiveKitServiceURL string + LiveKitRoom string + MatrixParticipantMXID id.UserID + MatrixSessionID string + SelectedPublisherID string + BridgeMembershipEventID id.EventID + SelectedMembershipEventID id.EventID + BridgeHandRaiseEventID id.EventID + SelectedHandRaiseEventID id.EventID + AudioPolicy string + State string + CreatedTS time.Time + JoinedTS time.Time + AnsweredTS time.Time + EndedTS time.Time + EndReason string + LastError string +} + +const ( + upsertMatrixRTCCallQuery = ` + INSERT INTO whatsapp_matrixrtc_call ( + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27) + ON CONFLICT (bridge_id, user_login_id, wa_call_id) DO UPDATE SET + room_id=excluded.room_id, + portal_id=excluded.portal_id, + portal_receiver=excluded.portal_receiver, + peer_jid=excluded.peer_jid, + direction=excluded.direction, + media_kind=excluded.media_kind, + focus_type=excluded.focus_type, + livekit_service_url=excluded.livekit_service_url, + livekit_room=excluded.livekit_room, + matrix_participant_mxid=excluded.matrix_participant_mxid, + matrix_session_id=excluded.matrix_session_id, + selected_publisher_id=excluded.selected_publisher_id, + bridge_membership_event_id=excluded.bridge_membership_event_id, + selected_membership_event_id=excluded.selected_membership_event_id, + bridge_hand_raise_event_id=excluded.bridge_hand_raise_event_id, + selected_hand_raise_event_id=excluded.selected_hand_raise_event_id, + audio_policy=excluded.audio_policy, + state=excluded.state, + joined_ts=excluded.joined_ts, + answered_ts=excluded.answered_ts, + ended_ts=excluded.ended_ts, + end_reason=excluded.end_reason, + last_error=excluded.last_error + ` + getMatrixRTCCallQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` + getActiveMatrixRTCCallsForLoginQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND ended_ts IS NULL + ` + getActiveMatrixRTCCallsInRoomQuery = ` + SELECT + bridge_id, user_login_id, wa_call_id, room_id, portal_id, portal_receiver, peer_jid, + direction, media_kind, focus_type, livekit_service_url, livekit_room, + matrix_participant_mxid, matrix_session_id, selected_publisher_id, + bridge_membership_event_id, selected_membership_event_id, + bridge_hand_raise_event_id, selected_hand_raise_event_id, + audio_policy, state, created_ts, joined_ts, answered_ts, ended_ts, + end_reason, last_error + FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND room_id=$2 AND ended_ts IS NULL + ` + markMatrixRTCCallEndedQuery = ` + UPDATE whatsapp_matrixrtc_call + SET state=$4, ended_ts=$5, end_reason=$6, last_error=$7 + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` + deleteMatrixRTCCallQuery = ` + DELETE FROM whatsapp_matrixrtc_call + WHERE bridge_id=$1 AND user_login_id=$2 AND wa_call_id=$3 + ` +) + +func (cq *MatrixRTCCallQuery) Put(ctx context.Context, call *MatrixRTCCall) error { + call.BridgeID = cq.BridgeID + return cq.Exec(ctx, upsertMatrixRTCCallQuery, call.sqlVariables()...) +} + +func (cq *MatrixRTCCallQuery) Get(ctx context.Context, loginID networkid.UserLoginID, waCallID string) (*MatrixRTCCall, error) { + return cq.QueryOne(ctx, getMatrixRTCCallQuery, cq.BridgeID, loginID, waCallID) +} + +func (cq *MatrixRTCCallQuery) GetActiveForLogin(ctx context.Context, loginID networkid.UserLoginID) ([]*MatrixRTCCall, error) { + return cq.QueryMany(ctx, getActiveMatrixRTCCallsForLoginQuery, cq.BridgeID, loginID) +} + +func (cq *MatrixRTCCallQuery) GetActiveInRoom(ctx context.Context, roomID id.RoomID) ([]*MatrixRTCCall, error) { + return cq.QueryMany(ctx, getActiveMatrixRTCCallsInRoomQuery, cq.BridgeID, roomID) +} + +func (cq *MatrixRTCCallQuery) MarkEnded(ctx context.Context, loginID networkid.UserLoginID, waCallID, state, reason, lastError string, ended time.Time) error { + return cq.Exec(ctx, markMatrixRTCCallEndedQuery, cq.BridgeID, loginID, waCallID, state, nullableUnix(ended), reason, lastError) +} + +func (cq *MatrixRTCCallQuery) Delete(ctx context.Context, loginID networkid.UserLoginID, waCallID string) error { + return cq.Exec(ctx, deleteMatrixRTCCallQuery, cq.BridgeID, loginID, waCallID) +} + +func (call *MatrixRTCCall) Scan(row dbutil.Scannable) (*MatrixRTCCall, error) { + var liveKitRoom, participantMXID, matrixSessionID, selectedPublisherID sql.NullString + var bridgeMembershipEventID, selectedMembershipEventID, bridgeHandRaiseEventID, selectedHandRaiseEventID sql.NullString + var endReason, lastError sql.NullString + var joinedTS, answeredTS, endedTS sql.NullInt64 + var createdTS int64 + err := row.Scan( + &call.BridgeID, + &call.UserLoginID, + &call.WACallID, + &call.RoomID, + &call.PortalKey.ID, + &call.PortalKey.Receiver, + &call.PeerJID, + &call.Direction, + &call.MediaKind, + &call.FocusType, + &call.LiveKitServiceURL, + &liveKitRoom, + &participantMXID, + &matrixSessionID, + &selectedPublisherID, + &bridgeMembershipEventID, + &selectedMembershipEventID, + &bridgeHandRaiseEventID, + &selectedHandRaiseEventID, + &call.AudioPolicy, + &call.State, + &createdTS, + &joinedTS, + &answeredTS, + &endedTS, + &endReason, + &lastError, + ) + if err != nil { + return nil, err + } + call.CreatedTS = unixToTime(createdTS) + call.JoinedTS = nullUnixToTime(joinedTS) + call.AnsweredTS = nullUnixToTime(answeredTS) + call.EndedTS = nullUnixToTime(endedTS) + call.LiveKitRoom = liveKitRoom.String + call.MatrixParticipantMXID = id.UserID(participantMXID.String) + call.MatrixSessionID = matrixSessionID.String + call.SelectedPublisherID = selectedPublisherID.String + call.BridgeMembershipEventID = id.EventID(bridgeMembershipEventID.String) + call.SelectedMembershipEventID = id.EventID(selectedMembershipEventID.String) + call.BridgeHandRaiseEventID = id.EventID(bridgeHandRaiseEventID.String) + call.SelectedHandRaiseEventID = id.EventID(selectedHandRaiseEventID.String) + call.EndReason = endReason.String + call.LastError = lastError.String + return call, nil +} + +func (call *MatrixRTCCall) sqlVariables() []any { + return []any{ + call.BridgeID, + call.UserLoginID, + call.WACallID, + call.RoomID, + call.PortalKey.ID, + call.PortalKey.Receiver, + call.PeerJID, + call.Direction, + call.MediaKind, + call.FocusType, + call.LiveKitServiceURL, + nullString(call.LiveKitRoom), + nullString(string(call.MatrixParticipantMXID)), + nullString(call.MatrixSessionID), + nullString(call.SelectedPublisherID), + nullString(string(call.BridgeMembershipEventID)), + nullString(string(call.SelectedMembershipEventID)), + nullString(string(call.BridgeHandRaiseEventID)), + nullString(string(call.SelectedHandRaiseEventID)), + call.AudioPolicy, + call.State, + nullableUnix(call.CreatedTS), + nullableUnix(call.JoinedTS), + nullableUnix(call.AnsweredTS), + nullableUnix(call.EndedTS), + nullString(call.EndReason), + nullString(call.LastError), + } +} + +func nullString(str string) *string { + if str == "" { + return nil + } + return &str +} + +func nullableUnix(ts time.Time) *int64 { + if ts.IsZero() { + return nil + } + unix := ts.Unix() + return &unix +} + +func unixToTime(ts int64) time.Time { + if ts == 0 { + return time.Time{} + } + return time.Unix(ts, 0) +} + +func nullUnixToTime(ts sql.NullInt64) time.Time { + if !ts.Valid { + return time.Time{} + } + return unixToTime(ts.Int64) +} diff --git a/pkg/connector/wadb/call_test.go b/pkg/connector/wadb/call_test.go new file mode 100644 index 0000000..1254d04 --- /dev/null +++ b/pkg/connector/wadb/call_test.go @@ -0,0 +1,31 @@ +package wadb + +import ( + "testing" + + "maunium.net/go/mautrix/id" +) + +func TestMatrixRTCCallSQLVariablesIncludeReactionEventIDs(t *testing.T) { + call := &MatrixRTCCall{ + BridgeMembershipEventID: id.EventID("$bridge-member"), + SelectedMembershipEventID: id.EventID("$selected-member"), + BridgeHandRaiseEventID: id.EventID("$bridge-hand"), + SelectedHandRaiseEventID: id.EventID("$selected-hand"), + } + variables := call.sqlVariables() + if len(variables) != 27 { + t.Fatalf("MatrixRTCCall.sqlVariables returned %d values, want 27", len(variables)) + } + for index, want := range map[int]string{ + 15: "$bridge-member", + 16: "$selected-member", + 17: "$bridge-hand", + 18: "$selected-hand", + } { + value, ok := variables[index].(*string) + if !ok || value == nil || *value != want { + t.Fatalf("SQL variable %d = %#v, want %q", index, variables[index], want) + } + } +} diff --git a/pkg/connector/wadb/database.go b/pkg/connector/wadb/database.go index 9d13568..2ba92eb 100644 --- a/pkg/connector/wadb/database.go +++ b/pkg/connector/wadb/database.go @@ -10,12 +10,13 @@ import ( type Database struct { *dbutil.Database - Conversation *ConversationQuery - Message *MessageQuery - PollOption *PollOptionQuery - MediaRequest *MediaRequestQuery - HSNotif *HistorySyncNotificationQuery - AvatarCache *AvatarCacheQuery + Conversation *ConversationQuery + Message *MessageQuery + PollOption *PollOptionQuery + MediaRequest *MediaRequestQuery + HSNotif *HistorySyncNotificationQuery + AvatarCache *AvatarCacheQuery + MatrixRTCCall *MatrixRTCCallQuery } func New(bridgeID networkid.BridgeID, db *dbutil.Database, log zerolog.Logger) *Database { @@ -51,5 +52,11 @@ func New(bridgeID networkid.BridgeID, db *dbutil.Database, log zerolog.Logger) * return &AvatarCacheEntry{} }), }, + MatrixRTCCall: &MatrixRTCCallQuery{ + BridgeID: bridgeID, + QueryHelper: dbutil.MakeQueryHelper(db, func(_ *dbutil.QueryHelper[*MatrixRTCCall]) *MatrixRTCCall { + return &MatrixRTCCall{} + }), + }, } } diff --git a/pkg/connector/wadb/message.go b/pkg/connector/wadb/message.go index 7a6c63d..4b16002 100644 --- a/pkg/connector/wadb/message.go +++ b/pkg/connector/wadb/message.go @@ -96,12 +96,12 @@ func (mq *MessageQuery) GetBetween(ctx context.Context, loginID networkid.UserLo args := []any{mq.BridgeID, loginID, chatJID} argNum := 4 if startTime != nil { - whereClauses += fmt.Sprintf(" AND timestamp > $%d", argNum) + whereClauses += fmt.Sprintf(" AND timestamp >= $%d", argNum) args = append(args, startTime.Unix()) argNum++ } if endTime != nil { - whereClauses += fmt.Sprintf(" AND timestamp < $%d", argNum) + whereClauses += fmt.Sprintf(" AND timestamp <= $%d", argNum) args = append(args, endTime.Unix()) } diff --git a/pkg/connector/wadb/upgrades/00-latest-schema.sql b/pkg/connector/wadb/upgrades/00-latest-schema.sql index 0f1fef9..648091c 100644 --- a/pkg/connector/wadb/upgrades/00-latest-schema.sql +++ b/pkg/connector/wadb/upgrades/00-latest-schema.sql @@ -1,4 +1,4 @@ --- v0 -> v10 (compatible with v3+): Latest revision +-- v0 -> v11 (compatible with v3+): Latest revision CREATE TABLE whatsapp_poll_option_id ( bridge_id TEXT NOT NULL, @@ -98,3 +98,40 @@ CREATE TABLE whatsapp_avatar_cache ( PRIMARY KEY (entity_jid, avatar_id) ); + +CREATE TABLE whatsapp_matrixrtc_call ( + bridge_id TEXT NOT NULL, + user_login_id TEXT NOT NULL, + wa_call_id TEXT NOT NULL, + room_id TEXT NOT NULL, + portal_id TEXT NOT NULL, + portal_receiver TEXT NOT NULL, + peer_jid TEXT NOT NULL, + direction TEXT NOT NULL, + media_kind TEXT NOT NULL, + focus_type TEXT NOT NULL, + livekit_service_url TEXT NOT NULL, + livekit_room TEXT, + matrix_participant_mxid TEXT, + matrix_session_id TEXT, + selected_publisher_id TEXT, + bridge_membership_event_id TEXT, + selected_membership_event_id TEXT, + bridge_hand_raise_event_id TEXT, + selected_hand_raise_event_id TEXT, + audio_policy TEXT NOT NULL, + state TEXT NOT NULL, + created_ts BIGINT NOT NULL, + joined_ts BIGINT, + answered_ts BIGINT, + ended_ts BIGINT, + end_reason TEXT, + last_error TEXT, + + PRIMARY KEY (bridge_id, user_login_id, wa_call_id), + CONSTRAINT whatsapp_matrixrtc_call_user_login_fkey FOREIGN KEY (bridge_id, user_login_id) + REFERENCES user_login (bridge_id, id) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT whatsapp_matrixrtc_call_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver) + REFERENCES portal (bridge_id, id, receiver) ON UPDATE CASCADE ON DELETE CASCADE +); +CREATE INDEX whatsapp_matrixrtc_call_room_idx ON whatsapp_matrixrtc_call (bridge_id, room_id, state); diff --git a/pkg/connector/wadb/upgrades/08-may-need-lid-dm-deletion.sql b/pkg/connector/wadb/upgrades/08-may-need-lid-dm-deletion.sql new file mode 100644 index 0000000..d5b4ca0 --- /dev/null +++ b/pkg/connector/wadb/upgrades/08-may-need-lid-dm-deletion.sql @@ -0,0 +1,2 @@ +-- v8 (compatible with v3+): Mark LID DMs for deletion +INSERT INTO kv_store (bridge_id, key, value) VALUES ('', 'whatsapp_lid_dms_deleted', 'false'); diff --git a/pkg/connector/wadb/upgrades/09-may-need-lid-dm-deletion-again.sql b/pkg/connector/wadb/upgrades/09-may-need-lid-dm-deletion-again.sql new file mode 100644 index 0000000..5c32e65 --- /dev/null +++ b/pkg/connector/wadb/upgrades/09-may-need-lid-dm-deletion-again.sql @@ -0,0 +1,3 @@ +-- v9 (compatible with v3+): Mark LID DMs for deletion (again) +DELETE FROM kv_store WHERE bridge_id='' AND key='whatsapp_lid_dms_deleted'; +INSERT INTO kv_store (bridge_id, key, value) VALUES ('', 'whatsapp_lid_dms_deleted', 'false'); diff --git a/pkg/connector/wadb/upgrades/10-history-sync-conversation-to-lid.sql b/pkg/connector/wadb/upgrades/10-history-sync-conversation-to-lid.sql deleted file mode 100644 index 334c597..0000000 --- a/pkg/connector/wadb/upgrades/10-history-sync-conversation-to-lid.sql +++ /dev/null @@ -1,22 +0,0 @@ --- v10 (compatible with v3+): Move history sync conversations to LIDs - --- Delete history sync conversations where a @lid conversation already exists -DELETE FROM whatsapp_history_sync_conversation -WHERE chat_jid LIKE '%@lid' AND EXISTS ( - SELECT 1 - FROM whatsapp_history_sync_conversation pnconv - WHERE pnconv.chat_jid=( - SELECT pn || '@s.whatsapp.net' - FROM whatsmeow_lid_map - WHERE lid=replace(whatsapp_history_sync_conversation.chat_jid, '@lid', '') - ) -); - --- Update all phone number conversations to lids if the lid is known -UPDATE whatsapp_history_sync_conversation -SET chat_jid=(SELECT lid || '@lid' FROM whatsmeow_lid_map WHERE pn=replace(chat_jid, '@s.whatsapp.net', '')) -WHERE chat_jid LIKE '%@s.whatsapp.net' - AND EXISTS (SELECT 1 FROM whatsmeow_lid_map WHERE pn=replace(chat_jid, '@s.whatsapp.net', '')); - --- Delete blank phone number portals -DELETE FROM portal WHERE id LIKE '%@s.whatsapp.net' AND (mxid IS NULL OR mxid='') AND room_type=''; diff --git a/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql b/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql new file mode 100644 index 0000000..0aba691 --- /dev/null +++ b/pkg/connector/wadb/upgrades/10-matrixrtc-call.sql @@ -0,0 +1,33 @@ +-- v10 (compatible with v3+): Add MatrixRTC/LiveKit call metadata +CREATE TABLE whatsapp_matrixrtc_call ( + bridge_id TEXT NOT NULL, + user_login_id TEXT NOT NULL, + wa_call_id TEXT NOT NULL, + room_id TEXT NOT NULL, + portal_id TEXT NOT NULL, + portal_receiver TEXT NOT NULL, + peer_jid TEXT NOT NULL, + direction TEXT NOT NULL, + media_kind TEXT NOT NULL, + focus_type TEXT NOT NULL, + livekit_service_url TEXT NOT NULL, + livekit_room TEXT, + matrix_participant_mxid TEXT, + matrix_session_id TEXT, + selected_publisher_id TEXT, + audio_policy TEXT NOT NULL, + state TEXT NOT NULL, + created_ts BIGINT NOT NULL, + joined_ts BIGINT, + answered_ts BIGINT, + ended_ts BIGINT, + end_reason TEXT, + last_error TEXT, + + PRIMARY KEY (bridge_id, user_login_id, wa_call_id), + CONSTRAINT whatsapp_matrixrtc_call_user_login_fkey FOREIGN KEY (bridge_id, user_login_id) + REFERENCES user_login (bridge_id, id) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT whatsapp_matrixrtc_call_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver) + REFERENCES portal (bridge_id, id, receiver) ON UPDATE CASCADE ON DELETE CASCADE +); +CREATE INDEX whatsapp_matrixrtc_call_room_idx ON whatsapp_matrixrtc_call (bridge_id, room_id, state); diff --git a/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql b/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql new file mode 100644 index 0000000..e91fa8c --- /dev/null +++ b/pkg/connector/wadb/upgrades/11-matrixrtc-reactions.sql @@ -0,0 +1,5 @@ +-- v11 (compatible with v3+): Persist MatrixRTC reaction relation targets +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN bridge_membership_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN selected_membership_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN bridge_hand_raise_event_id TEXT; +ALTER TABLE whatsapp_matrixrtc_call ADD COLUMN selected_hand_raise_event_id TEXT; diff --git a/pkg/connector/wadb/upgrades/upgrades_test.go b/pkg/connector/wadb/upgrades/upgrades_test.go new file mode 100644 index 0000000..1f04c00 --- /dev/null +++ b/pkg/connector/wadb/upgrades/upgrades_test.go @@ -0,0 +1,71 @@ +package upgrades + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +func TestMatrixRTCReactionUpgradeSQLite(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, statement := range []string{ + `CREATE TABLE user_login ( + bridge_id TEXT NOT NULL, + id TEXT NOT NULL, + PRIMARY KEY (bridge_id, id) + )`, + `CREATE TABLE portal ( + bridge_id TEXT NOT NULL, + id TEXT NOT NULL, + receiver TEXT NOT NULL, + PRIMARY KEY (bridge_id, id, receiver) + )`, + } { + if _, err = db.ExecContext(context.Background(), statement); err != nil { + t.Fatal(err) + } + } + + for _, name := range []string{"10-matrixrtc-call.sql", "11-matrixrtc-reactions.sql"} { + script, readErr := rawUpgrades.ReadFile(name) + if readErr != nil { + t.Fatal(readErr) + } + if _, err = db.ExecContext(context.Background(), string(script)); err != nil { + t.Fatalf("%s failed: %v", name, err) + } + } + + rows, err := db.QueryContext(context.Background(), "PRAGMA table_info(whatsapp_matrixrtc_call)") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var cid, notNull, primaryKey int + var name, dataType string + var defaultValue any + if err = rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &primaryKey); err != nil { + t.Fatal(err) + } + columns[name] = true + } + for _, name := range []string{ + "bridge_membership_event_id", + "selected_membership_event_id", + "bridge_hand_raise_event_id", + "selected_hand_raise_event_id", + } { + if !columns[name] { + t.Fatalf("upgraded MatrixRTC call table is missing %s", name) + } + } +} diff --git a/pkg/msgconv/from-matrix.go b/pkg/msgconv/from-matrix.go index 57f9113..49aedf8 100644 --- a/pkg/msgconv/from-matrix.go +++ b/pkg/msgconv/from-matrix.go @@ -150,7 +150,7 @@ func (mc *MessageConverter) ToWhatsApp( } lid := parsedID.Sender if lid.Server == types.DefaultUserServer { - lid, err = client.Store.LIDs.GetLIDForPN(ctx, lid) + lid, err = client.Store.LIDs.GetLIDForPN(ctx, parsedID.Sender) if err != nil { return nil, nil, fmt.Errorf("failed to get LID for PN: %w", err) } diff --git a/pkg/msgconv/from-whatsapp.go b/pkg/msgconv/from-whatsapp.go index e912732..e8cb032 100644 --- a/pkg/msgconv/from-whatsapp.go +++ b/pkg/msgconv/from-whatsapp.go @@ -136,6 +136,7 @@ func (mc *MessageConverter) ToMatrix( waMsg *waE2E.Message, rawWaMsg *waE2E.Message, info *types.MessageInfo, + origSource *types.MessageSource, isViewOnce bool, isBackfill bool, previouslyConvertedPart *bridgev2.ConvertedMessagePart, @@ -182,7 +183,7 @@ func (mc *MessageConverter) ToMatrix( case waMsg.PollCreationMessageV6 != nil: part, contextInfo = mc.convertPollCreationMessage(ctx, waMsg.PollCreationMessageV6) case waMsg.PollUpdateMessage != nil: - part, contextInfo = mc.convertPollUpdateMessage(ctx, info, waMsg.PollUpdateMessage) + part, contextInfo = mc.convertPollUpdateMessage(ctx, info, origSource, waMsg.PollUpdateMessage) case waMsg.EventMessage != nil: part, contextInfo = mc.convertEventMessage(ctx, waMsg.EventMessage) case waMsg.PinInChatMessage != nil: @@ -271,26 +272,30 @@ func (mc *MessageConverter) ToMatrix( if chat.IsEmpty() { chat, _ = waid.ParsePortalID(portal.ID) } + // We reroute all DMs to the phone number JID, so reroute reply participants too + pcp = rerouteMessageKey(ctx, chat, pcp, getPortal(ctx).Metadata.(*waid.PortalMetadata).AddressingMode == types.AddressingModeLID) + if store := getClient(ctx).Store; store != nil && chat.Server == types.DefaultUserServer && pcp.Server == types.HiddenUserServer { + pcpPN, _ := store.LIDs.GetPNForLID(ctx, pcp) + zerolog.Ctx(ctx).Debug(). + Stringer("orig_participant", pcp). + Stringer("rerouted_participant", pcpPN). + Msg("Rerouting reply target (PN recipient in LID DM)") + if !pcpPN.IsEmpty() { + pcp = pcpPN + } + } else if store != nil && chat.Server == types.GroupServer && pcp.Server == types.DefaultUserServer && getPortal(ctx).Metadata.(*waid.PortalMetadata).AddressingMode == types.AddressingModeLID { + pcpLID, _ := store.LIDs.GetLIDForPN(ctx, pcp) + zerolog.Ctx(ctx).Debug(). + Stringer("orig_participant", pcp). + Stringer("rerouted_participant", pcpLID). + Msg("Rerouting reply target (PN recipient in LID group)") + if !pcpLID.IsEmpty() { + pcp = pcpLID + } + } cm.ReplyTo = &networkid.MessageOptionalPartID{ MessageID: waid.MakeMessageID(chat, pcp, contextInfo.GetStanzaID()), } - var pn, lid types.JID - if pcp.Server == types.DefaultUserServer { - pn = pcp - lid, _ = client.Store.LIDs.GetLIDForPN(ctx, pcp) - } else if pcp.Server == types.HiddenUserServer { - lid = pcp - pn, _ = client.Store.LIDs.GetPNForLID(ctx, pcp) - } else if pcp.Server == types.BotServer { - lid = pcp - } - if !pn.IsEmpty() { - cm.ReplyToLogin = waid.MakeUserLoginID(pn) - } - if !lid.IsEmpty() { - cm.ReplyToUser = waid.MakeUserID(lid) - } - // TODO set reply to room } if contextInfo.GetIsForwarded() { hasCaption := part.Content.FileName != "" && part.Content.FileName != part.Content.Body diff --git a/pkg/msgconv/wa-poll.go b/pkg/msgconv/wa-poll.go index 881a5fb..dfe1a25 100644 --- a/pkg/msgconv/wa-poll.go +++ b/pkg/msgconv/wa-poll.go @@ -94,7 +94,31 @@ func (mc *MessageConverter) convertPollCreationMessage(ctx context.Context, msg }, msg.GetContextInfo() } +func rerouteMessageKey(ctx context.Context, chat, sender types.JID, groupLIDAddressing bool) types.JID { + if store := getClient(ctx).Store; store != nil && chat.Server == types.DefaultUserServer && sender.Server == types.HiddenUserServer { + senderPN, _ := store.LIDs.GetPNForLID(ctx, sender) + zerolog.Ctx(ctx).Debug(). + Stringer("orig_participant", sender). + Stringer("rerouted_participant", senderPN). + Msg("Rerouting message key (PN recipient in LID DM)") + if !senderPN.IsEmpty() { + return senderPN + } + } else if store != nil && chat.Server == types.GroupServer && sender.Server == types.DefaultUserServer && groupLIDAddressing { + senderLID, _ := store.LIDs.GetLIDForPN(ctx, sender) + zerolog.Ctx(ctx).Debug(). + Stringer("orig_participant", sender). + Stringer("rerouted_participant", senderLID). + Msg("Rerouting message key (PN recipient in LID group)") + if !senderLID.IsEmpty() { + return senderLID + } + } + return sender +} + func KeyToMessageID(ctx context.Context, client *whatsmeow.Client, chat, sender types.JID, key *waCommon.MessageKey) networkid.MessageID { + groupLIDAddressing := sender.Server == types.HiddenUserServer sender = sender.ToNonAD() var err error if !key.GetFromMe() { @@ -107,7 +131,7 @@ func KeyToMessageID(ctx context.Context, client *whatsmeow.Client, chat, sender if sender.Server == types.LegacyUserServer { sender.Server = types.DefaultUserServer } - } else if chat.Server == types.DefaultUserServer || chat.Server == types.HiddenUserServer || chat.Server == types.BotServer { + } else if chat.Server == types.DefaultUserServer || chat.Server == types.BotServer { if sender.User == client.Store.GetJID().User || sender.User == client.Store.GetLID().User { // Message key is not from the sender, but message sender (containing key) is me, // so message key sender is the other user in the DM @@ -115,11 +139,7 @@ func KeyToMessageID(ctx context.Context, client *whatsmeow.Client, chat, sender } else { // Message key is not from the sender, but message sender (containing key) is not me, // so message key sender is me - if chat.Server == types.HiddenUserServer { - sender = client.Store.GetLID().ToNonAD() - } else { - sender = client.Store.GetJID().ToNonAD() - } + sender = client.Store.GetJID().ToNonAD() } } else { zerolog.Ctx(ctx).Warn(). @@ -137,6 +157,10 @@ func KeyToMessageID(ctx context.Context, client *whatsmeow.Client, chat, sender chat = remoteJID } } + sender = rerouteMessageKey( + context.WithValue(ctx, contextKeyClient, client), + chat, sender, groupLIDAddressing, + ) return waid.MakeMessageID(chat, sender, key.GetID()) } @@ -146,7 +170,7 @@ var failedPollUpdatePart = &bridgev2.ConvertedMessagePart{ DontBridge: true, } -func (mc *MessageConverter) convertPollUpdateMessage(ctx context.Context, info *types.MessageInfo, msg *waE2E.PollUpdateMessage) (*bridgev2.ConvertedMessagePart, *waE2E.ContextInfo) { +func (mc *MessageConverter) convertPollUpdateMessage(ctx context.Context, info *types.MessageInfo, origSource *types.MessageSource, msg *waE2E.PollUpdateMessage) (*bridgev2.ConvertedMessagePart, *waE2E.ContextInfo) { log := zerolog.Ctx(ctx) pollMessageID := KeyToMessageID(ctx, getClient(ctx), info.Chat, info.Sender, msg.PollCreationMessageKey) pollMessage, err := mc.Bridge.DB.Message.GetPartByID(ctx, getPortal(ctx).Receiver, pollMessageID, "") @@ -157,8 +181,12 @@ func (mc *MessageConverter) convertPollUpdateMessage(ctx context.Context, info * log.Warn().Str("target_message_id", string(pollMessageID)).Msg("Poll update target message not found") return failedPollUpdatePart, nil } + infoForDecrypt := *info + if origSource != nil { + infoForDecrypt.MessageSource = *origSource + } vote, err := getClient(ctx).DecryptPollVote(ctx, &events.Message{ - Info: *info, + Info: infoForDecrypt, Message: &waE2E.Message{PollUpdateMessage: msg}, }) if err != nil { diff --git a/pkg/waid/id.go b/pkg/waid/id.go index 7f8fe5b..e522f5f 100644 --- a/pkg/waid/id.go +++ b/pkg/waid/id.go @@ -83,13 +83,6 @@ func MakeMessageID(chat, sender types.JID, id types.MessageID) networkid.Message return networkid.MessageID(fmt.Sprintf("%s:%s:%s", chat.ToNonAD().String(), sender.ToNonAD().String(), id)) } -func MakeMessageIDWithAltSender(chat, sender, altSender types.JID, id types.MessageID) networkid.MessageID { - if chat.Server == types.HiddenUserServer && sender.Server == types.DefaultUserServer && altSender.Server == types.HiddenUserServer { - sender = altSender - } - return MakeMessageID(chat, sender, id) -} - func MakeFakeMessageID(chat, sender types.JID, data string) networkid.MessageID { return networkid.MessageID(fmt.Sprintf("fake:%s:%s:%s", chat.ToNonAD().String(), sender.ToNonAD().String(), data)) }