From c63981be768f02aa11abc2505a7a6197de12bd38 Mon Sep 17 00:00:00 2001 From: Indent Date: Sun, 8 Nov 2026 06:27:35 +0000 Subject: [PATCH 1/6] backfill: don't requeue history sync conversations that are still being created --- pkg/connector/backfill.go | 9 +++++++++ pkg/connector/client.go | 2 ++ 2 files changed, 11 insertions(+) diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index cbc7eec8..2d335352 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -442,6 +442,9 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { // We don't currently support new PSAs, so don't bother backfilling them either wg.Done() continue + } else if wa.historySyncQueued.Has(conv.ChatJID) { + wg.Done() + continue } // TODO can the chat info fetch be avoided entirely? select { @@ -482,6 +485,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { wg.Done() continue } + wa.historySyncQueued.Add(conv.ChatJID) res := wa.UserLogin.QueueRemoteEvent(&simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventChatResync, @@ -497,6 +501,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { if err != nil { zerolog.Ctx(ctx).Err(err).Msg("Failed to mark conversation as bridged") } + wa.historySyncQueued.Remove(conv.ChatJID) wg.Done() }, }, @@ -504,6 +509,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { LatestMessageTS: conv.LastMessageTimestamp, }) if !res.Success { + wa.historySyncQueued.Remove(conv.ChatJID) log.Debug().Msg("Cancelling history sync portal creation loop") return } @@ -511,6 +517,9 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { log.Info().Int("conversation_count", len(conversations)).Msg("Finished creating portals from history sync") go func() { wg.Wait() + if wa.historySyncQueued.Size() > 0 { + return + } wa.UserLogin.Metadata.(*waid.UserLoginMetadata).HistorySyncPortalsNeedCreating = false err = wa.UserLogin.Save(ctx) if err != nil { diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 678c8d22..3630972b 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -50,6 +50,7 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. MC: noopMCInstance, historySyncWakeup: make(chan struct{}, 1), + historySyncQueued: exsync.NewSet[types.JID](), resyncQueue: make(map[types.JID]resyncQueueItem), directMediaRetries: make(map[networkid.MessageID]*directMediaRetry), mediaRetryLock: semaphore.NewWeighted(wa.Config.HistorySync.MediaRequests.MaxAsyncHandle), @@ -114,6 +115,7 @@ type WhatsAppClient struct { MC mClient historySyncWakeup chan struct{} + historySyncQueued *exsync.Set[types.JID] stopLoops atomic.Pointer[context.CancelFunc] resyncQueue map[types.JID]resyncQueueItem resyncQueueLock sync.Mutex From efdaad906581d8db6a24dd0a6c19a191711f5bbe Mon Sep 17 00:00:00 2001 From: Indent Date: Sun, 8 Nov 2026 07:23:11 +0000 Subject: [PATCH 2/6] backfill: let in-flight history sync entries expire so failed creations are retried --- pkg/connector/backfill.go | 12 +++++++----- pkg/connector/client.go | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index 2d335352..e2a18b18 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -399,6 +399,8 @@ func (wa *WhatsAppClient) handleWAHistorySync( return nil } +const historySyncQueuedTimeout = 10 * time.Minute + func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { log := wa.UserLogin.Log.With(). Str("action", "create portals from history sync"). @@ -442,7 +444,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { // We don't currently support new PSAs, so don't bother backfilling them either wg.Done() continue - } else if wa.historySyncQueued.Has(conv.ChatJID) { + } else if queuedAt, ok := wa.historySyncQueued.Get(conv.ChatJID); ok && time.Since(queuedAt) < historySyncQueuedTimeout { wg.Done() continue } @@ -485,7 +487,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { wg.Done() continue } - wa.historySyncQueued.Add(conv.ChatJID) + wa.historySyncQueued.Set(conv.ChatJID, time.Now()) res := wa.UserLogin.QueueRemoteEvent(&simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ Type: bridgev2.RemoteEventChatResync, @@ -501,7 +503,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { if err != nil { zerolog.Ctx(ctx).Err(err).Msg("Failed to mark conversation as bridged") } - wa.historySyncQueued.Remove(conv.ChatJID) + wa.historySyncQueued.Delete(conv.ChatJID) wg.Done() }, }, @@ -509,7 +511,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { LatestMessageTS: conv.LastMessageTimestamp, }) if !res.Success { - wa.historySyncQueued.Remove(conv.ChatJID) + wa.historySyncQueued.Delete(conv.ChatJID) log.Debug().Msg("Cancelling history sync portal creation loop") return } @@ -517,7 +519,7 @@ func (wa *WhatsAppClient) createPortalsFromHistorySync(ctx context.Context) { log.Info().Int("conversation_count", len(conversations)).Msg("Finished creating portals from history sync") go func() { wg.Wait() - if wa.historySyncQueued.Size() > 0 { + if wa.historySyncQueued.Len() > 0 { return } wa.UserLogin.Metadata.(*waid.UserLoginMetadata).HistorySyncPortalsNeedCreating = false diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 3630972b..9b3f190a 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -50,7 +50,7 @@ func (wa *WhatsAppConnector) LoadUserLogin(ctx context.Context, login *bridgev2. MC: noopMCInstance, historySyncWakeup: make(chan struct{}, 1), - historySyncQueued: exsync.NewSet[types.JID](), + historySyncQueued: exsync.NewMap[types.JID, time.Time](), resyncQueue: make(map[types.JID]resyncQueueItem), directMediaRetries: make(map[networkid.MessageID]*directMediaRetry), mediaRetryLock: semaphore.NewWeighted(wa.Config.HistorySync.MediaRequests.MaxAsyncHandle), @@ -115,7 +115,7 @@ type WhatsAppClient struct { MC mClient historySyncWakeup chan struct{} - historySyncQueued *exsync.Set[types.JID] + historySyncQueued *exsync.Map[types.JID, time.Time] stopLoops atomic.Pointer[context.CancelFunc] resyncQueue map[types.JID]resyncQueueItem resyncQueueLock sync.Mutex From 5c8c1fc8358c011891fbf6066f289b9ad5fee223 Mon Sep 17 00:00:00 2001 From: "indent[bot]" <297468268+indent[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:09:28 +0000 Subject: [PATCH 3/6] handlewhatsapp: ignore HD replacement messages without replacement content (#1037) Co-authored-by: Indent --- pkg/connector/handlewhatsapp.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/connector/handlewhatsapp.go b/pkg/connector/handlewhatsapp.go index 4391233b..e6174b92 100644 --- a/pkg/connector/handlewhatsapp.go +++ b/pkg/connector/handlewhatsapp.go @@ -374,6 +374,14 @@ func (wa *WhatsAppClient) handleWAMessage(ctx context.Context, evt *events.Messa if assocType := messageAssoc.GetAssociationType(); assocType == waE2E.MessageAssociation_HD_IMAGE_DUAL_UPLOAD || assocType == waE2E.MessageAssociation_HD_VIDEO_DUAL_UPLOAD { parentKey := messageAssoc.GetParentMessageKey() protocolMsg, shouldHideEdit := makeHDMediaReplacementEdit(evt.Message, parentKey) + if protocolMsg.GetEditedMessage() == nil { + wa.UserLogin.Log.Debug(). + Str("message_id", evt.Info.ID). + Str("parent_id", parentKey.GetID()). + Stringer("assoc_type", assocType). + Msg("Ignoring HD replacement message without replacement content") + return + } dontRenderEdited = shouldHideEdit wa.UserLogin.Log.Debug(). Str("message_id", evt.Info.ID). From 945474343fcd7579c3d273acb78f6c60c957e2e0 Mon Sep 17 00:00:00 2001 From: "indent[bot]" <297468268+indent[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:09:56 +0000 Subject: [PATCH 4/6] backfill: log stub and message type of unparseable history messages (#1034) Co-authored-by: Indent --- pkg/connector/backfill.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index cbc7eec8..8f006e0b 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -298,6 +298,9 @@ func (wa *WhatsAppClient) handleWAHistorySync( Int("msg_index", i). Str("msg_id", rawMsg.GetMessage().GetKey().GetID()). Uint64("msg_time_seconds", rawMsg.GetMessage().GetMessageTimestamp()). + Stringer("stub_type", rawMsg.GetMessage().GetMessageStubType()). + Str("message_type", getMessageType(rawMsg.GetMessage().GetMessage())). + Bool("from_me", rawMsg.GetMessage().GetKey().GetFromMe()). Msg("Dropping historical message due to parse error") log.Trace(). Any("web_message_info", rawMsg.GetMessage()). From a15089b18d41c3f3692d72b10d96d8a2aa9b7432 Mon Sep 17 00:00:00 2001 From: "indent[bot]" <297468268+indent[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:19:41 -0700 Subject: [PATCH 5/6] backfill: skip unparseable history messages without content (#1046) Co-authored-by: Indent --- pkg/connector/backfill.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index 8f006e0b..e2c7d114 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -293,7 +293,10 @@ func (wa *WhatsAppClient) handleWAHistorySync( for i, rawMsg := range rawMessages { // Don't store messages that will just be skipped. msgEvt, err := wa.Client.ParseWebMessage(jid, rawMsg.GetMessage()) - if err != nil { + if err != nil && getMessageType(rawMsg.GetMessage().GetMessage()) == "ignore" { + ignoredTypes++ + continue + } else if err != nil { log.Warn().Err(err). Int("msg_index", i). Str("msg_id", rawMsg.GetMessage().GetKey().GetID()). From 015ad733a3ef8a8776266b0c490ef6c740fe2ed8 Mon Sep 17 00:00:00 2001 From: "indent[bot]" <297468268+indent[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:53:07 +0000 Subject: [PATCH 6/6] msgconv: log size breakdown of converted parts over the cut-off limit (#1047) Co-authored-by: Indent --- pkg/msgconv/from-whatsapp.go | 1 + pkg/msgconv/wa-text.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pkg/msgconv/from-whatsapp.go b/pkg/msgconv/from-whatsapp.go index db18f35a..ec273076 100644 --- a/pkg/msgconv/from-whatsapp.go +++ b/pkg/msgconv/from-whatsapp.go @@ -322,6 +322,7 @@ func (mc *MessageConverter) ToMatrix( cm.ThreadRoot = ptr.Ptr(waid.MakeMessageID(chat, pcp, commentTarget.GetID())) } mc.cutOffLongMessage(part.Content) + logOversizedPart(ctx, part) return cm } diff --git a/pkg/msgconv/wa-text.go b/pkg/msgconv/wa-text.go index 52874fd8..7afcc81f 100644 --- a/pkg/msgconv/wa-text.go +++ b/pkg/msgconv/wa-text.go @@ -18,6 +18,7 @@ package msgconv import ( "context" + "encoding/json" "fmt" "html" "regexp" @@ -25,6 +26,7 @@ import ( "strconv" "strings" + "github.com/rs/zerolog" "go.mau.fi/whatsmeow/proto/waE2E" "maunium.net/go/mautrix/bridgev2" "maunium.net/go/mautrix/event" @@ -72,6 +74,36 @@ func (mc *MessageConverter) cutOffLongMessage(content *event.MessageEventContent } } +func logOversizedPart(ctx context.Context, part *bridgev2.ConvertedMessagePart) { + if part == nil || part.Content == nil { + return + } + contentJSON, _ := json.Marshal(part.Content) + extraJSON, _ := json.Marshal(part.Extra) + if len(contentJSON)+len(extraJSON) <= 40000 { + return + } + extraKeys := make([]string, 0, len(part.Extra)) + for key := range part.Extra { + extraKeys = append(extraKeys, key) + } + var mentionCount int + if part.Content.Mentions != nil { + mentionCount = len(part.Content.Mentions.UserIDs) + } + linkPreviewJSON, _ := json.Marshal(part.Content.BeeperLinkPreviews) + zerolog.Ctx(ctx).Warn(). + Str("msg_type", string(part.Content.MsgType)). + Int("content_json_size", len(contentJSON)). + Int("extra_json_size", len(extraJSON)). + Int("body_size", len(part.Content.Body)). + Int("formatted_body_size", len(part.Content.FormattedBody)). + Int("mention_count", mentionCount). + Int("link_preview_json_size", len(linkPreviewJSON)). + Strs("extra_keys", extraKeys). + Msg("Converted message part is larger than the cut-off limit") +} + func (mc *MessageConverter) parseFormatting(content *event.MessageEventContent, allowInlineURL, forceHTML bool) { parsedHTML := parseWAFormattingToHTML(content.Body, allowInlineURL) if forceHTML || parsedHTML != event.TextToHTML(content.Body) {