DB server memory usage gradually increases after starting Replication Manager (DB is remote)

[abi780]

Environment

• Replication Manager Version: v2.3.53 (Build: 2024-11-04)
• Replication Manager Host: 192.168.8.215
• Database Version: MySQL 5.6.49-enterprise-commercial-advanced-log
• Primary DB: 192.168.8.171
• Monitoring: Disabled (monitoring-scheduler = false)
• Failover Mode: Manual
• Memory Config: prov-db-memory = "256"

Problem

After starting Replication Manager (running on a separate host), memory usage on the DB server (192.168.8.171) gradually increases over time, even when:

• Monitoring is turned off
• No backups or scheduled jobs are enabled
• Queries per second (QPS) is already high (~34,000 QPS)
• Uptime of DB is stable (>200 days)

The only change is starting Replication Manager — stopping it halts the memory growth.

Notes

• Config file disables most features: no failover polling, no scheduled backups.
• The DB servers are defined in cluster1.toml.
• log-sql-in-monitoring = false is set.

##Config_file

[cygnet@mipl-p-ha-iams-vims replication-manager]$ hostname -i
192.168.8.215
[cygnet@mipl-p-ha-iams-vims replication-manager]$ cat cluster.d/
bkp-cluster1-13-06-2025.toml  cluster1.toml                 cluster1.toml.sample          
[cygnet@mipl-p-ha-iams-vims replication-manager]$ cat cluster.d/cluster1.toml
[cluster1]
title = "cluster1"
prov-orchestrator = "onpremise"
#prov-db-tags = "innodb,noquerycache,slow,pfs,pkg,linux,smallredolog,logtotable"
#prov-db-tags = "innodb,noquerycache,pkg,linux"
#prov-db-databases = "mysql,performance_schema"
#prov-db-ignore-schemas = "CygNetMicroScanPRODTXDB15Jun2022"
#prov-db-databases = "mysql,performance_schema,information_schema"
prov-db-ignore-schemas = "CygNetMicroScanPRODTXDB15Jun2022,MIPLPMSDB25FEB25"
prov-db-memory = "256"
prov-db-memory-shared-pct = "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0"
prov-db-disk-size = "1"
prov-db-cpu-cores = "1"
prov-db-disk-iops = "300"

db-servers-hosts = "192.168.8.171:3306,192.168.8.212:3306"
db-servers-prefered-master = "192.168.8.171:3306"
db-servers-credential = "repluser:12qwaszx"
db-servers-connect-timeout = 1
replication-credential = "repluser:12qwaszx"

verbose = false
log-failed-election  = true
log-level = 1
log-rotate-max-age = 7
log-rotate-max-backup = 7
log-rotate-max-size = 5
#log-sql-in-monitoring   = true
log-sql-in-monitoring   = false
log-sst = true

##############
## TOPOLOGY ##
##############

replication-multi-master = false
replication-multi-tier-slave = false

############
# BACKUPS ##
###########


backup-streaming = false
backup-streaming-aws-access-key-id = "admin"
backup-streaming-aws-access-secret = "xxxx"
backup-streaming-endpoint= "https://s3.signal18.io/"
backup-streaming-region= "fr-1"
backup-streaming-bucket= "repman"

#backup-restic = true
backup-restic = false
backup-restic-aws =  false
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
backup-restic-aws-access-secret = "xxxx"
backup-restic-password = "xxxx"
backup-restic-binary-path = "/usr/local/bin/restic"

#monitoring-scheduler = true
monitoring-scheduler = false
scheduler-db-servers-logical-backup  = false
scheduler-db-servers-logical-backup-cron= "0 0 1 * * 6"
scheduler-db-servers-logs   =  false
scheduler-db-servers-logs-cron = "0 0 * * * *"
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate  = false
scheduler-db-servers-logs-table-rotate-cron = "0 0 0/6 * * *"
scheduler-db-servers-optimize  = false
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = false
scheduler-db-servers-physical-backup-cron = "0 0 0 * * *"

##############
## FAILOVER ##
##############

#failover-mode = "automatic"
failover-mode = "manual"
failover-pre-script = "/root/pre_failover.expect"
failover-post-script = "/root/post_failover.expect"

## Slaves will re enter with read-only

failover-readonly-state = true
failover-event-scheduler = false
failover-event-status = false

## Failover after N failures detection

failover-falsepositive-ping-counter = 5

## Cancel failover if already N failover
## Cancel failover if last failover was N seconds before
## Cancel failover in semi-sync when one slave is not in sync
## Cancel failover if one slave receive master heartbeat
## Cancel failover when replication delay is more than N seconds

failover-limit = 0
failover-time-limit = 0
failover-at-sync = false
failover-max-slave-delay = 30
failover-restart-unsafe = false

# failover-falsepositive-heartbeat = true
# failover-falsepositive-heartbeat-timeout = 3
# failover-falsepositive-maxscale = false
# failover-falsepositive-maxscale-timeout = 14
# failover-falsepositive-external = false
# failover-falsepositive-external-port = 80

################
## SWITCHOVER ##
################

## In switchover Wait N milliseconds before killing long running transactions
## Cancel switchover if transaction running more than N seconds
## Cancel switchover if write query running more than N seconds
## Cancel switchover if one of the slaves is not synced based on GTID equality

switchover-wait-kill = 5000
switchover-wait-trx = 10
switchover-wait-write-query = 10
switchover-at-equal-gtid = false
switchover-at-sync = false
switchover-max-slave-delay = 30

############
## REJOIN ##
############

autorejoin = true
autorejoin-script = ""
autorejoin-semisync = true
autorejoin-backup-binlog = true
autorejoin-flashback = false
autorejoin-mysqldump = false

####################
## CHECKS & FORCE ##
####################

#check-replication-filters = true
#check-binlog-filters = true
#check-replication-state = true

check-replication-filters = false
check-binlog-filters = false
check-replication-state = false


force-slave-heartbeat= false
force-slave-heartbeat-retry = 5
force-slave-heartbeat-time = 3
force-slave-gtid-mode = false
force-slave-semisync = false
force-slave-failover-readonly-state = false
force-binlog-row = false
force-binlog-annotate = false
force-binlog-slowqueries = false
force-binlog-compress = false
force-binlog-checksum = false
force-inmemory-binlog-cache-size = false
force-disk-relaylog-size-limit = false
force-disk-relaylog-size-limit-size = 1000000000
force-sync-binlog = false
force-sync-innodb = false

##############
## MAXSCALE ##
##############

## for 2 nodes cluster maxscale can be driven by replication manager

maxscale = false
maxscale-binlog = false
maxscale-servers = "192.168.0.201"
maxscale-port = 4003
maxscale-user = "admin"
maxscale-pass = "mariadb"

## When true replication manager drive maxscale server state
## Not required unless multiple maxscale or release does not support detect_stale_slave

maxscale-disable-monitor = false

## maxinfo|maxadmin

maxscale-get-info-method = "maxadmin"
maxscale-maxinfo-port = 4002

maxscale-write-port = 4007
maxscale-read-port = 4008
maxscale-read-write-port = 4006
maxscale-binlog-port = 4000

#############
## HAPROXY ##
#############

## Wrapper mode unless maxscale or proxysql required to be located with replication-manager

haproxy = false
haproxy-binary-path = "/usr/sbin/haproxy"

## Read write traffic
## Read only load balance least connection traffic
haproxy-write-port = 3306
haproxy-read-port = 3307

####################
## SHARDING PROXY ##
####################

mdbshardproxy = false
mdbshardproxy-hosts = "127.0.0.1:3306"
mdbshardproxy-user = "root:mariadb"

Request

1. Can you confirm if Replication Manager performs any background replication checks even with monitoring and backups disabled?
2. Is this memory behavior expected in v2.3.53?
3. Any flags or config options to reduce background DB activity further?

Thank you.

[svaroqui]

Hello thanks for that report, Please report on version 3.1 It is probably caused by schema-monitoring that was enable by default in old versions , we also improved a lot the processlist information to be less aggressive some major changes in 3.0 was also moving to context queries that would make such accumulation for connections less possible

[svaroqui]

The list of features that you can disable can be found replication-manager-osc monitor --help | grep monito

[svaroqui]

monitoring-schema-change monitoring-capture monitoring-innodb-status monitoring-long-query-with-process monitoring-long-query-with-table monitoring-performance-schema monitoring-processlist monitoring-variable-diff monitoring-plugins are all the options you could disable to see what is causing the issue don't remember what you get in that release

[svaroqui]

We have 99% of our sponsor using MariaDB so far, so good to see a serious user on MySQL FYI the 3.0 and 3.1 code have improved on the errant transaction detection that can make you failover process failed on this release you are using

[abi780]

Thanks for your responce
can u pls share the download link for latest replication manager

[svaroqui]

https://ci.signal18.io/mrm/builds/tags/

[abi780]

Issue Persists After Updating to Latest Replication Manager Version

As per your suggestion, we have replaced the existing Replication Manager with the latest version. However, we are still facing the same issue even with the newer version.

Kindly assist us in resolving this.

##DB_server_Memory

[cygnet@mipl-p-m-oss-bss-db ~]$ free -g
total used free shared buff/cache available
Mem: 204 81 48 3 74 117
Swap: 0 0 0
[cygnet@mipl-p-m-oss-bss-db ~]$ free -m
total used free shared buff/cache available
Mem: 209287 83477 49384 3922 76425 120040
Swap: 0 0 0
[cygnet@mipl-p-m-oss-bss-db ~]$ free -m
total used free shared buff/cache available
Mem: 209287 83483 49528 3922 76275 120033
Swap: 0 0 0
[cygnet@mipl-p-m-oss-bss-db ~]$ free -m
total used free shared buff/cache available
Mem: 209287 83485 49531 3922 76270 120031
Swap: 0 0 0
[cygnet@mipl-p-m-oss-bss-db ~]$ free -m
total used free shared buff/cache available
Mem: 209287 83485 49519 3922 76281 120030
Swap: 0 0 0
[cygnet@mipl-p-m-oss-bss-db ~]$ free -m
total used free shared buff/cache available
Mem: 209287 83503 49458 3922 76325 120013
Swap: 0 0 0

##Replicationmanager_version

[cygnet@mipl-p-ha-iams-vims ~]$ replication-manager-cli version
Replication Manager v3.1.5 for MariaDB 10.x and MySQL 5.7 Series
Full Version: v3.1.5
Build Time: 2025-06-12T10:21:44+0000

[caffeinated92]

Can you try to disable monitoring-schema-change?

…

On Tue, Jul 1, 2025, 18:16 abi780 ***@***.***> wrote: *abi780* left a comment (signal18/replication-manager#1155) <#1155 (comment)> Issue Persists After Updating to Latest Replication Manager Version As per your suggestion, we have replaced the existing Replication Manager with the latest version. However, we are still facing the same issue even with the newer version. Kindly assist us in resolving this. ##DB_server_Memory ***@***.*** ~]$ free -g total used free shared buff/cache available Mem: 204 81 48 3 74 117 Swap: 0 0 0 ***@***.*** ~]$ free -m total used free shared buff/cache available Mem: 209287 83477 49384 3922 76425 120040 Swap: 0 0 0 ***@***.*** ~]$ free -m total used free shared buff/cache available Mem: 209287 83483 49528 3922 76275 120033 Swap: 0 0 0 ***@***.*** ~]$ free -m total used free shared buff/cache available Mem: 209287 83485 49531 3922 76270 120031 Swap: 0 0 0 ***@***.*** ~]$ free -m total used free shared buff/cache available Mem: 209287 83485 49519 3922 76281 120030 Swap: 0 0 0 ***@***.*** ~]$ free -m total used free shared buff/cache available Mem: 209287 83503 49458 3922 76325 120013 Swap: 0 0 0 ##Replicationmanager_version ***@***.*** ~]$ replication-manager-cli version Replication Manager v3.1.5 for MariaDB 10.x and MySQL 5.7 Series Full Version: v3.1.5 Build Time: 2025-06-12T10:21:44+0000 — Reply to this email directly, view it on GitHub <#1155 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/BFCXGAYYL2RBJWW4D2RXBJT3GJUYXAVCNFSM6AAAAACANL76CSVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZTAMRTGQ4DEOBZHA> . You are receiving this because you are subscribed to this thread.Message ID: ***@***.***>

[svaroqui]

Yes and f you have performance schema i'm sure MariaDB fixed tons of memory leak bugs in that zone , in any case this is to look after what queries are consuming the memory, can be from slow log capture in tables , performance schema , mysql binlog copy , table structure monitoring , all this can be disable if the leak happens in show slave status you have not other choice than trying to found a better MySQL version, do you found some sort of pending connections in sleep in show processlist from the replication-manager host , i think w need to found out if connections are piling and because not properly closed would consume memory , or if this is one query send by replication-manager that make this MySQL release memory leack

[svaroqui]

Also from the free report i don't see memory leak but more usage of the FS Cache , could it just be because you have now enable binlogs for the replication

[svaroqui]

You can join us on meet.signal18.io via first creating on account in gitlab.signal18.io

[abi780]

we have found monitoring sheduling only

[root@mipl-p-ha-iams-vims INFRA]# cat cluster1.toml|grep -i "monitoring"
#log-sql-in-monitoring = true
log-sql-in-monitoring = false
#monitoring-scheduler = true
monitoring-scheduler = false
[root@mipl-p-ha-iams-vims INFRA]#

[tanji]

@abi780 can you attach the output of ps waux when increased memory usage is witnessed?

[abi780]

ps.txt

[abi780]

Whenever we start the Replication Manager, we observe that the attached queries begin running automatically. This behavior is consistent and occurs on each startup.

Could you please verify if this is expected behavior? If not, kindly assist in resolving this issue.

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 4 NULL

124699661 repluser 192.168.8.215:44142 NULL Query 0 Opening tables /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='CRM_BEFORE_ENT_CHANGE_250CT24'

124699662 repluser 192.168.8.215:44154 NULL Sleep 0 NULL

124699699 repluser 192.168.8.215:44160 NULL Sleep 0 NULL

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 291 NULL

124700898 repluser 192.168.8.215:42538 NULL Query 151 Writing to net /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='MIPLPMSDB25FEB25'

124702107 repluser 192.168.8.215:34580 NULL Sleep 0 NULL

124702115 repluser 192.168.8.215:34612 NULL Sleep 1 NULL

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 292 NULL

124700898 repluser 192.168.8.215:42538 NULL Query 152 Writing to net /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='MIPLPMSDB25FEB25'

124702107 repluser 192.168.8.215:34580 NULL Query 0 Opening tables /replication-manager/ SHOW CREATE TABLE MIPLPMSDB25FEB25.CAMBIUM_DEVICESTATUS_24_20250120

124702115 repluser 192.168.8.215:34612 NULL Sleep 2 NULL

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 292 NULL

124700898 repluser 192.168.8.215:42538 NULL Query 152 Writing to net /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='MIPLPMSDB25FEB25'

124702107 repluser 192.168.8.215:34580 NULL Sleep 0 NULL

124702115 repluser 192.168.8.215:34612 NULL Sleep 2 NULL

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 293 NULL

124700898 repluser 192.168.8.215:42538 NULL Query 153 Writing to net /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='MIPLPMSDB25FEB25'

124702115 repluser 192.168.8.215:34612 NULL Sleep 0 NULL

124702129 repluser 192.168.8.215:34012 NULL Sleep 0 NULL

[cygnet@mipl-p-m-oss-bss-db ~]$ mysql -ucygnet -p12qwaszx -h192.168.8.171 -P3306 -e "show full processlist" | grep -i "repluser"

mysql: [Warning] Using a password on the command line interface can be insecure.

124699493 repluser 192.168.8.215:37328 NULL Sleep 294 NULL

124700898 repluser 192.168.8.215:42538 NULL Query 154 Writing to net /replication-manager/ SELECT a.TABLE_SCHEMA as Table_schema , a.TABLE_NAME as Table_name, COALESCE(a.ENGINE,'') as Engine,COALESCE(a.TABLE_ROWS,0) as Table_rows ,COALESCE(a.DATA_LENGTH,0) as Data_length,COALESCE(a.INDEX_LENGTH,0) as Index_length , 0 as Table_crc FROM information_schema.TABLES a WHERE a.TABLE_TYPE='BASE TABLE' AND a.TABLE_SCHEMA='MIPLPMSDB25FEB25'

124702115 repluser 192.168.8.215:34612 NULL Sleep 0 NULL

124702129 repluser 192.168.8.215:34012 NULL Sleep 1 NULL