Automate SQL Server Replication Monitoring

Problem

In my environment we have a few servers where we have SQL Server replication setup and we need to make sure replication is always in sync, because business users connect to these servers for current up to date data. In this tip I will show a process we put in place to notify us of any issues with SQL Server replication.

Solution

We can automate SQL Server replication monitoring in this step by step process that will send email notifications. This will be done using a centralized server to collect the data and send out the notification email.

SQL Server Replication Notification via Email Step 1 - Create Linked Servers

Create alinked server to each distributor that is part of replication. These should be created on your centralized monitoring server. Also, test each linked server to make sure you can successfully connect.

Step 2 - Create Table to Store the List of Servers

Create a table to store details of each distributor server which is used for replication.

CREATE TABLE [dbo].[Distributor_Servers]([S[SN] [int] IDENTITY(1,1) NOT NULL,
[DISTRIBUTOR] [sysname] NULL) ON [PRIMARY]
GO Step 3 - Insert Records into Table

Insert records into this table of all distributors as shown below:

Step 4 - Create Stored Procedure on each Replication Distributor

Create the below stored procedure on each distributo r to pull the replication information. I recommend creating this in a database specifically for database administration tasks. I have a database called DBA on each of my instances where I put objects like this.

CREATE PROCEDURE [dbo].[usp_GetReplicationMonitorData]--@Mailtext VARCHAR(5000) OUTPUT
AS
BEGIN
SET NOCOUNT ON
DECLARE @Sqltext NVARCHAR(MAX)
DECLARE @Publisher SYSNAME, @PublisherDB SYSNAME, @Publication SYSNAME, @Distdb SYSNAME,
@Subscriber SYSNAME, @SubscriberDB SYSNAME
IF OBJECT_ID('tempdb..#REPLMONITORPUBLISHER') IS NOT NULL
DROP TABLE #REPLMONITORPUBLISHER
CREATE TABLE #REPLMONITORPUBLISHER
(
SN INT IDENTITY(1,1)
,PUBLISHER SYSNAME NULL
,DISTRIBUTION_DB SYSNAME NULL
,STATUS INT NULL
,WARNING INT NULL
,PUBLICATIONCOUNT INT NULL
,RETURNSTAMP VARCHAR(100) NULL
)
IF OBJECT_ID('tempdb..#REPLMONITORSUBSCRIPTION') IS NOT NULL
DROP TABLE #REPLMONITORSUBSCRIPTION
CREATE TABLE #REPLMONITORSUBSCRIPTION
(
SN INT IDENTITY(1,1)
,PUBLISHER SYSNAME NULL
,DISTRIBUTION_DB SYSNAME NULL
,STATUS INT NULL
,WARNING INT NULL
,SUBSCRIBER SYSNAME NULL
,SUBSCRIBER_DB SYSNAME NULL
,PUBLISHER_DB SYSNAME NULL
,PUBLICATION SYSNAME NULL
,PUBLICATION_TYPE INT NULL
,SUBTYPE INT NULL
,LATENCY INT NULL
,LATENCYTHRESHOLD FLOAT NULL
,AGENTNOTRUNNING INT NULL
,AGENTNOTRUNNINGTHRESHOLD INT NULL
,TIMETOEXPIRATION INT NULL
,EXPIRATIONTHRESHOLD INT NULL
,LAST_DISTSYNC DATETIME NULL
,DISTRIBUTION_AGENTNAME SYSNAME NULL
,MONITORRANKING INT NULL
,DISTRIBUTIONAGENTJOBID BINARY(16) NULL
,DISTRIBUTIONAGENTID INT NULL
,DISTRIBUTIONAGENTPROFILEID INT NULL
,LOGREADERAGENTNAME SYSNAME NULL
)
IF OBJECT_ID('tempdb..#REPLLOGREADERAGENT') IS NOT NULL
DROP TABLE #REPLLOGREADERAGENT
CREATE TABLE #REPLLOGREADERAGENT
(
SN INT IDENTITY(1,1)
,DBNAME SYSNAME NULL
,NAME SYSNAME NULL
,STATUS INT NULL
,PUBLISHER SYSNAME NULL
,PUBLISHER_DB SYSNAME NULL
,START_TIME DATETIME NULL
,TIME DATETIME NULL
,DURATION INT NULL
,COMMENTS NVARCHAR(1000)
,DELIVERY_TIME INT NULL
,DELIVERED_TRANSACTIONS INT NULL
,DELIVERED_COMMANDS INT NULL
,AVERAGE_COMMANDS INT NULL
,DELIVERY_RATE INT NULL
,DELIVERY_LATENCY INT NULL
,ERROR_ID INT NULL
,JOB_ID BINARY(16) NULL
,LOCAL_JOB INT NULL
,PROFILE_ID INT NULL
,AGENT_ID INT NULL
,LOCAL_TIMESTAMP BINARY(8) NULL
)
IF OBJECT_ID('tempdb..#REPLDISTRIBUTORAGENT') IS NOT NULL
DROP TABLE #REPLDISTRIBUTORAGENT
CREATE TABLE #REPLDISTRIBUTORAGENT
(
SN INT IDENTITY(1,1)
,DBNAME SYSNAME NULL
,NAME SYSNAME NULL
,STATUS INT NULL
,PUBLISHER SYSNAME NULL
,PUBLISHER_DB SYSNAME NULL
,PUBLICATION SYSNAME NULL
,SUBSCRIBER SYSNAME NULL
,SUBSCRIBER_DB SYSNAME NULL
,SUBSCRIPTION_TYPE INT NULL
,START_TIME DATETIME NULL
,TIME DATETIME NULL
,DURATION INT NULL
,COMMENTS NVARCHAR(1000)
,DELIVERY_TIME INT NULL
,DELIVERED_TRANSACTIONS INT NULL
,DELIVERED_COMMANDS INT NULL
,AVERAGE_COMMANDS INT NULL
,DELIVERY_RATE INT NULL
,DELIVERY_LATENCY INT NULL
,ERROR_ID INT NULL
,JOB_ID BINARY(16) NULL
,LOCAL_JOB INT NULL
,PROFILE_ID INT NULL
,AGENT_ID INT NULL
,LOCAL_TIMESTAMP BINARY(8) NULL
,OFFLOAD_ENABLED BIT NULL
,OFFLOAD_SERVER VARCHAR(50)
,SUBSCRIBER_TYPE INT NULL
)
INSERT INTO #REPLMONITORPUBLISHER
SELECT * FROM OPENROWSET ('SQLOLEDB','Server=(local);TRUSTED_CONNECTION=YES;',
'set fmtonly off exec sp_replmonitorhelppublisher')
DECLARE CURSORPUBLISHER CURSOR FOR
SELECT PUBLISHER, DISTRIBUTION_DB
FROM #REPLMONITORPUBLISHER;
OPEN CURSORPUBLISHER
FETCH NEXT FROM CURSORPUBLISHER
INTO @Publisher, @Distdb
WHILE @@FETCH_STATUS = 0
BEGIN
SET @Sqltext = 'SELECT ''' + @Publisher + ''', ''' + @Distdb + ''', STATUS, WARNING,
SUBSCRIBER, SUBSCRIBER_DB,PUBLISHER_DB, PUBLICATION , PUBLICATION_TYPE, SUBTYPE,
LATENCY, LATENCYTHRESHOLD, AGENTNOTRUNNING, AGENTNOTRUNNINGTHRESHOLD,
TIMETOEXPIRATION, EXPIRATIONTHRESHOLD, LAST_DISTSYNC, DISTRIBUTION_AGENTNAME,
MONITORRANKING, DISTRIBUTIONAGENTJOBID, DISTRIBUTIONAGENTID,
DISTRIBUTIONAGENTPROFILEID, LOGREADERAGENTNAME FROM
OPENROWSET (''SQLOLEDB'',''Server=(local);TRUSTED_CONNECTION=YES;'',
''set fmtonly off EXEC ' + @Distdb + '..sp_replmonitorhelpsubscription @publisher = '''''
+ @Publisher +''''', @publication_type = 0 '') '
INSERT INTO #REPLMONITORSUBSCRIPTION (PUBLISHER, DISTRIBUTION_DB, STATUS, WARNING,
SUBSCRIBER, SUBSCRIBER_DB, PUBLISHER_DB, PUBLICATION ,PUBLICATION_TYPE, SUBTYPE,
LATENCY, LATENCYTHRESHOLD, AGENTNOTRUNNING, AGENTNOTRUNNINGTHRESHOLD,
TIMETOEXPIRATION, EXPIRATIONTHRESHOLD, LAST_DISTSYNC, DISTRIBUTION_AGENTNAME,
MONITORRANKING, DISTRIBUTIONAGENTJOBID, DISTRIBUTIONAGENTID, DISTRIBUTIONAGENTPROFILEID,
LOGREADERAGENTNAME)
EXEC (@Sqltext)
FETCH NEXT FROM CURSORPUBLISHER
INTO @Publisher, @Distdb
END
CLOSE CURSORPUBLISHER;
DEALLOCATE CURSORPUBLISHER;
INSERT INTO #REPLLOGREADERAGENT
SELECT * FROM OPENROWSET ('SQLOLEDB','Server=(local);TRUSTED_CONNECTION=YES;',
'set fmtonly off EXEC sp_MSenum_replication_agents @type = 2')
INSERT INTO #REPLDISTRIBUTORAGENT
SELECT * FROM OPENROWSET ('SQLOLEDB','Server=(local);TRUSTED_CONNECTION=YES;',
'set fmtonly off EXEC sp_MSenum_replication_agents @type = 3')
-- Default latency threshold is 30 sec.
--It is common to have 1-2 minute Publisher to Subscriber latency in replication.
-- With default threshold, we are getting frequent latency alerts.
--Hence changing latency threshold to 120 sec for monitoring solution.
UPDATE #REPLMONITORSUBSCRIPTION SET LATENCYTHRESHOLD = 120
SELECT S.PUBLISHER, S.PUBLISHER_DB, S.PUBLICATION, S.DISTRIBUTION_DB, S.SUBSCRIBER,
S.SUBSCRIBER_DB, S.LATENCY,
S.LAST_DISTSYNC,
CASE WHEN (S.LATENCY/S.LATENCYTHRESHOLD)* 100 < 34 THEN 'Excellent'
WHEN (S.LATENCY/S.LATENCYTHRESHOLD)* 100 < 59 THEN 'Good'
WHEN (S.LATENCY/S.LATENCYTHRESHOLD)* 100 < 84
THEN 'Fair' WHEN (S.LATENCY/S.LATENCYTHRESHOLD)* 100 < 99
THEN 'Poor' ELSE 'Critical' END AS PERFORMANCE,
CASE S.WARNING WHEN 0 THEN 'NA' WHEN 1 THEN 'Expiration'
WHEN 2 THEN 'Latency' ELSE 'Unknown' END AS WARNING,
CASE L.STATUS WHEN 1 THEN 'Started' WHEN 2 THEN 'Stopped'
WHEN 3 THEN 'In Progress' WHEN 4 THEN 'Idle' WHEN 5
THEN 'Retrying' WHEN 6 THEN 'Failed' ELSE 'Unknown' END AS LOGREADERAGENTSTATUS,
CASE D.STATUS WHEN 1 THEN 'Started' WHEN 2 THEN 'Stopped' WHEN 3 THEN 'In Progress'
WHEN 4 THEN 'Idle' WHEN 5
THEN 'Retrying' WHEN 6 THEN 'Failed' ELSE 'Unknown' END AS DISTRIBUTIONAGENTSTATUS
FROM