Metadata Driven SQL Server Integration Services ETL Development Using BIML

By:Nat Sundar || Related Tips:More > Integration Services Development

Problem

BIML is a great way to generate SQL Server Integration Services (SSIS) packages from code. How can I use my technical metadata repository to generate SSIS packages using BIML?

Solution

Metadata means "data about data" and describes the relevant aspect of the data. Metadata can be classified in three ways:

Business Metadata Operational Metadata Technical Metadata

These are high level benefits for using metadata for SSIS package creation for an enterprise:

No need to develop a SSIS package. The SSIS package can be generated based on the configuration. Highly transparent business rules. Rules can be modified easily by changing the values in the metadata tables. Less dependency on the development team. Technical Metadata

Technical metadata stores information about ETL processes. It contains details about data mapping and transformations from source to target in a data system.

This metadata is used by applications to generate ETL solutions dynamically. In addition, it helps the ETL Modelers, Developers and Analysts understand the control flow and data flow for the data load.

Most commercial ETL applications provide a metadata repository with an integrated metadata management system to manage the ETL process definition. The definition of technical metadata is usually more complex than the business metadata and it sometimes involves multiple dependencies.

Technical metadata contains the definition of the following:

Source Database / Source System Definition - Can be a relational database, third party system, file system, etc. Target Database - Usually a Data Warehouse Database Source Tables Source Columns Target Tables Target Columns Transformation

In this tip, we will learn and practice using technical metadata for ETL development.

Setup Metadata for Examples

The below script will help us setup a simple metadata model for ETL development.

USE [master];
GO
IF EXISTS (SELECT name FROM sys.databases WHERE name = N'Staging')
DROP DATABASE Staging;
GO
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'Staging')
CREATE DATABASE Staging;
GO
USE Staging;
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[Customer]')
AND type IN (N'U'))
Drop table [dbo].[Customer]
CREATE TABLE [dbo].[Customer]( [Customer_ID] [int] NOT NULL, [Name] [nvarchar](250) NOT NULL,
[Address] [nvarchar](250) NOT NULL,
CONSTRAINT [PK_Customer] PRIMARY KEY CLUSTERED ( [Customer_ID] ASC ))
Go
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[Customer_Copy]')
AND type IN (N'U'))
Drop table [dbo].[Customer_Copy]
CREATE TABLE [dbo].[Customer_Copy]( [Customer_ID] [int] NOT NULL, [Name] [nvarchar](250) NOT NULL,
[Address] [nvarchar](250) NOT NULL, CONSTRAINT [PK_Customer_Copy]
PRIMARY KEY CLUSTERED ( [Customer_ID] ASC ))
GO
IF EXISTS (SELECT name FROM sys.databases WHERE name = N'MetadataModel')
DROP DATABASE MetadataModel;
GO
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'MetadataModel')
CREATE DATABASE MetadataModel;
GO
USE MetadataModel;
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ListConnectionDetails]')
AND type IN (N'P', N'PC'))
Drop Procedure dbo.[ListConnectionDetails]
go
CREATE procedure [dbo].[ListConnectionDetails] as
Select RDP.ServerName,RDP.DatabaseName,
LTrim(RTrim(DP.Name)) as ConnectionName,
'Data Source=' + RDP.ServerName +
';Persist Security Info=true;Integrated Security=SSPI;Provider=SQLNCLI11.1;Initial Catalog='
+RDP.DatabaseName as ConnectionString
from DataPackage DP
INNER JOIN RelationalDataPackage RDP
on DP.DataPackageID = RDP.DataPackageID
Where [Type]='REL'
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ListCustomerCopyMapping]')
AND type IN (N'P', N'PC'))
Drop Procedure dbo.[ListCustomerCopyMapping]
go
CREATE procedure [dbo].[ListCustomerCopyMapping] as
Select
MAP.MappingComments as DataFlowName,
RDGS.TableName as SourceTableName,
RDGD. TableName as DestinationTableName,
RTrim(LTrim(DP.Name)) as ConnectionName
from [dbo].[SourceToTargetMapping] MAP
INNER JOIN dbo.DataGroup DGS
on MAP.SourceDataGroupID = DGS.DataGroupIDI and
Upper(DGS.[Type])='TABLE'
INNER JOIN dbo.RelationalDataGroup RDGS
on DGS.DataGroupID = RDGS.DataGroupID
INNER JOIN dbo.DataGroup DGD
on MAP.DestinationDataGrouplD = DGD.DataGroupID and
Upper(DGD.[Type])='TABLE'
INNER JOIN dbo.RelationalDataGroup RDGD
on DGD.DataGroupID = RDGD.DataGroupID
INNER JOIN dbo.DataPackage DP
on DGS.ContainedDataPackageID = DP.DataPackageID
Where MAP.[MappingName] ='CustomerDataCopy'
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataGroup]')
AND type IN (N'U'))
Drop table [dbo].[DataGroup]
CREATE TABLE [dbo].[DataGroup]( [DataGroupID] [int] IDENTITY(1,1) NOT NULL,
[Type] [nvarchar](50) NOT NULL, [Description] [nvarchar](50) NOT NULL,
[ContainedDataPackageID] [int] NOT NULL,
CONSTRAINT [PK_DataGroup] PRIMARY KEY CLUSTERED ( [DataGroupID] ASC )) ON [PRIMARY]
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataPackage]')
AND type IN (N'U'))
Drop table [dbo].[DataPackage]
CREATE TABLE DataPackage ([DataPackageID] [int] IDENTITY(1,1) NOT NULL,
[Type] [nvarchar](50) NOT NULL, [Description] [nvarchar](50) NOT NULL,[Name] [nvarchar](50) NULL,
CONSTRAINT [PK_DataPackage] PRIMARY KEY CLUSTERED ([DataPackageID] ASC )) ON [PRIMARY]
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[RelationalDataGroup]')
AND type IN (N'U'))
Drop table [dbo].[RelationalDataGroup]
CREATE TABLE [dbo].[RelationalDataGroup]( [TableName] [nvarchar](50) NOT NULL,
[DataGroupID] [int] NOT NULL, [Usage] [nvarchar](50) NULL ) ON [PRIMARY]
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[RelationalDataPackage]')
AND type IN (N'U'))
Drop table [dbo].[RelationalDataPackage]
CREATE TABLE [dbo].[RelationalDataPackage]( [DataPackageID] [int] NOT NULL,
[ServerName] [nvarchar](50) NOT NULL, [DatabaseName] [nvarchar](50) NOT NULL )
ON [PRIMARY]
GO
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[SourceToTargetMapping]')
AND type IN (N'U'))
Drop table [dbo].[SourceToTargetMapping]
CREATE TABLE [dbo].[SourceToTargetMapping]( [SourceDataGrouplD] [int] NOT NULL,
[DestinationDataGroupID] [int] NOT NULL, [MappingComments] [nvarchar](200) NOT NULL,
[MappingName] [nvarchar](50) NULL,
CONSTRAINT [PK_SourceToTargetMapping] PRIMARY KEY CLUSTERED
([SourceDataGrouplD] ASC, [DestinationDataGroupID] ASC )) ON [PRIMARY]
GO
ALTER TABLE [dbo].[DataGroup] WITH CHECK ADD CONSTRAINT [FK_DataGroup_DataPackage]
FOREIGN KEY([ContainedDataPackageID]) REFERENCES [dbo].[DataPackage] ([DataPackageID])
GO
ALTER TABLE [dbo].[DataGroup] CHECK CONSTRAINT [FK_DataGroup_DataPackage]
GO
ALTER TABLE [dbo].[RelationalDataGroup] WITH CHECK ADD CONSTRAINT [FK_RelationalDataGroup_DataGroup]
FOREIGN KEY([DataGroupID]) REFERENCES [dbo].[DataGroup] ([DataGroupID])
GO
ALTER TABLE [dbo].[RelationalDataGroup] CHECK CONSTRAINT [FK_RelationalDataGroup_DataGroup]
GO
ALTER TABLE [dbo].[RelationalDataPackage] WITH CHECK
ADD CONSTRAINT [FK_RelationalDataPackage_DataPackage] FOREIGN KEY(DataPackageID) REFERENCES
[dbo].[DataPackage] ([DataPackageID])
GO
ALTER TABLE [dbo].[RelationalDataPackage] CHECK CONSTRAINT [FK_RelationalDataPackage_DataPackage]
GO The relati