DataHub: enable FE to DataHub, implement dataHubProvider

esbuild.package.js

@@ -16,6 +16,7 @@ esbuild
 			path.resolve(__dirname, 'forward_engineering', 'api.js'),
 			path.resolve(__dirname, 'forward_engineering', 'ddlProvider.js'),
 			path.resolve(__dirname, 'forward_engineering', 'dbtProvider.js'),
+			path.resolve(__dirname, 'forward_engineering', 'dataHubProvider.js'),
 			path.resolve(__dirname, 'reverse_engineering', 'api.js'),
 		],
 		bundle: true,

forward_engineering/config.json

@@ -444,5 +444,65 @@
 				}
 			]
 		}
-	]
+	],
+	"dataHub": {
+		"urn": "urn:li:dataPlatform:snowflake",
+		"containers": [
+			{
+				"type": "Database",
+				"source": "container",
+				"properties": {
+					"name": "database"
+				}
+			},
+			"Schema"
+		],
+		"datasets": ["Table", "View"],
+		"typeMapping": {
+			"DATE": "DateType",
+			"BIGINT": "NumberType",
+			"BINARY": "BytesType",
+			"BOOLEAN": "BooleanType",
+			"CHAR": "StringType",
+			"CHARACTER": "StringType",
+			"DATETIME": "TimeType",
+			"DEC": "NumberType",
+			"DECIMAL": "NumberType",
+			"NUMERIC": "NumberType",
+			"DOUBLE": "NumberType",
+			"DOUBLE PRECISION": "NumberType",
+			"FIXED": "NumberType",
+			"FLOAT": "NumberType",
+			"FLOAT4": "NumberType",
+			"FLOAT8": "NumberType",
+			"INT": "NumberType",
+			"INTEGER": "NumberType",
+			"NUMBER": "NumberType",
+			"OBJECT": "RecordType",
+			"REAL": "NumberType",
+			"BYTEINT": "NumberType",
+			"SMALLINT": "NumberType",
+			"STRING": "StringType",
+			"TEXT": "StringType",
+			"TIME": "TimeType",
+			"TIMESTAMP": "TimeType",
+			"TIMESTAMP_TZ": "TimeType",
+			"TIMESTAMP_LTZ": "TimeType",
+			"TIMESTAMP_NTZ": "TimeType",
+			"TINYINT": "NumberType",
+			"VARBINARY": "BytesType",
+			"VARCHAR": "StringType",
+			"CHARACTER VARYING": "StringType",
+			"VARIANT": "RecordType",
+			"ARRAY": "ArrayType",
+			"GEOGRAPHY": "NullType"
+		},
+		"options": {
+			"convertUrnsToLowerCase": {
+				"use": true,
+				"default": true
+			}
+		},
+		"viewLanguage": "SQL"
+	}
 }

forward_engineering/dataHubProvider.js

@@ -0,0 +1,111 @@
+/**
+ * @typedef {import('./types').AppInstance} AppInstance
+ * @typedef {import('./types').ColumnDefinition} ColumnDefinition
+ * @typedef {import('./types').ConstraintDto} ConstraintDto
+ * @typedef {import('./types').JsonSchema} JsonSchema
+ */
+const { toLower } = require('lodash');
+
+const types = require('./configs/types');
+const defaultTypes = require('./configs/defaultTypes');
+const getKeyHelper = require('./helpers/keyHelper');
+const getColumnDefinitionHelper = require('./helpers/columnDefinitionHelper');
+const { createView, hydrateView, hydrateViewColumn } = require('./helpers/viewHelper');
+const { FORMATS } = require('./helpers/constants');
+const { hydrateJsonSchemaColumn } = require('./helpers/hydrateJsonSchema');
+
+class DataHubProvider {
+	/**
+	 * @type {AppInstance}
+	 */
+	#appInstance;
+
+	/**
+	 * @param {{ appInstance: AppInstance }}
+	 */
+	constructor({ appInstance }) {
+		this.#appInstance = appInstance;
+	}
+
+	/**
+	 * @param {{ appInstance }}
+	 * @returns {DataHubProvider}
+	 */
+	static createDataHubProvider({ appInstance }) {
+		return new DataHubProvider({ appInstance });
+	}
+
+	/**
+	 * @param {string} type
+	 * @returns {string | undefined}
+	 */
+	getDefaultType(type) {
+		return defaultTypes[type];
+	}
+
+	/**
+	 * @returns {Record<string, object>}
+	 */
+	getTypesDescriptors() {
+		return types;
+	}
+
+	/**
+	 * @param {string} type
+	 * @returns {boolean}
+	 */
+	hasType(type) {
+		return Object.keys(types).map(toLower).includes(toLower(type));
+	}
+
+	/**
+	 * @param {{ columnDefinition: ColumnDefinition; }}
+	 * @returns {{ type: string; }}
+	 */
+	decorateType({ type, columnDefinition }) {
+		const columnDefinitionHelper = getColumnDefinitionHelper(this.#appInstance);
+
+		if (['VARCHAR', 'STRING', 'TEXT'].includes(type) && !columnDefinition.length) {
+			columnDefinition.length = 16777216;
+		} else if (['CHAR', 'CHARACTER'].includes(type) && !columnDefinition.length) {
+			columnDefinition.length = 1;
+		}
+
+		return columnDefinitionHelper.decorateType(type, columnDefinition);
+	}
+
+	createView(viewData, dbData, isActivated) {
+		const { getViewSelectStatement } = require('./helpers/tableHelper')(this.#appInstance);
+		const keyHelper = require('./helpers/keyHelper')(this.#appInstance);
+
+		return createView({
+			viewData,
+			isActivated,
+			scriptFormat: FORMATS.SNOWSIGHT,
+			getViewSelectStatement,
+			keyHelper,
+		});
+	}
+
+	hydrateView({ viewData, entityData }) {
+		return hydrateView({ viewData, entityData });
+	}
+
+	hydrateViewColumn(data) {
+		return hydrateViewColumn(data);
+	}
+
+	hydrateJsonSchemaColumn(jsonSchema, definitionJsonSchema) {
+		return hydrateJsonSchemaColumn(jsonSchema, definitionJsonSchema);
+	}
+
+	getPlatformSchema() {
+		return {
+			'com.linkedin.schema.MySqlDDL': {
+				tableSchema: '',
+			},
+		};
+	}
+}
+
+module.exports = DataHubProvider;

forward_engineering/ddlProvider.js

@@ -56,6 +56,8 @@ const {
 	prepareObjectTagsData,
 	isEmptyTags,
 } = require('./helpers/tagHelper');
+const { createView, hydrateView, hydrateViewColumn } = require('./helpers/viewHelper');
+const { hydrateJsonSchemaColumn } = require('./helpers/hydrateJsonSchema');
 
 const DEFAULT_SNOWFLAKE_SEQUENCE_START = 1;
 const DEFAULT_SNOWFLAKE_SEQUENCE_INCREMENT = 1;
@@ -307,11 +309,7 @@ module.exports = (baseProvider, options, app) => {
 		},
 
 		hydrateJsonSchemaColumn(jsonSchema, definitionJsonSchema) {
-			if (jsonSchema.type === 'variant') {
-				return omit(jsonSchema, ['subtype', 'mode']);
-			}
-
-			return jsonSchema;
+			return hydrateJsonSchemaColumn(jsonSchema, definitionJsonSchema);
 		},
 
 		createTable(tableData, isActivated) {
@@ -582,76 +580,12 @@ module.exports = (baseProvider, options, app) => {
 		},
 
 		createView(viewData, dbData, isActivated) {
-			const orReplace = preSpace(viewData.orReplace && 'OR REPLACE');
-			const ifNotExist = preSpace(viewData.ifNotExist && 'IF NOT EXISTS');
-			const { columnList, tableColumns, tables } = viewData.keys.reduce(
-				(result, key) => {
-					result.columnList.push({
-						name: `${getName(viewData.isCaseSensitive, key.alias || key.name)}`,
-						isActivated: key.isActivated,
-						comment: preSpace(
-							key.definition?.description &&
-								`COMMENT ${escapeString(scriptFormat, key.definition.description)}`,
-						),
-					});
-					result.tableColumns.push({
-						name: `${getName(viewData.isCaseSensitive, key.entityName)}.${getName(viewData.isCaseSensitive, key.name)}`,
-						isActivated: key.isActivated,
-					});
-
-					if (key.entityName) {
-						const tableName = getFullName(key.dbName, key.entityName);
-
-						if (!result.tables.includes(tableName)) {
-							result.tables.push(tableName);
-						}
-					}
-
-					return result;
-				},
-				{
-					columnList: [],
-					tableColumns: [],
-					tables: [],
-				},
-			);
-
-			if (isEmpty(tables) && !viewData.selectStatement) {
-				return '';
-			}
-
-			const viewColumns = viewColumnsToString(tableColumns, isActivated);
-			const selectStatement = getViewSelectStatement({
-				tables,
+			return createView({
 				viewData,
-				viewColumns,
-			});
-
-			const tagStatement = getTagStatement({
-				tags: viewData.viewTags,
-				isCaseSensitive: viewData.isCaseSensitive,
-				indent: '',
-			});
-
-			const clustering = viewData.materialized
-				? keyHelper.getClusteringKey({
-						clusteringKey: viewData.clusteringKey,
-						isParentActivated: isActivated,
-					})
-				: undefined;
-
-			return assignTemplates(templates.createView, {
-				orReplace,
-				ifNotExist,
-				secure: preSpace(viewData.secure && 'SECURE'),
-				materialized: preSpace(viewData.materialized && 'MATERIALIZED'),
-				name: viewData.fullName,
-				column_list: viewColumnsToString(columnList, isActivated),
-				copy_grants: viewData.copyGrants ? 'COPY GRANTS\n' : '',
-				comment: viewData.comment ? `COMMENT=${escapeString(scriptFormat, viewData.comment)}\n` : '',
-				select_statement: selectStatement,
-				tag: tagStatement ? tagStatement + '\n' : '',
-				clustering,
+				isActivated,
+				scriptFormat,
+				getViewSelectStatement,
+				keyHelper,
 			});
 		},
 
@@ -1006,39 +940,11 @@ module.exports = (baseProvider, options, app) => {
 		},
 
 		hydrateView({ viewData, entityData }) {
-			const firstTab = entityData[0];
-			const { databaseName, schemaName } = viewData.schemaData;
-			const viewName = getName(firstTab.isCaseSensitive, viewData.name);
-			const fullName = getFullName(getFullName(databaseName, schemaName), viewName);
-
-			return {
-				...viewData,
-				orReplace: firstTab.orReplace,
-				ifNotExist: firstTab.ifNotExist,
-				name: getName(firstTab.isCaseSensitive, viewData.name),
-				selectStatement: firstTab.selectStatement,
-				isCaseSensitive: firstTab.isCaseSensitive,
-				copyGrants: firstTab.copyGrants,
-				comment: firstTab.description,
-				secure: firstTab.secure,
-				materialized: firstTab.materialized,
-				fullName,
-				clusteringKey: firstTab.clusteringKey,
-				viewTags: firstTab.viewTags ?? [],
-			};
+			return hydrateView({ viewData, entityData });
 		},
 
 		hydrateViewColumn(data) {
-			if (!data.entityName) {
-				return data;
-			}
-
-			return {
-				...data,
-				name: getName(data.definition?.isCaseSensitive, data.name),
-				dbName: getName(head(data.containerData)?.isCaseSensitive, data.dbName),
-				entityName: getName(head(data.entityData)?.isCaseSensitive, data.entityName),
-			};
+			return hydrateViewColumn(data);
 		},
 
 		commentIfDeactivated(statement, data, isPartOfLine) {

forward_engineering/helpers/hydrateJsonSchema.js

@@ -0,0 +1,11 @@
+const { omit } = require('lodash');
+
+const hydrateJsonSchemaColumn = (jsonSchema, definitionJsonSchema) => {
+	if (jsonSchema.type === 'variant') {
+		return omit(jsonSchema, ['subtype', 'mode']);
+	}
+
+	return jsonSchema;
+};
+
+module.exports = { hydrateJsonSchemaColumn };