De-duplication Task - JSON Example

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - De-duplication Task - JSON Example

# Recipe URL - https://app.lingk.io/a/10932/tf/18408

# Description -  This recipe shows how our De-duplication Task can find differences between two data sets.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - JSON
# Data Flows - Single Direction
# Connection Type - Static Data

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure JSON connector in your Environment before running this recipe
  # JSON Setup - https://help.lingk.io/en/articles/74-json-connector-reference

-
    name: sourceLeadData
    type: json
    properties:
      jsonObject: >
       [
         { "Id": 1, "leadId": 1, "firstName": "Victoria", "lastName": "Murpy", "city": "London", "leadScore": 88 },
         { "Id": 2, "leadId": 2, "firstName": "Harry", "lastName": "Walsh", "city": "Berlin", "leadScore": 32 },
         { "Id": 3, "leadId": 3, "firstName": "William", "lastName": "Smih", "city": "Boston", "leadScore": 32 },
         { "Id": 4, "leadId": 4, "firstName": "Ethan", "lastName": "O'Brien", "city": "New York", "leadScore": 32 },
         { "Id": 5, "leadId": 5, "firstName": "Michael", "lastName": "Byne", "city": "Austin", "leadScore": 32 },
         { "Id": 6, "leadId": 6, "firstName": "Alexader", "lastName": "Jacbs", "city": "Las Vegas", "leadScore": 32 },
         { "Id": 7, "leadId": 7, "firstName": "Joseph", "lastName": "Wilson", "city": "Houston", "leadScore": 32 },
         { "Id": 8, "leadId": 8, "firstName": "Margaret", "lastName": "Jnes", "city": "Dallas", "leadScore": 32 },
         { "Id": 9, "leadId": 9, "firstName": "Bethay", "lastName": "Garcia", "city": "San Antonio", "leadScore": 32 },
         { "Id": 10, "leadId": 10, "firstName": "Victoria", "lastName": "Brown", "city": "El Paso", "leadScore": 32 }
       ]

-
    name: masterLeadData
    type: json
    properties:
      jsonObject: >
       [
         { "Id": 1, "leadId": 1, "firstName": "Victoria", "lastName": "Murphy", "city": "London", "leadScore": 88 },
         { "Id": 2, "leadId": 3, "firstName": "William", "lastName": "Smith", "city": "Paris", "leadScore": 32 },
         { "Id": 3, "leadId": 4, "firstName": "Ithan", "lastName": "O'Brien", "city": "El Paso", "leadScore": 32 },
         { "Id": 4, "leadId": 5, "firstName": "Michael", "lastName": "Byrne", "city": "Boston", "leadScore": 32 },
         { "Id": 5, "leadId": 6, "firstName": "Alexander", "lastName": "Jacobs", "city": "Salt Lake City", "leadScore": 32 },
         { "Id": 6, "leadId": 7, "firstName": "Joseph", "lastName": "Wilson", "city": "Miami", "leadScore": 32 },
         { "Id": 7, "leadId": 8, "firstName": "Margaret", "lastName": "Jones", "city": "Dallas", "leadScore": 32 },
         { "Id": 8, "leadId": 9, "firstName": "Bethany", "lastName": "Garcia", "city": "San Antonio", "leadScore": 32 },
         { "Id": 9, "leadId": 10, "firstName": "Victoria", "lastName": "Brown", "city": "El Paso", "leadScore": 32 }
       ]

#   _______        _        
#  |__   __|      | |       
#     | | __ _ ___| | _____ 
#     | |/ _` / __| |/ / __|
#     | | (_| \__ \   <\__ \
#     |_|\__,_|___/_|\_\___/

# Batch task used for parameterized by statements/connectors

tasks:
-
    name: deduplicationTask
    type: dataOperation 
    function: dedup   
    parameters:
       inputData: sourceLeadData
       targetData: masterLeadData
       recordType: Contact
       duplicateOutputName: duplicateRecords
       dedupResult: dedupResult

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:
  - statement: (sourceLeadDataRes) => select * from sourceLeadData

- statement: print sourceLeadDataRes

- statement: (masterLeadDataRes) => select * from masterLeadData

- statement: print masterLeadDataRes

- statement: |
        execute task 
            --name deduplicationTask
            --result dedupResult

- statement: (duplicateRecordsRes) => select * from duplicateRecords

- statement: print duplicateRecordsRes

- statement: (dedupResultRes) => select * from dedupResult

- statement: print dedupResultRes

# Add more statements to convert, join, aggregrate, transform, and integrate your data