Aggregate and append to large public data sources to Excel file

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - AGGREGATE AND APPEND TO LARGE PUBLIC DATA SOURCES TO EXCEL FILE

# Recipe ID - 
# Recipe URL - https://app.lingk.io/a/10932/tf/17957

# Description - # To use this recipe, upload the Chicago crimes file from data.gov into the Local File connector for your Environment
                # for more information on the calculations here, see: https://blog.godatadriven.com/impala-haversine.html

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - LocalFile
# Data Flows - Single Direction
# Connection Type - CSV to Excel

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure HTTP credentials in your Environment before running this recipe
  # HTTP Setup - https://help.lingk.io/en/articles/304-http-connector-setup-guide

###### Start - LocalFile connectors ######

-
    name: sampleFile
    type: localFileReader
    format: csvFormat
    properties:
      fileName: "Crimes_-_2001_to_present.csv"

-
    name: outputFile
    type: localFileWriter
    format: excelFormat
    properties:
      fileName: "crimes_retailtheft_farthest100.xlsx"

###### End - LocalFile connectors ######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

# FORMATS specify how files should be processed as they are being read or written to

readFormats:
-
    name: csvFormat
    type: delimited
    properties:
      quoteAllFields: true
      delimiter: ','
      header: true

writeFormats:      
-
    name: excelFormat
    type: excel
    properties:
      worksheets: data
      useHeader: true

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

#  - statement: (data) => select * from sampleFile
# NOTE: This print statement will increase the amount of time the recipe runs significantly. Use a WHERE statement to reduce time or comment once you understand the data.
#  - statement: print data

- statement: (dataCount) => select count(*) from sampleFile
  #- statement: print dataCount

# Downtown chicago Lat/Lon hard coded as Lat1/Lon1 values
# Get the distances from downtown of all retail thefts
  - statement: |
        (distance) => SELECT
          2 * asin(
              sqrt(
                cos(radians(41.881832)) *
                cos(radians(Latitude)) *
                pow(sin(radians((-87.623177 - Longitude)/2)), 2)
                    +
                pow(sin(radians((41.881832 - Latitude)/2)), 2)
              )
            ) * 6371 Distance_from_downtown_km,
          2 * asin(
              sqrt(
                cos(radians(41.881832)) *
                cos(radians(Latitude)) *
                pow(sin(radians((-87.623177 - Longitude)/2)), 2)
                    +
                pow(sin(radians((41.881832 - Latitude)/2)), 2)
              )
            ) * 3956 Distance_from_downtown_miles,
            *
            FROM sampleFile
            WHERE Description = "RETAIL THEFT"
            ORDER BY (2 * asin(
              sqrt(
                cos(radians(41.881832)) *
                cos(radians(Latitude)) *
                pow(sin(radians((-87.623177 - Longitude)/2)), 2)
                    + 
                pow(sin(radians((41.881832 - Latitude)/2)), 2)
              )
            ) * 3956) DESC
            LIMIT 100

#- statement: print distance

- statement: INSERT distance INTO outputFile

# For large datasets, consider building a recipe that loads the data into RedShift, BigQuery or Snowflake. Then use another recipe to query, transform, and generate feeds the data.

# Add more statements to convert, join, aggregrate, transform, and integrate your data