Read accumulated Logs by Organization

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - Read accumulated Logs by Organization

# Recipe URL - https://app.lingk.io/a/10932/tf/18502

# Description -  This recipe shows you how to read out the accumulated custom logs for your organization.

# Systems - 
# Connectors - logReader
# Data Flows - Single Direction
# Connection Type -

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

-
    name: logReader
    type: logReader

-
    name: logReader1
    type: logReader
    properties:
      tenantKey: {val}

-
    name: logReader2
    type: logReader
    properties:
      tenantKey: {val}
      environmentKey: {val}

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:
  - statement: print logReader 
  - statement: (logReaderCount) => select count(*) from logReader
  - statement: print logReaderCount

# This statement selects and returns the first value of expr for a group of rows.
  # The PARTITION BY is dividing the rows into small partitions by createdDate preference field
  - statement: |
        (data) => 
          SELECT first(date(createdDate)), first(recipeName), first(message), max(createdDate) OVER (
          PARTITION BY first(date(createdDate)) 
          ORDER BY CAST(first(date(createdDate)) AS timestamp) 
          RANGE BETWEEN INTERVAL 7 DAYS PRECEDING AND CURRENT ROW
            ) AS count FROM logReader
          WHERE message like "%Total number of enrollments to insert in canvas: 0%" and environmentKey = "production"
          GROUP BY date(createdDate), recipeName, message

- statement: print data

- statement: print logReader1 
  - statement: (logReaderCount) => select count(*) from logReader1
  - statement: print logReaderCount

- statement: print logReader2 
  - statement: (logReaderCount) => select count(*) from logReader2
  - statement: print logReaderCount

# Add more statements to convert, join, aggregrate, transform, and integrate your data