azure-pipelines.yml

# Build/CI checker for Microsoft Sentinel Notebooks

trigger:
- master
pr:
- master

schedules:
  - cron: 0 4 * * 1
    branches:
      include:
      - master
    always: true
jobs:
- job: PythonTests
  displayName: Python install and test Notebooks
  strategy:
    matrix:
      Python310:
        python.version: '3.10'
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
    displayName: 'Use Python $(python.version)'

  - task: Cache@2
    displayName: Cache pip packages
    continueOnError: True
    inputs:
      key: requirements.txt
      restoreKeys: python
      path: $(PIP_CACHE_DIR)
    
  - script: |
      python -m pip install --upgrade ipykernel
      python -m pip install --upgrade pip wheel
      python -m pip install --upgrade msticpy
      python -m pip install -r requirements.txt
    displayName: 'Install dependencies'

  - script: |
      python -m utils.check_nb_kernel check  -p "*.ipynb"
    displayName: 'Kernelspec check'
  - script: |
      python -m pip install nbconvert

      nbconver_opts="--ExecutePreprocessor.timeout=60 --ExecutePreprocessor.kernel_name=python3 --log-level 20 --allow-errors --to notebook"
      mkdir -p $(System.DefaultWorkingDirectory)/nb_output
      nbconver_opts="$nbconver_opts --output-dir=$(System.DefaultWorkingDirectory)/nb_output"
      jupyter nbconvert $nbconver_opts --execute "Machine Learning in Notebooks Examples.ipynb"
      tour_nb="A Tour of Cybersec notebook features.ipynb"
      if [[ -f "$tour_nb" ]]
      then
        jupyter nbconvert $nbconver_opts --execute "$tour_nb"
      fi
    displayName: AutorunNotebooks
  - script: |
      logfile=url_with_locales.log
      locale_match="(docs.microsoft.com|azure.microsoft.com)(\/[a-z]{2}-[a-z]{2})"
      grep --directories=recurse -E -n --exclude=*.pkl  --exclude=*.log $locale_match * > $logfile
      if [ -s $logfile ]
      then
        echo Locale-specific URLS found in one or more files.
        echo Please replace with locale-agnostic versions
        cat $logfile
        exit 1
      fi
    displayName: 'URL Locale check'

- job: SecurityChecks
  displayName: Security check
  pool:
    vmImage: 'windows-latest'
  steps:
  - task: AntiMalware@4
    inputs:
      InputType: 'Basic'
      ScanType: 'CustomScan'
      FileDirPath: '$(Build.StagingDirectory)'
      TreatSignatureUpdateFailureAs: 'Warning'
      SignatureFreshness: 'UpToDate'
      TreatStaleSignatureAs: 'Error'
  - task: CredScan@3
  - task: SdtReport@2
  - task: PostAnalysis@2
    inputs:
      GdnBreakAllTools: true