validmind · validbeck · Nov 26, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 15, 2024
diff --git a/.env.example b/.env.example
@@ -1 +1,5 @@
-OPENAI_API_KEY=ADD_YOUR_KEY_HERE
+OPENAI_API_KEY=ADD_YOUR_KEY_HERE
+
+# API INFO TO RUN /notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb
+PLATFORM_API_KEY=ADD_YOUR_API_KEY_HERE
+PLATFORM_API_SECRET=ADD_YOUR_API_SECRET_HERE
diff --git a/.github/actions/demo-notebook/action.yml b/.github/actions/demo-notebook/action.yml
@@ -0,0 +1,37 @@
+name: Execute demo notebook
+description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with development heap tracking
+
+inputs: {}
+
+runs: 
+  using: "composite"
+  steps: 
+    - name: Install python3 for Jupyter Notebooks
+      shell: bash
+      run: |
+        python3 -m pip install jupyter
+        python -m pip install --upgrade pip
+
+    - name: Install validmind for notebook execution
+      shell: bash
+      run: |
+        pip install validmind 
+        pip install validmind[llm] 
+        pip install fairlearn aequitas 
+        pip install shap==0.44.1
+
+    - name: Check dependencies
+      shell: bash
+      run: |
+        pip install pycairo
+        pip check
+
+    - name: Execute ONLY the Intro for Model Developers notebook with heap development
+      shell: bash
+      run: |
+          cd site 
+          quarto render --profile exe-demo notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb &> render_errors.log || { 
+          echo "Execute for intro_for_model_developers_EXECUTED.ipynb failed";
+          cat render_errors.log;
+          exit 1;
+          }
diff --git a/.github/actions/fetch-quarto/action.yml b/.github/actions/fetch-quarto/action.yml
@@ -0,0 +1,21 @@
+name: Fetch Quarto
+description: Checks out repo then downloads & installs the latest version of Quarto
+
+inputs: {}
+
+runs: 
+  using: "composite"
+  steps:
+    - name: Get latest Quarto release URL
+      id: get-quarto-url
+      shell: bash
+      run: |
+        API_URL="https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest"
+        DOWNLOAD_URL=$(curl -s $API_URL | jq -r '.assets[] | select(.name | endswith("linux-amd64.deb")).browser_download_url')
+        echo "DOWNLOAD_URL=$DOWNLOAD_URL" >> $GITHUB_ENV
+
+    - name: Download and install Quarto
+      shell: bash
+      run: |
+        wget ${{ env.DOWNLOAD_URL }} -O quarto-latest-linux-amd64.deb
+        sudo dpkg -i quarto-latest-linux-amd64.deb
diff --git a/.github/actions/prod-notebook/action.yml b/.github/actions/prod-notebook/action.yml
@@ -0,0 +1,37 @@
+name: Execute prod notebook
+description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with production heap tracking
+
+inputs: {}
+
+runs: 
+  using: "composite"
+  steps: 
+    - name: Install python3 for Jupyter Notebooks
+      shell: bash
+      run: |
+        python3 -m pip install jupyter
+        python -m pip install --upgrade pip
+
+    - name: Install validmind for notebook execution
+      shell: bash
+      run: |
+        pip install validmind 
+        pip install validmind[llm] 
+        pip install fairlearn aequitas 
+        pip install shap==0.44.1
+
+    - name: Check dependencies
+      shell: bash
+      run: |
+        pip install pycairo
+        pip check
+
+    - name: Execute ONLY the Intro for Model Developers notebook with heap production
+      shell: bash
+      run: |
+          cd site 
+          quarto render --profile exe-prod notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb &> render_errors.log || { 
+          echo "Execute for intro_for_model_developers_EXECUTED.ipynb failed";
+          cat render_errors.log;
+          exit 1;
+          }
diff --git a/.github/actions/staging-notebook/action.yml b/.github/actions/staging-notebook/action.yml
@@ -0,0 +1,37 @@
+name: Execute staging notebook
+description: Installs python3, validmind, checks dependencies then executes ONLY the Intro for Model Developers notebook with staging heap tracking
+
+inputs: {}
+
+runs: 
+  using: "composite"
+  steps: 
+    - name: Install python3 for Jupyter Notebooks
+      shell: bash
+      run: |
+        python3 -m pip install jupyter
+        python -m pip install --upgrade pip
+
+    - name: Install validmind for notebook execution
+      shell: bash
+      run: |
+        pip install validmind 
+        pip install validmind[llm] 
+        pip install fairlearn aequitas 
+        pip install shap==0.44.1
+
+    - name: Check dependencies
+      shell: bash
+      run: |
+        pip install pycairo
+        pip check
+
+    - name: Execute ONLY the Intro for Model Developers notebook with heap staging
+      shell: bash
+      run: |
+          cd site 
+          quarto render --profile exe-staging notebooks/tutorials/intro_for_model_developers_EXECUTED.ipynb &> render_errors.log || { 
+          echo "Execute for intro_for_model_developers_EXECUTED.ipynb failed";
+          cat render_errors.log;
+          exit 1;
+          }
diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml
@@ -18,19 +18,10 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v2
 
-      - name: Get latest Quarto release URL
-        id: get-quarto-url
-        run: |
-          API_URL="https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest"
-          DOWNLOAD_URL=$(curl -s $API_URL | jq -r '.assets[] | select(.name | endswith("linux-amd64.deb")).browser_download_url')
-          echo "DOWNLOAD_URL=$DOWNLOAD_URL" >> $GITHUB_ENV
-
-      - name: Download and install Quarto
-        run: |
-          wget ${{ env.DOWNLOAD_URL }} -O quarto-latest-linux-amd64.deb
-          sudo dpkg -i quarto-latest-linux-amd64.deb
+      - name: Fetch Quarto
+        uses: ./.github/actions/fetch-quarto
 
-      - name: Render docs site
+      - name: Render prod docs site
         run: |
             cd site 
             quarto render --profile production &> render_errors.log || { 
@@ -39,9 +30,27 @@ jobs:
             exit 1;
             }
 
+      # See if site/notebooks/ has updates
+      - name: Filter changed files
+        uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            notebooks:
+              - 'site/notebooks/**'
+
+      # If yes then execute the prod notebook
+      - name: Execute prod Intro for Model Developers notebook
+        if: steps.filter.outputs.notebooks == 'true'
+        uses: ./.github/actions/prod-notebook
+        id: execute-prod-notebook
+        env:
+          PLATFORM_API_KEY: ${{ secrets.PLATFORM_API_KEY }}
+          PLATFORM_API_SECRET: ${{ secrets.PLATFORM_API_SECRET }}
+
       # Prod bucket is in us-east-1
       - name: Configure AWS credentials
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} && aws configure set default.region us-east-1
 
-      - name: Deploy docs site
+      - name: Deploy docs prod site
         run: aws s3 sync site/_site s3://docs-ci-cd-prod/site --delete && aws cloudfront create-invalidation --distribution-id E2I9R40IH01NW3 --paths "/*" --no-cli-pager
diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml
@@ -18,19 +18,10 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v2
 
-      - name: Get latest Quarto release URL
-        id: get-quarto-url
-        run: |
-          API_URL="https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest"
-          DOWNLOAD_URL=$(curl -s $API_URL | jq -r '.assets[] | select(.name | endswith("linux-amd64.deb")).browser_download_url')
-          echo "DOWNLOAD_URL=$DOWNLOAD_URL" >> $GITHUB_ENV
-
-      - name: Download and install Quarto
-        run: |
-          wget ${{ env.DOWNLOAD_URL }} -O quarto-latest-linux-amd64.deb
-          sudo dpkg -i quarto-latest-linux-amd64.deb
+      - name: Fetch Quarto
+        uses: ./.github/actions/fetch-quarto
 
-      - name: Render docs site
+      - name: Render staging docs site
         run: |
             cd site 
             quarto render --profile staging &> render_errors.log || { 
@@ -39,9 +30,27 @@ jobs:
             exit 1;
             }
 
+      # See if site/notebooks/ has updates
+      - name: Filter changed files
+        uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            notebooks:
+              - 'site/notebooks/**'
+
+      # If yes then execute the staging notebook
+      - name: Execute staging Intro for Model Developers notebook
+        if: steps.filter.outputs.notebooks == 'true'
+        uses: ./.github/actions/staging-notebook
+        id: execute-staging-notebook
+        env:
+          PLATFORM_API_KEY: ${{ secrets.PLATFORM_API_KEY }}
+          PLATFORM_API_SECRET: ${{ secrets.PLATFORM_API_SECRET }}
+
       # Staging bucket is in us-west-2
       - name: Configure AWS credentials
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2
 
-      - name: Deploy docs site
+      - name: Deploy docs staging site
         run: aws s3 sync site/_site s3://docs-ci-cd-staging/site --delete && aws cloudfront create-invalidation --distribution-id E2FB73KGY63MV6 --paths "/*" --no-cli-pager
diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml
@@ -17,26 +17,35 @@ jobs:
     - name: Check out repository
       uses: actions/checkout@v2
 
-    - name: Get latest Quarto release URL
-      id: get-quarto-url
-      run: |
-        API_URL="https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest"
-        DOWNLOAD_URL=$(curl -s $API_URL | jq -r '.assets[] | select(.name | endswith("linux-amd64.deb")).browser_download_url')
-        echo "DOWNLOAD_URL=$DOWNLOAD_URL" >> $GITHUB_ENV
-
-    - name: Download and install Quarto
-      run: |
-        wget ${{ env.DOWNLOAD_URL }} -O quarto-latest-linux-amd64.deb
-        sudo dpkg -i quarto-latest-linux-amd64.deb
+    - name: Fetch Quarto
+      uses: ./.github/actions/fetch-quarto
 
-    - name: Render site
+    - name: Render demo docs site
       run: |
           cd site 
           quarto render --profile development &> render_errors.log || { 
           echo "Quarto render failed immediately";
           cat render_errors.log;
           exit 1;
           }
+
+    # See if site/notebooks/ has updates
+    - name: Filter changed files
+      uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          notebooks:
+            - 'site/notebooks/**'
+
+    # If yes then execute the demo notebook
+    - name: Execute demo Intro for Model Developers notebook
+      if: steps.filter.outputs.notebooks == 'true'
+      uses: ./.github/actions/demo-notebook
+      id: execute-demo-notebook
+      env:
+        PLATFORM_API_KEY: ${{ secrets.PLATFORM_API_KEY }}
+        PLATFORM_API_SECRET: ${{ secrets.PLATFORM_API_SECRET }}
 
     - name: Test for warnings or errors 
       run: |
@@ -48,7 +57,7 @@ jobs:
           echo "No warnings or errors detected during Quarto render"
         fi
 
-      # Demo bucket is in us-east-1
+      # Demo bucket is in us-east-1  
     - name: Configure AWS credentials
       run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} && aws configure set default.region us-east-1
 

diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,6 @@ site/.quarto/
 site/.gitignore
 
 # Rendered docs site
-site/_site/
+site/_site/
+site/_freeze/
+site/notebooks/tutorials/my_tests/
diff --git a/README.md b/README.md
@@ -135,4 +135,4 @@ After you pull in the changes, commit them to this repo as part of the release n
 
 <!-- September 16, 2024: Need to mention rendered Python `.html` docs and generated `.md` test descriptions -->
 
-<!-- Testing simplified merge strategy -->
+<!-- Testing conditional changes on site/notebooks/  -->
diff --git a/site/_quarto-exe-demo.yml b/site/_quarto-exe-demo.yml
@@ -0,0 +1,12 @@
+format:
+  html:
+    page-layout: full
+    navbar: false
+    code-tools: true
+    include-in-header: 
+      - environments/heap-development.html
+    include-after-body: 
+      - https://unpkg.com/@jupyter-widgets/html-manager@*/dist/embed-amd.js
+execute: 
+  enabled: true
+  freeze: auto
diff --git a/site/_quarto-exe-prod.yml b/site/_quarto-exe-prod.yml
@@ -0,0 +1,12 @@
+format:
+  html:
+    page-layout: full
+    navbar: false
+    code-tools: true
+    include-in-header: 
+      - environments/heap-production.html
+    include-after-body: 
+      - https://unpkg.com/@jupyter-widgets/html-manager@*/dist/embed-amd.js
+execute: 
+  enabled: true
+  freeze: auto
diff --git a/site/_quarto-exe-staging.yml b/site/_quarto-exe-staging.yml
@@ -0,0 +1,12 @@
+format:
+  html:
+    page-layout: full
+    navbar: false
+    code-tools: true
+    include-in-header: 
+      - environments/heap-staging.html
+    include-after-body: 
+      - https://unpkg.com/@jupyter-widgets/html-manager@*/dist/embed-amd.js
+execute: 
+  enabled: true
+  freeze: auto
Original file line number	Diff line number	Diff line change
Expand Up		@@ -135,4 +135,4 @@ After you pull in the changes, commit them to this repo as part of the release n

		<!-- September 16, 2024: Need to mention rendered Python `.html` docs and generated `.md` test descriptions -->

		<!-- Testing simplified merge strategy -->
		<!-- Testing conditional changes on site/notebooks/ -->