Skip to content

Update scrap.yml

Update scrap.yml #117

Workflow file for this run

name: Scrape YouTube, Twitch Channels, and GitHub Contributors
on:
push:
branches:
- main
schedule:
- cron: '0 0 * * 1' # Exécution quotidienne à minuit (UTC) pour toutes les plateformes
jobs:
scrape:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Install dependencies
run: |
pip install requests beautifulsoup4
# Cache YouTube data to avoid unnecessary requests
- name: Cache YouTube data
uses: actions/cache@v3
with:
path: src/assets/misc/channels_info.json
key: youtube-cache-${{ github.ref }}-${{ github.run_number }}-${{ github.event.schedule }} # Stable cache key with branch reference and weekly renewal
restore-keys: |
youtube-cache-${{ github.ref }}-
- name: Run YouTube scraping script
run: |
python src/assets/misc/scrapyb.py
- name: Display YouTube JSON content
run: |
cat src/assets/misc/channels_info.json
# Cache Twitch data to avoid unnecessary requests
- name: Cache Twitch data
uses: actions/cache@v3
with:
path: src/assets/misc/twitch_channels_info.json
key: twitch-cache-${{ github.ref }}-${{ github.run_number }}-${{ github.event.schedule }} # Stable cache key with branch reference and weekly renewal
restore-keys: |
twitch-cache-${{ github.ref }}-
- name: Run Twitch scraping script
run: |
python src/assets/misc/scraptwitch.py
- name: Display Twitch JSON content
run: |
cat src/assets/misc/twitch_channels_info.json
# Cache Discord data to avoid unnecessary requests
- name: Cache Discord data
uses: actions/cache@v3
with:
path: src/assets/misc/discord_servers_info.json
key: discord-cache-${{ github.ref }}-${{ github.run_number }}-${{ github.event.schedule }} # Stable cache key with branch reference and weekly renewal
restore-keys: |
discord-cache-${{ github.ref }}-
- name: Run Discord scraping script
run: |
python src/assets/misc/scrapdiscord.py
- name: Display Discord JSON content
run: |
cat src/assets/misc/discord_servers_info.json
# Run GitHub contributors scraping (no cache needed)
- name: Run GitHub contributors scraping script
run: |
python src/assets/misc/scrapcontributors.py
- name: Display GitHub contributors JSON content
run: |
cat src/assets/cards.json
# Check if any JSON files have changed before committing
- name: Check if JSON files have changed
run: |
git diff --exit-code src/assets/misc/channels_info.json src/assets/misc/twitch_channels_info.json src/assets/cards.json src/assets/misc/discord_servers_info.json
id: changes_check
continue-on-error: true
# Commit and push changes only if the JSON files have been updated
- name: Commit and push changes if files changed
if: steps.changes_check.outcome == 'failure'
run: |
git config --local user.name "github-actions"
git config --local user.email "[email protected]"
git add src/assets/misc/channels_info.json src/assets/misc/twitch_channels_info.json src/assets/cards.json src/assets/misc/discord_servers_info.json
git commit -m "Update JSON files from GitHub Actions"
git push origin main