mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
347 lines
11 KiB
YAML
347 lines
11 KiB
YAML
name: Comprehensive CI
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
|
|
jobs:
|
|
testing:
|
|
name: Python Testing
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Cache pip dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-${{ hashFiles('output/ci_artifacts/requirements-dev.txt', 'pyproject.toml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-
|
|
|
|
- name: Cache pytest results
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: .pytest_cache
|
|
key: ${{ runner.os }}-pytest-${{ hashFiles('tests/', 'nodupe/') }}-${{ matrix.python-version }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pytest-${{ matrix.python-version }}-
|
|
${{ runner.os }}-pytest-
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r output/ci_artifacts/requirements.txt
|
|
pip install -r output/ci_artifacts/requirements-dev.txt
|
|
|
|
- name: Run pytest
|
|
run: |
|
|
pytest tests/ --cov=nodupe --cov-report=xml --cov-report=term
|
|
|
|
- name: Upload coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
TOKEN_REMOVED: ${{ SECRET_REMOVEDs.CODECOV_TOKEN }} # github-actions: allow-SECRET_REMOVED-access
|
|
file: ./coverage.xml
|
|
flags: unittests
|
|
name: codecov-umbrella
|
|
fail_ci_if_error: false
|
|
|
|
quality-checks:
|
|
name: Code Quality Checks
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- name: Cache pip dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-${{ hashFiles('output/ci_artifacts/requirements-dev.txt', 'pyproject.toml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-
|
|
|
|
- name: Cache npm dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.npm
|
|
key: ${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-npm-
|
|
|
|
- name: Cache linting results
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
.pylint.d
|
|
.mypy_cache
|
|
key: ${{ runner.os }}-lint-${{ hashFiles('nodupe/', 'tests/') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-lint-
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r output/ci_artifacts/requirements.txt
|
|
pip install -r output/ci_artifacts/requirements-dev.txt
|
|
|
|
- name: Run pylint (strict mode)
|
|
continue-on-error: true
|
|
run: |
|
|
pylint nodupe/ --disable=all --enable=missing-docstring,invalid-name,missing-module-docstring,missing-class-docstring,missing-function-docstring
|
|
|
|
- name: Run black (strict mode)
|
|
continue-on-error: true
|
|
run: |
|
|
black --check --diff nodupe/ tests/
|
|
|
|
- name: Run isort (strict mode)
|
|
continue-on-error: true
|
|
run: |
|
|
isort --check --diff nodupe/ tests/
|
|
|
|
- name: Run mypy (strict mode)
|
|
continue-on-error: true
|
|
run: |
|
|
mypy nodupe/ --ignore-missing-imports --no-strict-optional
|
|
|
|
- name: Run markdownlint
|
|
continue-on-error: true
|
|
run: |
|
|
npx markdownlint "**/*.md" --ignore node_modules || true
|
|
|
|
- name: Check docstring coverage (100% required)
|
|
continue-on-error: true
|
|
run: |
|
|
python -c "
|
|
import os
|
|
import ast
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
def check_docstrings():
|
|
missing_docstrings = []
|
|
total_classes = 0
|
|
total_functions = 0
|
|
missing_classes = 0
|
|
missing_functions = 0
|
|
|
|
for root, dirs, files in os.walk('nodupe'):
|
|
for file in files:
|
|
if file.endswith('.py') and not file.startswith('__'):
|
|
filepath = os.path.join(root, file)
|
|
with open(filepath, 'r') as f:
|
|
try:
|
|
tree = ast.parse(f.read())
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ClassDef):
|
|
total_classes += 1
|
|
if not ast.get_docstring(node):
|
|
missing_classes += 1
|
|
missing_docstrings.append(f'Class {node.name} in {filepath}')
|
|
elif isinstance(node, ast.FunctionDef) and not node.name.startswith('_'):
|
|
total_functions += 1
|
|
if not ast.get_docstring(node):
|
|
missing_functions += 1
|
|
missing_docstrings.append(f'Function {node.name} in {filepath}')
|
|
|
|
except Exception as e:
|
|
print(f'Error parsing {filepath}: {e}')
|
|
pass
|
|
|
|
return missing_docstrings, total_classes, total_functions, missing_classes, missing_functions
|
|
|
|
missing, total_classes, total_functions, missing_classes, missing_functions = check_docstrings()
|
|
|
|
print(f'Docstring Coverage Report:')
|
|
print(f' Total Classes: {total_classes}')
|
|
print(f' Total Functions: {total_functions}')
|
|
print(f' Classes with docstrings: {total_classes - missing_classes}/{total_classes}')
|
|
print(f' Functions with docstrings: {total_functions - missing_functions}/{total_functions}')
|
|
|
|
if total_classes > 0:
|
|
class_coverage = ((total_classes - missing_classes) / total_classes) * 100
|
|
print(f' Class docstring coverage: {class_coverage:.1f}%')
|
|
else:
|
|
class_coverage = 100.0
|
|
|
|
if total_functions > 0:
|
|
func_coverage = ((total_functions - missing_functions) / total_functions) * 100
|
|
print(f' Function docstring coverage: {func_coverage:.1f}%')
|
|
else:
|
|
func_coverage = 100.0
|
|
|
|
if missing:
|
|
print(f'\\n❌ DOCSTRING REQUIREMENT FAILURE: {len(missing)} items missing docstrings')
|
|
print('Missing docstrings found:')
|
|
for item in missing:
|
|
print(f' - {item}')
|
|
|
|
# Calculate overall coverage
|
|
total_items = total_classes + total_functions
|
|
missing_items = missing_classes + missing_functions
|
|
overall_coverage = ((total_items - missing_items) / total_items) * 100 if total_items > 0 else 100.0
|
|
|
|
print(f'\\nOverall docstring coverage: {overall_coverage:.1f}%')
|
|
print('❌ 100% docstring coverage required. CI failed due to missing docstrings.')
|
|
sys.exit(1)
|
|
else:
|
|
print('✅ All public classes and functions have docstrings!')
|
|
print('✅ 100% docstring coverage achieved!')
|
|
"
|
|
|
|
security-checks:
|
|
name: Security Checks
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- name: Cache pip dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-security-${{ hashFiles('output/ci_artifacts/requirements-dev.txt') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-security-
|
|
|
|
- name: Install security tools
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install bandit safety
|
|
|
|
- name: Run bandit security scan
|
|
run: |
|
|
bandit -r nodupe/ -f json -o bandit-results.json || true
|
|
|
|
- name: Run safety dependency check
|
|
run: |
|
|
safety check --full-report || true
|
|
|
|
- name: Upload security reports
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: security-reports
|
|
path: |
|
|
bandit-results.json
|
|
safety-report.json
|
|
|
|
auto-fix:
|
|
name: Auto-fix Formatting Issues
|
|
runs-on: ubuntu-latest
|
|
needs: quality-checks
|
|
if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]'
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.head_ref }}
|
|
TOKEN_REMOVED: ${{ SECRET_REMOVEDs.GITHUB_TOKEN }}
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- name: Cache pip dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-${{ hashFiles('output/ci_artifacts/requirements-dev.txt') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install black>=22.0.0 isort>=5.10.0 pep8>=1.7.0
|
|
|
|
- name: Run black auto-formatting
|
|
run: |
|
|
black nodupe/ tests/
|
|
echo "BLACK_CHANGES=$(git diff --name-only)" >> $GITHUB_ENV
|
|
|
|
- name: Run isort auto-formatting
|
|
run: |
|
|
isort nodupe/ tests/
|
|
echo "ISORT_CHANGES=$(git diff --name-only)" >> $GITHUB_ENV
|
|
|
|
- name: Check for formatting changes
|
|
id: check_changes
|
|
run: |
|
|
if [ -n "$BLACK_CHANGES" ] || [ -n "$ISORT_CHANGES" ]; then
|
|
echo "changes_detected=true" >> $GITHUB_OUTPUT
|
|
echo "::notice::Auto-formatting applied changes to files"
|
|
else
|
|
echo "changes_detected=false" >> $GITHUB_OUTPUT
|
|
echo "::notice::No formatting changes needed"
|
|
fi
|
|
|
|
- name: Commit and push auto-formatting changes
|
|
if: steps.check_changes.outputs.changes_detected == 'true'
|
|
run: |
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git add .
|
|
git commit -m "chore: auto-fix formatting issues [skip ci]"
|
|
git push origin ${{ github.head_ref }}
|
|
echo "::notice::Auto-formatting changes committed and pushed"
|
|
|
|
integration-tests:
|
|
name: Integration Tests
|
|
runs-on: ubuntu-latest
|
|
needs: [testing, quality-checks, auto-fix]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- name: Cache pip dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-${{ hashFiles('output/ci_artifacts/requirements-dev.txt', 'pyproject.toml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r output/ci_artifacts/requirements.txt
|
|
pip install -r output/ci_artifacts/requirements-dev.txt
|
|
|
|
- name: Run integration tests
|
|
continue-on-error: true
|
|
run: |
|
|
python -m pytest tests/ -v --ignore=tests/integration/ || echo "Tests completed with some failures"
|