diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index af64b29d..6e9647f6 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -41,7 +41,43 @@ jobs: - name: Lint with flake8 run: flake8 . --count --max-complexity=10 --max-line-length=120 --statistics + - name: Create test environment file + run: | + mkdir -p /tmp + echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/test_db" > .env.test + echo "DATABASE_TEST_URL=postgresql://postgres:postgres@localhost:5432/test_db" >> .env.test + echo "SECRET_KEY=test-secret-key-for-ci" >> .env.test + echo "AUTH0_DOMAIN=example.auth0.com" >> .env.test + echo "AUTH0_CLIENT_ID=test-client-id" >> .env.test + echo "AUTH0_CLIENT_SECRET=test-client-secret" >> .env.test + echo "AUTH0_AUDIENCE=https://api.example.com" >> .env.test + echo "OPENAI_API_KEY=sk-test-key" >> .env.test + + - name: Check environment variables + env: + CI: "true" + TESTING: "True" + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/test_db" + DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/test_db" + SECRET_KEY: "test-secret-key-for-ci" + AUTH0_DOMAIN: "example.auth0.com" + AUTH0_CLIENT_ID: "test-client-id" + AUTH0_CLIENT_SECRET: "test-client-secret" + AUTH0_AUDIENCE: "https://api.example.com" + OPENAI_API_KEY: "sk-test-key" + run: python scripts/check_env.py --env-file .env.test --no-exit + - name: Run tests with pytest + env: + TESTING: "True" + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/test_db" + DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/test_db" + SECRET_KEY: "test-secret-key-for-ci-environment" + AUTH0_DOMAIN: "example.auth0.com" + AUTH0_CLIENT_ID: "test-client-id" + AUTH0_CLIENT_SECRET: "test-client-secret" + AUTH0_AUDIENCE: "https://api.example.com" + OPENAI_API_KEY: "sk-test-key" run: pytest --cov=app --cov-report=xml - name: Upload coverage to Codecov diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml index ed991eaa..d5d9d4fc 100644 --- a/.github/workflows/frontend-ci.yml +++ b/.github/workflows/frontend-ci.yml @@ -39,7 +39,28 @@ jobs: - name: Type check run: npm run typecheck || npm run check:types || npm run tsc + - name: Create test environment file + run: | + mkdir -p /tmp + echo "VITE_API_URL=http://localhost:8000/api/v1" > .env.test + echo "VITE_AUTH0_DOMAIN=example.auth0.com" >> .env.test + echo "VITE_AUTH0_CLIENT_ID=test-client-id" >> .env.test + echo "VITE_AUTH0_AUDIENCE=https://api.example.com" >> .env.test + + - name: Install dotenv for env checking + run: npm install dotenv --no-save + + - name: Check environment variables + run: node scripts/check-env.js .env.test + - name: Build + env: + CI: "true" + VITE_API_URL: "http://localhost:8000/api/v1" + VITE_AUTH0_DOMAIN: "example.auth0.com" + VITE_AUTH0_CLIENT_ID: "test-client-id" + VITE_AUTH0_AUDIENCE: "https://api.example.com" + VITE_DEV_MODE: "false" run: npm run build - name: Test diff --git a/README.md b/README.md index 6a28a93c..165e28fa 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# TeamInsight - Contribution Analytics Platform +# Toban Contribution Viewer TeamInsight is an AI-powered analytics platform designed to extract, analyze, and visualize team contributions across digital workspaces. The platform connects to Slack, GitHub, and Notion via their APIs to collect activity data, processes it using AI to identify meaningful contributions, and presents actionable insights through an intuitive dashboard. @@ -43,9 +43,9 @@ TeamInsight is an AI-powered analytics platform designed to extract, analyze, an ### Prerequisites -- Python 3.8+ -- Node.js 16+ -- PostgreSQL +- Python 3.12+ +- Node.js 18+ +- PostgreSQL 13+ - API keys for: - Slack - GitHub @@ -104,6 +104,38 @@ TeamInsight is an AI-powered analytics platform designed to extract, analyze, an npm run dev ``` +## Environment Variables Management + +The project uses a structured approach to environment variables management to ensure proper configuration across environments. + +### Backend Environment Variables + +Backend environment variables are managed through: + +1. **Configuration Definition**: All environment variables are defined in `app/config.py` using Pydantic for validation +2. **Environment Validation**: The application validates required variables at startup and logs warnings if any are missing +3. **Testing Utility**: A utility (`app/core/env_test.py`) is provided to check environment configurations +4. **Command-line Verification**: The `scripts/check_env.py` script can be used to verify environment variables before deployment + +Required backend environment variables: +- `DATABASE_URL`: PostgreSQL connection string +- `SECRET_KEY`: Application secret key for security +- `AUTH0_DOMAIN`, `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_AUDIENCE`: Auth0 authentication settings +- `OPENAI_API_KEY`: For AI-powered analysis + +### Frontend Environment Variables + +Frontend environment variables are managed through: + +1. **Centralized Configuration**: All environment variables are accessed through the `src/config/env.ts` module +2. **Validation at Runtime**: The application validates required variables during initialization +3. **Build-time Verification**: The `npm run check-env` script verifies environment variables during build +4. **Typed Access**: Strongly-typed access to environment variables with proper error handling + +Required frontend environment variables: +- `VITE_API_URL`: URL to the backend API +- `VITE_AUTH0_DOMAIN`, `VITE_AUTH0_CLIENT_ID`, `VITE_AUTH0_AUDIENCE`: Auth0 authentication settings + ## Contributing Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct and the process for submitting pull requests. diff --git a/backend/app/config.py b/backend/app/config.py index 30e87c0a..dc25cf48 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,19 +1,72 @@ +import os from functools import lru_cache +from typing import Any, List, Optional +from pydantic import PostgresDsn, SecretStr, validator from pydantic_settings import BaseSettings class Settings(BaseSettings): + # API Settings PROJECT_NAME: str = "Toban Contribution Viewer API" PROJECT_DESCRIPTION: str = "API for tracking and visualizing contributions across various platforms" PROJECT_VERSION: str = "0.1.0" - + API_PREFIX: str = "/api/v1" + DEBUG: bool = False + ALLOWED_HOSTS: List[str] = ["localhost", "127.0.0.1"] + + # Secret Keys + SECRET_KEY: str + + # Database Settings + DATABASE_URL: PostgresDsn + DATABASE_TEST_URL: Optional[PostgresDsn] = None + + # Authentication Settings + AUTH0_DOMAIN: str + AUTH0_CLIENT_ID: str + AUTH0_CLIENT_SECRET: SecretStr + AUTH0_AUDIENCE: str + + # Third-Party API Keys + OPENAI_API_KEY: SecretStr + SLACK_CLIENT_ID: Optional[str] = None + SLACK_CLIENT_SECRET: Optional[SecretStr] = None + SLACK_SIGNING_SECRET: Optional[SecretStr] = None + GITHUB_CLIENT_ID: Optional[str] = None + GITHUB_CLIENT_SECRET: Optional[SecretStr] = None + NOTION_API_KEY: Optional[SecretStr] = None + + # Feature Flags + ENABLE_SLACK_INTEGRATION: bool = True + ENABLE_GITHUB_INTEGRATION: bool = True + ENABLE_NOTION_INTEGRATION: bool = True + + # Logging + LOG_LEVEL: str = "INFO" + + # Validators + @validator("DATABASE_URL", pre=True) + def validate_database_url(cls, v: Optional[str]) -> Any: + if os.environ.get("TESTING") == "True": + # Use test database during testing + test_url = os.environ.get("DATABASE_TEST_URL") + return test_url if test_url else v + return v + class Config: env_file = ".env" + case_sensitive = True + env_file_encoding = "utf-8" @lru_cache() def get_settings() -> Settings: + """ + Get application settings from environment variables. + + Using lru_cache to avoid reloading settings for each request. + """ return Settings() diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 00000000..b9a7cf7f --- /dev/null +++ b/backend/app/core/__init__.py @@ -0,0 +1 @@ +# Core package initialization \ No newline at end of file diff --git a/backend/app/core/env_test.py b/backend/app/core/env_test.py new file mode 100644 index 00000000..e5b1b03f --- /dev/null +++ b/backend/app/core/env_test.py @@ -0,0 +1,90 @@ +""" +Utility for testing environment variable configurations. +This module helps verify that all required environment variables are present +and properly formatted before the application starts. +""" + +import os +import sys +from typing import Dict, List, Optional + +from pydantic import ValidationError + +from app.config import Settings + + +def test_env_vars(env_file: Optional[str] = None) -> Dict[str, List[str]]: + """ + Test environment variables configuration. + + Args: + env_file: Optional path to an environment file to test. + + Returns: + Dict with 'missing' and 'invalid' lists of environment variables. + """ + result = { + "missing": [], + "invalid": [], + } + + # If env_file is provided, read environment variables from it + if env_file and os.path.exists(env_file): + env_vars = {} + with open(env_file, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + key, value = line.split("=", 1) + env_vars[key] = value + else: + env_vars = os.environ.copy() + + # Try to create settings from environment variables + try: + Settings(**env_vars) + except ValidationError as e: + for error in e.errors(): + field = error["loc"][0] + if "missing" in error["msg"]: + result["missing"].append(field) + else: + result["invalid"].append(field) + + return result + + +def check_env(env_file: Optional[str] = None, exit_on_error: bool = True) -> bool: + """ + Check environment variables and optionally exit if any are missing or invalid. + + Args: + env_file: Optional path to an environment file to test. + exit_on_error: Whether to exit the application if any variables are missing/invalid. + + Returns: + True if all environment variables are valid, False otherwise. + """ + result = test_env_vars(env_file) + + if result["missing"] or result["invalid"]: + print("Environment variable configuration errors:") + + if result["missing"]: + print("Missing variables:") + for var in result["missing"]: + print(f" - {var}") + + if result["invalid"]: + print("Invalid variables:") + for var in result["invalid"]: + print(f" - {var}") + + if exit_on_error: + print("Exiting due to environment configuration errors.") + sys.exit(1) + + return False + + return True \ No newline at end of file diff --git a/backend/app/main.py b/backend/app/main.py index de66de88..46f83a46 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,14 +1,47 @@ +import logging + from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware from app.config import settings +from app.core.env_test import check_env + +# Configure logging +logging.basicConfig( + level=getattr(logging, settings.LOG_LEVEL), + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) +# Check environment variables on startup +if not check_env(exit_on_error=False): + logger.warning("Application started with environment configuration issues") + +# Create FastAPI application app = FastAPI( title=settings.PROJECT_NAME, description=settings.PROJECT_DESCRIPTION, version=settings.PROJECT_VERSION, + docs_url="/docs" if settings.DEBUG else None, + redoc_url="/redoc" if settings.DEBUG else None, + openapi_url="/openapi.json" if settings.DEBUG else None, ) +# Configure CORS +app.add_middleware( + CORSMiddleware, + allow_origins=[str(origin) for origin in settings.ALLOWED_HOSTS], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) +# Root endpoint @app.get("/") async def root(): - return {"message": "Welcome to Toban Contribution Viewer API"} \ No newline at end of file + return {"message": "Welcome to Toban Contribution Viewer API"} + +# Health check endpoint +@app.get("/health") +async def health_check(): + return {"status": "ok"} \ No newline at end of file diff --git a/backend/scripts/__init__.py b/backend/scripts/__init__.py new file mode 100644 index 00000000..fa07a308 --- /dev/null +++ b/backend/scripts/__init__.py @@ -0,0 +1 @@ +# Scripts package initialization \ No newline at end of file diff --git a/backend/scripts/check_env.py b/backend/scripts/check_env.py new file mode 100755 index 00000000..bafb79fe --- /dev/null +++ b/backend/scripts/check_env.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +Script to check environment variables before running the application. +This can be used in CI/CD pipelines to validate environment configuration. +""" + +import argparse +import os +import sys +from pathlib import Path + +# Add the parent directory to sys.path so we can import app modules +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# Set environment variables for testing if we're in CI mode +if os.environ.get('CI') == 'true': + os.environ['TESTING'] = 'True' + +from app.core.env_test import check_env # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description="Check environment variables configuration") + parser.add_argument( + "--env-file", "-e", + help="Path to .env file to check", + default=".env" + ) + parser.add_argument( + "--no-exit", "-n", + help="Don't exit with error code if validation fails", + action="store_true" + ) + + args = parser.parse_args() + env_file = args.env_file if os.path.exists(args.env_file) else None + + if env_file: + print(f"Checking environment variables from file: {env_file}") + else: + print("Checking environment variables from current environment") + + check_env(env_file=env_file, exit_on_error=not args.no_exit) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index b95ea991..5ae1cd96 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,14 +5,15 @@ "type": "module", "scripts": { "dev": "vite", - "build": "tsc -b && vite build", + "build": "npm run check-env && tsc -b && vite build", "lint": "eslint .", "preview": "vite preview", "format": "prettier --write \"src/**/*.{ts,tsx,css}\"", "format:check": "prettier --check \"src/**/*.{ts,tsx,css}\"", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "check-env": "node scripts/check-env.js" }, "dependencies": { "@chakra-ui/icons": "^2.1.1", @@ -31,6 +32,7 @@ "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", "@vitejs/plugin-react": "^4.3.4", + "dotenv": "^16.3.1", "eslint": "^9.21.0", "eslint-plugin-react-hooks": "^5.1.0", "eslint-plugin-react-refresh": "^0.4.19", diff --git a/frontend/scripts/check-env.js b/frontend/scripts/check-env.js new file mode 100755 index 00000000..1dc0b87d --- /dev/null +++ b/frontend/scripts/check-env.js @@ -0,0 +1,84 @@ +#!/usr/bin/env node + +/** + * Environment variables validation script + * Verifies that all required environment variables are present in the .env file + * Can be run as a pre-build check in CI/CD pipelines + */ + +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +import dotenv from 'dotenv'; + +// Convert ESM __dirname equivalent +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Required environment variables that must be defined +const REQUIRED_ENV_VARS = [ + 'VITE_API_URL', + 'VITE_AUTH0_DOMAIN', + 'VITE_AUTH0_CLIENT_ID', + 'VITE_AUTH0_AUDIENCE', +]; + +// Function to parse .env file +function parseEnvFile(filePath) { + try { + const envConfig = dotenv.parse(fs.readFileSync(filePath)); + return envConfig; + } catch (error) { + console.error(`Error reading .env file: ${filePath}`); + console.error(error); + return {}; + } +} + +// Function to check env vars from an object +function checkEnvVars(envVars) { + // Check for missing required variables + const missingVars = REQUIRED_ENV_VARS.filter(varName => !envVars[varName]); + + if (missingVars.length > 0) { + console.error('Missing required environment variables:'); + missingVars.forEach(varName => { + console.error(` - ${varName}`); + }); + process.exit(1); + } + + console.log('✅ All required environment variables are present.'); + return true; +} + +// Main function to check environment variables +function checkEnv() { + // Determine which environment file to check + const envFile = process.argv[2] || '.env'; + const envPath = path.resolve(process.cwd(), envFile); + + // Check if the environment file exists - if CI, create a temp file from env vars + if (!fs.existsSync(envPath)) { + if (process.env.CI === 'true') { + // In CI, use environment variables directly + console.log('Running in CI mode, using environment variables'); + return checkEnvVars(process.env); + } else { + console.error(`Environment file not found: ${envPath}`); + process.exit(1); + } + } + + console.log(`Checking environment variables in: ${envPath}`); + + // Parse the environment file + const envVars = parseEnvFile(envPath); + + // Check variables + return checkEnvVars(envVars); +} + +// Run the check +checkEnv(); diff --git a/frontend/src/config/env.ts b/frontend/src/config/env.ts new file mode 100644 index 00000000..7efa8b79 --- /dev/null +++ b/frontend/src/config/env.ts @@ -0,0 +1,107 @@ +/** + * Environment variables management and validation + * Utility for centralizing access to environment variables and validating their presence + */ + +// Required environment variables that must be defined +const REQUIRED_ENV_VARS = [ + 'VITE_API_URL', + 'VITE_AUTH0_DOMAIN', + 'VITE_AUTH0_CLIENT_ID', + 'VITE_AUTH0_AUDIENCE', +] as const; + +// Optional environment variables with default values +const OPTIONAL_ENV_VARS = { + VITE_DEV_MODE: 'false', + VITE_ENABLE_NOTION_INTEGRATION: 'true', + VITE_ENABLE_SLACK_INTEGRATION: 'true', + VITE_ENABLE_GITHUB_INTEGRATION: 'true', + VITE_AUTH0_REDIRECT_URI: window.location.origin + '/callback', +} as const; + +// Type definitions +type RequiredEnvVar = typeof REQUIRED_ENV_VARS[number]; +type OptionalEnvVar = keyof typeof OPTIONAL_ENV_VARS; +type EnvVar = RequiredEnvVar | OptionalEnvVar; + +// Validate required environment variables +const validateEnv = (): { valid: boolean; missing: string[] } => { + const missing: string[] = []; + + for (const envVar of REQUIRED_ENV_VARS) { + if (!import.meta.env[envVar]) { + missing.push(envVar); + console.error(`Missing required environment variable: ${envVar}`); + } + } + + return { + valid: missing.length === 0, + missing, + }; +}; + +// Get a specific environment variable with fallback to defaults for optional ones +export const getEnvVar = (name: T): string => { + const value = import.meta.env[name]; + + // If it's a required environment variable and it's missing, throw an error + if (!value && REQUIRED_ENV_VARS.includes(name as RequiredEnvVar)) { + throw new Error(`Missing required environment variable: ${name}`); + } + + // For optional environment variables, use the default value if not provided + if (!value && name in OPTIONAL_ENV_VARS) { + return OPTIONAL_ENV_VARS[name as OptionalEnvVar]; + } + + return value as string; +}; + +// Helper to get a boolean environment variable +export const getBooleanEnvVar = (name: EnvVar): boolean => { + const value = getEnvVar(name).toLowerCase(); + return value === 'true' || value === '1' || value === 'yes'; +}; + +// Validate environment on load +export const validateEnvironment = (): boolean => { + const { valid, missing } = validateEnv(); + + if (!valid) { + console.error( + 'The app is missing required environment variables:', + missing.join(', ') + ); + + // Only show alert in development mode to avoid exposing errors to users + if (getBooleanEnvVar('VITE_DEV_MODE')) { + alert( + `Missing required environment variables: ${missing.join(', ')}. ` + + 'Please check your .env file and restart the development server.' + ); + } + } + + return valid; +}; + +// Create a config object with all environment variables +export const env = { + apiUrl: getEnvVar('VITE_API_URL'), + auth: { + domain: getEnvVar('VITE_AUTH0_DOMAIN'), + clientId: getEnvVar('VITE_AUTH0_CLIENT_ID'), + audience: getEnvVar('VITE_AUTH0_AUDIENCE'), + redirectUri: getEnvVar('VITE_AUTH0_REDIRECT_URI'), + }, + features: { + enableNotion: getBooleanEnvVar('VITE_ENABLE_NOTION_INTEGRATION'), + enableSlack: getBooleanEnvVar('VITE_ENABLE_SLACK_INTEGRATION'), + enableGithub: getBooleanEnvVar('VITE_ENABLE_GITHUB_INTEGRATION'), + }, + isDev: getBooleanEnvVar('VITE_DEV_MODE'), +}; + +export default env; \ No newline at end of file diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index bef5202a..61f3dfc3 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -1,10 +1,41 @@ -import { StrictMode } from 'react' -import { createRoot } from 'react-dom/client' -import './index.css' -import App from './App.tsx' +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import './index.css'; +import App from './App.tsx'; +import { validateEnvironment } from './config/env'; -createRoot(document.getElementById('root')!).render( - - - , -) +// Validate environment variables before rendering the app +const envValid = validateEnvironment(); + +// Render the application +const rootElement = document.getElementById('root'); + +if (rootElement) { + const root = createRoot(rootElement); + + root.render( + + {envValid ? ( + + ) : ( +
+

Environment Configuration Error

+

+ The application is missing required environment variables. + Please check the console for more information. +

+ {import.meta.env.DEV && ( +
+

Development Instructions

+

+ Make sure you have a .env.local file in the project root with all required variables. + See .env.example for a list of required variables. +

+

After updating your environment variables, restart the development server.

+
+ )} +
+ )} + , + ); +}