diff --git a/.gitea/workflows/dev.yml b/.gitea/workflows/dev.yml new file mode 100644 index 0000000..fb7ac53 --- /dev/null +++ b/.gitea/workflows/dev.yml @@ -0,0 +1,83 @@ +name: Development CI +on: + push: + branches: [dev] +permissions: + contents: read +concurrency: + group: alphapulse-dev + cancel-in-progress: false +jobs: + check-and-build: + if: gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository + runs-on: docker + timeout-minutes: 55 + container: + image: docker:29-cli + volumes: + - /docker/gitea/mirrors/AlphaPulse:/docker/gitea/mirrors/AlphaPulse + outputs: + backend: ${{ steps.publish.outputs.backend }} + web: ${{ steps.publish.outputs.web }} + version: ${{ steps.publish.outputs.version }} + commit: ${{ steps.publish.outputs.commit }} + env: + GITEA_SHA: ${{ gitea.sha }} + GITEA_REF: ${{ gitea.ref }} + GITEA_EVENT_NAME: ${{ gitea.event_name }} + CI_RESOURCE_ID: dev-${{ gitea.run_id }}-${{ gitea.run_attempt }} + DOCKER_CONFIG: /tmp/alphapulse-docker-auth + steps: + - name: Install container tools + run: apk add --no-cache python3 nodejs git docker-cli-buildx docker-cli-compose + - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + persist-credentials: false + - name: Test CI control behavior + run: python3 -m unittest discover -s scripts/tests -v + - name: Check, build and verify isolated services + run: python3 scripts/ci.py check + - name: Publish verified development images + id: publish + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/dev' + env: + ALPHAPULSE_GITEA_TOKEN: ${{ secrets.ALPHAPULSE_GITEA_TOKEN }} + run: | + mkdir -p "$DOCKER_CONFIG" + chmod 700 "$DOCKER_CONFIG" + printf '%s' "$ALPHAPULSE_GITEA_TOKEN" | docker login git.134333.xyz --username eezd --password-stdin + python3 scripts/ci.py publish + - uses: https://github.com/actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: build-receipt + path: build.json + if-no-files-found: error + retention-days: 14 + - name: Clean this attempt only + if: always() + run: | + python3 scripts/ci.py cleanup + rm -rf "$DOCKER_CONFIG" + + deploy-development: + needs: check-and-build + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/dev' + runs-on: docker + timeout-minutes: 15 + container: + image: docker:29-cli + volumes: + - /docker/gitea/dev/AlphaPulse:/docker/gitea/dev/AlphaPulse + steps: + - name: Install container tools + run: apk add --no-cache python3 nodejs git docker-cli-compose + - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: https://github.com/actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: build-receipt + path: . + - name: Deploy the current accepted dev commit + run: python3 scripts/deploy.py update diff --git a/.gitea/workflows/main.yml b/.gitea/workflows/main.yml new file mode 100644 index 0000000..6cb4fa7 --- /dev/null +++ b/.gitea/workflows/main.yml @@ -0,0 +1,44 @@ +name: Main CI +on: + push: + branches: [main] + pull_request: + branches: [main] +permissions: + contents: read +concurrency: + group: alphapulse-main + cancel-in-progress: false +jobs: + check-and-build: + if: gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository + runs-on: docker + timeout-minutes: 55 + env: + GITEA_SHA: ${{ gitea.sha }} + GITEA_REF: ${{ gitea.ref }} + GITEA_EVENT_NAME: ${{ gitea.event_name }} + CI_RESOURCE_ID: main-${{ gitea.run_id }}-${{ gitea.run_attempt }} + DOCKER_CONFIG: /tmp/alphapulse-docker-auth + steps: + - name: Install container tools + run: apk add --no-cache python3 nodejs git docker-cli-buildx docker-cli-compose + - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + persist-credentials: false + - name: Test CI control behavior + run: python3 -m unittest discover -s scripts/tests -v + - name: Check, build and verify isolated services + run: python3 scripts/ci.py check + - uses: https://github.com/actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: build-receipt + path: build.json + if-no-files-found: error + retention-days: 14 + - name: Clean this attempt only + if: always() + run: | + python3 scripts/ci.py cleanup + rm -rf "$DOCKER_CONFIG" diff --git a/.gitea/workflows/mirror.yml b/.gitea/workflows/mirror.yml new file mode 100644 index 0000000..f1b7cfc --- /dev/null +++ b/.gitea/workflows/mirror.yml @@ -0,0 +1,34 @@ +name: GitHub Git mirror +on: + push: + branches: ['**'] + tags: ['**'] + create: + delete: + schedule: + - cron: '29 * * * *' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: alphapulse-github-mirror + cancel-in-progress: false +jobs: + mirror: + runs-on: docker + timeout-minutes: 15 + container: + image: docker:29-cli + volumes: + - /docker/gitea/mirrors/AlphaPulse:/docker/gitea/mirrors/AlphaPulse + steps: + - name: Install mirror tools + run: apk add --no-cache python3 nodejs git git-lfs + - uses: https://github.com/actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: main + persist-credentials: false + - name: Reconcile the current source references + env: + ALPHAPULSE_GITHUB_SSH_KEY: ${{ secrets.ALPHAPULSE_GITHUB_SSH_KEY }} + run: python3 scripts/mirror.py diff --git a/.gitignore b/.gitignore index d1d9e06..4780bd6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ backups/ .venv/ __pycache__/ /docs/ +/build.json diff --git a/AGENTS.md b/AGENTS.md index dcc094c..27770df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,3 +87,5 @@ - 后端验证：`cd backend && uv run pytest`、`uvx ruff check --select F,E9,B src tests`。前端验证：`cd web && pnpm lint:check && pnpm typecheck && pnpm test && pnpm build`；测试使用现有 pytest、Node 内置测试及 TS/Vue 编译器，不另加测试框架。 - Redis DB 14 与其他应用共用，只操作 `ap:*`，禁止 FLUSHDB/FLUSHALL。测试隔离数据库、Redis、模型和交易 I/O，不读取生产凭据，不 Reset 用户 Chat 或改真实 Skills；不自动启动采集或交易。 - 生产部署、重启、数据迁移执行、提交和推送均需本任务单独授权；不新增项目定时备份。本文件是唯一规范来源，目前没有其他工具规则入口；以后新增入口应链接本文件。 + +- CI/CD：Gitea 为唯一源，GitHub 单向镜像全部业务分支/TAG/历史；dev push 通过隔离检查后自动部署唯一实例，main 仅独立检查，不部署或正式发版。DEV 按用户决定复用现有业务数据与配置，域名为 `alpha.134333.xyz`，实例目录 `/docker/gitea/dev/AlphaPulse`；测试仍必须隔离。部署、恢复和回退统一使用 `scripts/deploy.py` 与实例 `.deploy.lock`，不直接绕过锁更新容器；数据库迁移继续独立授权。操作说明见 `deploy/README.md`。 diff --git a/backend/.dockerignore b/backend/.dockerignore index 4fc95ea..ea61462 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,3 +1,7 @@ .venv __pycache__ *.pyc +.env* +.pytest_cache +.ruff_cache +dist diff --git a/backend/src/alphapulse/api/main.py b/backend/src/alphapulse/api/main.py index dd06477..3c73316 100644 --- a/backend/src/alphapulse/api/main.py +++ b/backend/src/alphapulse/api/main.py @@ -1,3 +1,5 @@ +import os + from fastapi import Depends, FastAPI from fastapi.exceptions import RequestValidationError from fastapi.responses import RedirectResponse @@ -31,7 +33,8 @@ app.include_router(calendar.router) @app.get("/api/health", include_in_schema=False) async def health() -> ApiResponse[dict[str, str]]: - return success({"status": "ok"}) + return success({"status": "ok", "version": os.environ.get("APP_VERSION", "source"), + "commit": os.environ.get("APP_COMMIT", "unknown")}) @app.get("/api/openapi.json", include_in_schema=False, dependencies=[Depends(require_admin)]) diff --git a/backend/tests/test_auth_protocol.py b/backend/tests/test_auth_protocol.py index 3dc710b..0ab0183 100644 --- a/backend/tests/test_auth_protocol.py +++ b/backend/tests/test_auth_protocol.py @@ -238,7 +238,14 @@ def test_protocol_status_validation_redaction_openapi(auth_app, monkeypatch): monkeypatch.setattr(market, "tradable_symbols", AsyncMock(return_value=[{"symbol": "BTCUSDT", "base": "BTC", "quote": "USDT", "price_precision": 2, "volume_precision": 5}])) async def check(): async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client: - assert (await client.get("/api/health")).json() == {"code": 0, "data": {"status": "ok"}, "message": ""} + assert (await client.get("/api/health")).json() == { + "code": 0, "data": {"status": "ok", "version": "source", "commit": "unknown"}, "message": "", + } + monkeypatch.setenv("APP_VERSION", "20261029-aaaaaaaa") + monkeypatch.setenv("APP_COMMIT", "a" * 40) + assert (await client.get("/api/health")).json()["data"] == { + "status": "ok", "version": "20261029-aaaaaaaa", "commit": "a" * 40, + } invalid = await client.post("/api/auth/login", json={"password": {"sensitive": "do-not-echo"}}) assert invalid.status_code == 422 and invalid.json()["code"] == 422 assert "do-not-echo" not in invalid.text and "input" not in invalid.text diff --git a/deploy/Dockerfile.backend b/deploy/Dockerfile.backend index 4fc6303..ec50330 100644 --- a/deploy/Dockerfile.backend +++ b/deploy/Dockerfile.backend @@ -1,8 +1,18 @@ -FROM python:3.13-slim -COPY --from=ghcr.io/astral-sh/uv:0.12 /uv /usr/local/bin/uv -ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PROJECT_ENVIRONMENT=/opt/venv PATH=/opt/venv/bin:$PATH +FROM python:3.13-slim AS base +COPY --from=ghcr.io/astral-sh/uv:0.12 /uv /uvx /usr/local/bin/ +ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_NO_CACHE=1 UV_PROJECT_ENVIRONMENT=/opt/venv PATH=/opt/venv/bin:$PATH WORKDIR /app COPY pyproject.toml uv.lock ./ RUN uv sync --locked --no-dev --no-install-project COPY . . RUN uv sync --locked --no-dev + +FROM base AS check +RUN uv sync --locked && uv run pytest && uvx ruff==0.14.14 check --select F,E9,B src tests + +FROM base AS runtime +ARG APP_VERSION=source +ARG APP_COMMIT=unknown +ENV APP_VERSION=$APP_VERSION APP_COMMIT=$APP_COMMIT +LABEL org.opencontainers.image.version=$APP_VERSION org.opencontainers.image.revision=$APP_COMMIT +LABEL org.opencontainers.image.source=https://git.134333.xyz/eezd/AlphaPulse diff --git a/deploy/Dockerfile.web b/deploy/Dockerfile.web index ad16afa..e44de88 100644 --- a/deploy/Dockerfile.web +++ b/deploy/Dockerfile.web @@ -1,11 +1,22 @@ -FROM node:24-alpine AS build -RUN corepack enable +FROM node:24-alpine AS check +RUN corepack enable && corepack prepare pnpm@12.11.2 --activate WORKDIR /app COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ RUN pnpm install --frozen-lockfile COPY . . -RUN pnpm build +RUN pnpm lint:check && pnpm typecheck && pnpm test -FROM nginx:alpine +FROM check AS build +RUN pnpm build +ARG APP_VERSION=source +ARG APP_COMMIT=unknown +ENV APP_VERSION=$APP_VERSION APP_COMMIT=$APP_COMMIT +RUN node -e 'require("fs").writeFileSync("dist/version.json", JSON.stringify({version:process.env.APP_VERSION,commit:process.env.APP_COMMIT}))' + +FROM nginx:alpine AS runtime +ARG APP_VERSION=source +ARG APP_COMMIT=unknown +LABEL org.opencontainers.image.version=$APP_VERSION org.opencontainers.image.revision=$APP_COMMIT +LABEL org.opencontainers.image.source=https://git.134333.xyz/eezd/AlphaPulse COPY --from=build /app/dist /usr/share/nginx/html COPY nginx.conf /etc/nginx/conf.d/default.conf diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..a682eb1 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,45 @@ +# AlphaPulse CI/CD + +Gitea (`https://git.134333.xyz/eezd/AlphaPulse`) is the source repository; GitHub is a public, one-way mirror of all branches, tags and history. Develop on `dev`. `main` has independent checks and no deployed instance or release workflow. + +## Checks and images + +Both branch workflows run Python 3.13/uv checks, Node 24/pnpm 12.11.2 checks, Linux amd64 image builds and acceptance against temporary PostgreSQL/Redis. Acceptance runs on an internal Docker network with synthetic data; no worker or external model/trading service starts there. Foreign PRs do not enter the shared Docker runner. + +Only a `dev` push publishes the accepted images: + +- `git.134333.xyz/eezd/dev/backend/alphapulse:YYYYMMDD-xxxxxxxx` +- `git.134333.xyz/eezd/dev/web/alphapulse:YYYYMMDD-xxxxxxxx` + +The date comes from the commit's committer timestamp in UTC, and the suffix is the first eight characters of its full SHA. Existing versions are immutable. Images are public on Gitea only; private configuration and data are excluded. `/api/health` and `/version.json` identify the deployed API and web revision. `build.json` is the CI receipt, not a tracked source file. + +## The single data-bearing instance + +`/docker/gitea/dev/AlphaPulse` contains the runtime Compose configuration, `.env`, digest selection, deployment receipts and `.deploy.lock`. The repository-root `.env` points to that sole private configuration. The Compose project is `alphapulse-dev`; `AlphaPulse-dev` is the web container, and the other five service names append `-api`, `-crawler`, `-market`, `-calendar` or `-trader`. + +This project's DEV intentionally uses the existing database, Redis DB 14 (`ap:*` only), model credentials and Demo trading configuration. The user authorized restoring all six services. It is the only instance, served at `https://alpha.134333.xyz` through `127.0.0.1:21005`. CI never uses these data for testing. Do not start a second Compose project against this configuration. + +Automatic and manual updates use the same command and persistent lock: + +```sh +python3 scripts/deploy.py update --build /path/to/accepted/build.json +``` + +The command checks image identities, current dev SHA and database migration compatibility before updating. It waits for active Chat/trading analysis, records pending state, then starts API/web before the workers. An outdated dev build is skipped. A pending deployment blocks updates. Do not invoke `docker compose up` directly to bypass the lock. + +A failed update with a previous deployment restores the prior images/configuration while retaining data. With no previous deployment, the pending marker remains for diagnosis. Recovery and rollback are explicit operator commands: + +```sh +python3 scripts/deploy.py recover +python3 scripts/deploy.py rollback +``` + +Neither command restores database contents or resets trading orders. Schema migrations require separate authorization and are never part of CI auto-deployment. No scheduled backup is installed. + +## Credentials and mirroring + +Gitea secrets are `ALPHAPULSE_GITEA_TOKEN` (image publication) and `ALPHAPULSE_GITHUB_SSH_KEY` (a write-enabled deploy key restricted to the GitHub mirror). Ordinary checks and the deployment job do not receive publication secrets. Credentials are removed from job-local files on exit. + +Git mirroring is independent of build results. Push/create/delete events, hourly reconciliation and manual dispatch use the same `.mirror.lock` in `/docker/gitea/mirrors/AlphaPulse`. Only business heads/tags are managed; internal PR references are excluded. Missing or incomplete source reads fail rather than imply deletion. GitHub is not an independent development source. GitHub Actions is disabled on the mirror. + +Only resources labeled for the current CI attempt are cleaned; there is no global Docker prune. The existing runner registration, capacity and other projects' mount allowances are preserved. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 147e150..781d457 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,43 +1,56 @@ -name: alphapulse +# Run through scripts/deploy.py so every deployment shares the instance lock. +name: alphapulse-dev x-backend: &backend - build: - context: ../backend - dockerfile: ../deploy/Dockerfile.backend - image: alphapulse-backend - env_file: ../.env + image: ${ALPHAPULSE_BACKEND_IMAGE:?Set a verified backend digest} + env_file: .env restart: unless-stopped + stop_grace_period: 60s networks: [dev_default] services: api: <<: *backend + container_name: AlphaPulse-dev-api command: [uvicorn, alphapulse.api.main:app, --host, 0.0.0.0, --port, "8000", --proxy-headers, --forwarded-allow-ips, "172.18.0.0/16"] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=3)"] + interval: 5s + timeout: 4s + retries: 12 crawler: <<: *backend + container_name: AlphaPulse-dev-crawler command: [python, -m, alphapulse.crawler.manager] - # 留时间给 scrapy 子进程优雅退出并刷完日志 - stop_grace_period: 30s market: <<: *backend + container_name: AlphaPulse-dev-market command: [python, -m, alphapulse.market.worker] trader: <<: *backend + container_name: AlphaPulse-dev-trader command: [python, -m, alphapulse.trading.engine] calendar: <<: *backend + container_name: AlphaPulse-dev-calendar command: [python, -m, alphapulse.calendar.worker] web: - build: - context: ../web - dockerfile: ../deploy/Dockerfile.web + image: ${ALPHAPULSE_WEB_IMAGE:?Set a verified web digest} + container_name: AlphaPulse-dev restart: unless-stopped - depends_on: [api] + depends_on: + api: + condition: service_healthy + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1/api/health"] + interval: 5s + timeout: 4s + retries: 12 ports: - 127.0.0.1:21005:80 networks: [dev_default] diff --git a/deploy/migrate.sh b/deploy/migrate.sh index ecbf1b4..49bf89f 100755 --- a/deploy/migrate.sh +++ b/deploy/migrate.sh @@ -1,10 +1,30 @@ #!/bin/sh -# 迁移前先备份：只有一套环境，迁移即生产操作 +# Migration remains an explicitly authorized operator action; never call this from CI. set -eu -cd "$(dirname "$0")" -mkdir -p ../backups -dump="../backups/alphapulse-premigrate-$(date +%Y%m%d-%H%M%S).dump" +umask 077 +instance=/docker/gitea/dev/AlphaPulse +exec 9>"$instance/.deploy.lock" +flock -x 9 +if [ -e "$instance/deployment.pending.json" ]; then + echo '存在未完成部署，请先处理部署状态' >&2 + exit 1 +fi +if [ -z "${ALPHAPULSE_BACKEND_IMAGE:-}" ]; then + echo '请通过 ALPHAPULSE_BACKEND_IMAGE 明确指定已验收的候选镜像摘要；迁移须独立授权' >&2 + exit 1 +fi +case "$ALPHAPULSE_BACKEND_IMAGE" in + git.134333.xyz/eezd/dev/backend/alphapulse@sha256:*) ;; + *) echo '迁移必须使用开发镜像摘要' >&2; exit 1 ;; +esac +if [ -n "$(docker ps -q --filter label=com.docker.compose.project=alphapulse-dev)" ]; then + echo '迁移前须在维护窗口内停止应用服务；不要并发运行采集或交易' >&2 + exit 1 +fi +# Keep migration backups explicit; no periodic backup is introduced. +backup_dir="$(dirname "$0")/../backups" +mkdir -p "$backup_dir" +dump="$backup_dir/alphapulse-premigrate-$(date +%Y%m%d-%H%M%S).dump" docker exec postgres pg_dump -U alphapulse -Fc alphapulse > "$dump" echo "已备份：$dump" -docker compose build api -docker compose run --rm -T api alembic upgrade head +docker run --rm --network dev_default --env-file "$instance/.env" "$ALPHAPULSE_BACKEND_IMAGE" alembic upgrade head diff --git a/scripts/ci.py b/scripts/ci.py new file mode 100644 index 0000000..33f0efa --- /dev/null +++ b/scripts/ci.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Build, verify and publish immutable AlphaPulse development images.""" + +import argparse +import base64 +import contextlib +from datetime import UTC, datetime +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile +import zlib +import time +import urllib.error +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +REGISTRY = 'git.134333.xyz' +REPOSITORY = 'eezd/AlphaPulse' +SOURCE = f'https://{REGISTRY}/{REPOSITORY}.git' +IMAGES = {part: f'{REGISTRY}/eezd/dev/{part}/alphapulse' for part in ('backend', 'web')} +LOCK_DIR = Path('/docker/gitea/mirrors/AlphaPulse') +LABEL = 'io.alphapulse.ci.run' + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def run(*args, input=None, env=None): + result = subprocess.run(list(map(str, args)), input=input, env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode: + # Callers never pass credentials as command arguments. + raise RuntimeError(f'{args[0]} failed ({result.returncode}): {result.stderr[-6000:]}{result.stdout[-6000:]}') + return result.stdout.strip() + + +def save(path, value): + path = Path(path) + temporary = path.with_name(path.name + '.tmp') + with temporary.open('w') as stream: + json.dump(value, stream, indent=2) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +@contextlib.contextmanager +def locked(path): + with Path(path).open('a') as stream: + fcntl.flock(stream, fcntl.LOCK_EX) + yield + + +def version(commit, timestamp): + require(re.fullmatch('[0-9a-f]{40}', commit), 'Expected a complete commit SHA') + return datetime.fromtimestamp(int(timestamp), UTC).strftime('%Y%m%d') + '-' + commit[:8] + + +def metadata(): + commit = run('git', 'rev-parse', 'HEAD') + require(os.environ.get('GITEA_SHA', commit) == commit, 'Checkout differs from the event commit') + return {'commit': commit, 'version': version(commit, run('git', 'show', '-s', '--format=%ct', commit)), + 'built_at': datetime.now(UTC).isoformat(), 'platform': 'linux/amd64', 'images': {}} + + +def request_json(url, token=None, data=None, method=None): + headers = {'User-Agent': 'AlphaPulse-CI', 'Accept': 'application/json'} + if token: + headers['Authorization'] = 'token ' + token + if data is not None: + headers['Content-Type'] = 'application/json' + data = json.dumps(data).encode() + with urllib.request.urlopen(urllib.request.Request(url, data=data, headers=headers, method=method), timeout=30) as response: + return json.load(response) + + +def branch_head(): + return request_json(f'https://{REGISTRY}/api/v1/repos/{REPOSITORY}/branches/dev')['commit']['id'] + + +def registry_json(path, token=None): + url = f'https://{REGISTRY}/v2/{path}' + headers = {'User-Agent': 'AlphaPulse-CI', + 'Accept': 'application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'} + if token: + headers['Authorization'] = 'Bearer ' + token + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as response: + raw = response.read() + return json.loads(raw), response.headers.get('Docker-Content-Digest') or 'sha256:' + hashlib.sha256(raw).hexdigest() + except urllib.error.HTTPError as error: + if error.code == 404: + return None + if error.code != 401 or token: + raise + challenge = error.headers.get('WWW-Authenticate', '') + fields = dict(re.findall(r'(\w+)="([^"]*)"', challenge)) + require(fields.get('realm', '').startswith(f'https://{REGISTRY}/'), 'Unexpected registry authentication endpoint') + auth_headers = {'User-Agent': 'AlphaPulse-CI'} + secret = os.environ.get('ALPHAPULSE_GITEA_TOKEN') + if secret: + auth_headers['Authorization'] = 'Basic ' + base64.b64encode(('eezd:' + secret).encode()).decode() + auth_url = fields['realm'] + '?' + urllib.parse.urlencode({k: fields[k] for k in ('service', 'scope') if k in fields}) + with urllib.request.urlopen(urllib.request.Request(auth_url, headers=auth_headers), timeout=30) as response: + bearer = json.load(response)['token'] + return registry_json(path, bearer) + + +def existing_image(part, build): + repository = IMAGES[part].split('/', 1)[1] + result = registry_json(f'{repository}/manifests/{build["version"]}') + if result is None: + return None + manifest, digest = result + require('config' in manifest, 'Expected a single Linux amd64 image manifest') + config = registry_json(f'{repository}/blobs/{manifest["config"]["digest"]}')[0] + labels = config['config'].get('Labels') or {} + require(labels.get('org.opencontainers.image.revision') == build['commit'], 'Existing version belongs to a different commit') + require(labels.get('org.opencontainers.image.version') == build['version'], 'Existing image version differs') + require(config.get('architecture') == 'amd64' and config.get('os') == 'linux', 'Existing image platform differs') + return IMAGES[part] + '@' + digest + + +def wait_command(*command, attempts=60): + for attempt in range(attempts): + try: + return run(*command) + except RuntimeError: + if attempt == attempts - 1: + raise + time.sleep(1) + + +def cleanup(label, images=()): + for identifier in run('docker', 'ps', '-aq', '--filter', f'label={LABEL}={label}').split(): + run('docker', 'rm', '-fv', identifier) + for identifier in run('docker', 'network', 'ls', '-q', '--filter', f'label={LABEL}={label}').split(): + run('docker', 'network', 'rm', identifier) + for image in images: + if image.startswith('alphapulse-ci:'): + run('docker', 'image', 'rm', image) + + +def smoke(build, label): + network = 'ap-ci-' + label + names = {part: network + '-' + part for part in ('db', 'redis', 'api', 'web')} + # Pull before entering the internal network; no runtime service can reach external I/O. + for image in ('postgres:18-alpine', 'redis:8-alpine'): + run('docker', 'pull', image) + run('docker', 'network', 'create', '--internal', '--label', f'{LABEL}={label}', network) + common = ['--network', network, '--label', f'{LABEL}={label}'] + try: + run('docker', 'run', '-d', '--name', names['db'], *common, '-e', 'POSTGRES_PASSWORD=ci-only', + '-e', 'POSTGRES_USER=ci', '-e', 'POSTGRES_DB=ci', 'postgres:18-alpine') + run('docker', 'run', '-d', '--name', names['redis'], *common, 'redis:8-alpine') + wait_command('docker', 'exec', names['db'], 'pg_isready', '-U', 'ci', '-d', 'ci') + run('docker', 'exec', names['db'], 'psql', '-U', 'ci', '-d', 'ci', '-v', 'ON_ERROR_STOP=1', + '-c', 'CREATE EXTENSION IF NOT EXISTS pg_trgm') + image = build['images']['backend']['local'] + password_hash = run('docker', 'run', '--rm', '--network', 'none', image, 'python', '-c', + 'from argon2 import PasswordHasher; print(PasswordHasher().hash("ci-only"))') + env = {'DATABASE_URL': f'postgresql+asyncpg://ci:ci-only@{names["db"]}:5432/ci', + 'REDIS_URL': f'redis://{names["redis"]}:6379/0', 'JWT_SECRET': 'isolated-ci-only-secret-32-characters', + 'SECRET_ENCRYPTION_KEY': 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=', + 'ADMIN_USERNAME': 'ci', 'ADMIN_PASSWORD_HASH': password_hash} + env_args = [entry for key, value in env.items() for entry in ('-e', key + '=' + value)] + run('docker', 'run', '-d', '--name', names['api'], '--network-alias', 'api', *common, *env_args, + image, 'sh', '-ec', 'alembic upgrade head && exec uvicorn alphapulse.api.main:app --host 0.0.0.0') + run('docker', 'run', '-d', '--name', names['web'], '--network-alias', 'web', *common, build['images']['web']['local']) + wait_command('docker', 'exec', names['api'], 'python', '-c', + 'import urllib.request; urllib.request.urlopen("http://localhost:8000/api/health", timeout=3)') + output = run('docker', 'exec', '-i', '-e', 'SMOKE_WEB=' + names['web'], names['api'], 'python', '-', + input=(ROOT / 'scripts/smoke_app.py').read_text()) + print(output, flush=True) + # Force a different API address while retaining the synthetic database and Redis. + old_address = json.loads(run('docker', 'inspect', names['api']))[0]['NetworkSettings']['Networks'][network]['IPAddress'] + run('docker', 'rm', '-f', names['api']) + run('docker', 'run', '-d', '--name', network + '-address-holder', *common, 'redis:8-alpine', 'sleep', '180') + run('docker', 'run', '-d', '--name', names['api'], '--network-alias', 'api', *common, *env_args, + image, 'sh', '-ec', 'alembic upgrade head && exec uvicorn alphapulse.api.main:app --host 0.0.0.0') + new_address = json.loads(run('docker', 'inspect', names['api']))[0]['NetworkSettings']['Networks'][network]['IPAddress'] + require(new_address != old_address, 'Acceptance did not exercise an API address change') + wait_command('docker', 'exec', names['api'], 'python', '-c', + 'import urllib.request; urllib.request.urlopen("http://web/api/health", timeout=3)') + run('docker', 'exec', '-i', '-e', 'SMOKE_WEB=' + names['web'], names['api'], 'python', '-', + input=(ROOT / 'scripts/smoke_app.py').read_text()) + except BaseException: + for part in ('api', 'web'): + result = subprocess.run(['docker', 'logs', '--tail', '30', names[part]], capture_output=True, text=True) + print(result.stdout + result.stderr, flush=True) + raise + finally: + cleanup(label) + + +def check(): + build = metadata() + label = os.environ.get('CI_RESOURCE_ID', 'local-' + str(os.getpid())) + require(re.fullmatch('[a-zA-Z0-9-]+', label), 'Invalid CI resource identifier') + build['resource_id'] = label + save(ROOT / 'build.json', build) + for part in IMAGES: + print('Checking ' + part, flush=True) + run('docker', 'build', '--provenance=false', '--platform', 'linux/amd64', '--target', 'check', + '-f', ROOT / f'deploy/Dockerfile.{part}', ROOT / part) + reused = existing_image(part, build) + local = reused or 'alphapulse-ci:' + label + '-' + part + build['images'][part] = {'local': local, 'reused': bool(reused)} + save(ROOT / 'build.json', build) + if reused: + run('docker', 'pull', reused) + else: + print('Building ' + part, flush=True) + run('docker', 'build', '--provenance=false', '--platform', 'linux/amd64', '--target', 'runtime', + '--build-arg', 'APP_VERSION=' + build['version'], '--build-arg', 'APP_COMMIT=' + build['commit'], + '--label', f'{LABEL}={label}', '--tag', local, '-f', ROOT / f'deploy/Dockerfile.{part}', ROOT / part) + info = json.loads(run('docker', 'image', 'inspect', local))[0] + require(info['Architecture'] == 'amd64' and info['Os'] == 'linux', 'Unexpected image platform') + require(info['Config']['Labels']['org.opencontainers.image.revision'] == build['commit'], 'Image commit mismatch') + smoke(build, label) + build['verified'] = True + save(ROOT / 'build.json', build) + print('All checks and isolated acceptance passed', flush=True) + + +def check_upload_layers(image): + # Docker may send an entire compressed layer in one registry request. + with tempfile.TemporaryDirectory(prefix='ap-layer-check-') as temporary: + archive_path = Path(temporary) / 'image.tar' + run('docker', 'save', '-o', archive_path, image) + with tarfile.open(archive_path) as archive: + manifest = json.load(archive.extractfile('manifest.json'))[0] + for name in manifest['Layers']: + with archive.extractfile(name) as stream: + first = stream.read(1024 * 1024) + compressor = None if first.startswith((b'\x1f\x8b', b'\x28\xb5\x2f\xfd')) else zlib.compressobj(wbits=31) + size = len(compressor.compress(first)) if compressor else len(first) + while chunk := stream.read(1024 * 1024): + size += len(compressor.compress(chunk)) if compressor else len(chunk) + if compressor: + size += len(compressor.flush()) + require(size < 90_000_000, f'A compressed image layer ({size} bytes) exceeds the CDN upload budget') + + +def publish(): + require(os.environ.get('GITEA_EVENT_NAME') == 'push' and os.environ.get('GITEA_REF') == 'refs/heads/dev', + 'Only a dev push may publish development images') + build = json.loads((ROOT / 'build.json').read_text()) + require(build.get('verified') and build['commit'] == os.environ['GITEA_SHA'], 'Missing acceptance for this commit') + with locked(LOCK_DIR / '.publish.lock'): + for part in IMAGES: + current = existing_image(part, build) + if not current: + tag = IMAGES[part] + ':' + build['version'] + check_upload_layers(build['images'][part]['local']) + run('docker', 'tag', build['images'][part]['local'], tag) + run('docker', 'push', tag) + current = existing_image(part, build) + require(current, 'Published manifest is missing') + build['images'][part]['digest'] = current + save(ROOT / 'build.json', build) + if output := os.environ.get('GITHUB_OUTPUT'): + with open(output, 'a') as stream: + stream.write(f'version={build["version"]}\ncommit={build["commit"]}\n') + for part in IMAGES: + stream.write(f'{part}={build["images"][part]["digest"]}\n') + print('Verified development digests published', flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('check', 'publish', 'cleanup')) + args = parser.parse_args() + if args.command == 'check': + check() + elif args.command == 'publish': + publish() + elif (ROOT / 'build.json').exists(): + build = json.loads((ROOT / 'build.json').read_text()) + cleanup(build['resource_id'], [v['local'] for v in build['images'].values()]) + + +if __name__ == '__main__': + main() diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100644 index 0000000..6b86333 --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Update the one data-bearing dev instance using verified image digests.""" + +import argparse +import json +from pathlib import Path +import re +import subprocess +import time + +from ci import IMAGES, ROOT, branch_head, locked, require, run, save + +INSTANCE = Path('/docker/gitea/dev/AlphaPulse') +SERVICES = ('api', 'web', 'market', 'crawler', 'calendar', 'trader') + +SCHEMA = ''' +import asyncio, json +from alembic.config import Config +from alembic.script import ScriptDirectory +from sqlalchemy import text +from alphapulse.core.db import engine +async def main(): + async with engine.connect() as connection: + async with connection.begin(): + await connection.execute(text("SET TRANSACTION READ ONLY")) + actual = list((await connection.execute(text("SELECT version_num FROM alembic_version"))).scalars()) + expected = ScriptDirectory.from_config(Config("alembic.ini")).get_heads() + assert sorted(actual) == sorted(expected), "Database migration requires separate authorization" + await engine.dispose() + print(json.dumps({"schema": actual})) +asyncio.run(main()) +''' + +IDLE = ''' +import asyncio, json +import httpx +from sqlalchemy import text +from alphapulse.core.auth import create_token, require_session, revoke_session +from fastapi.security import HTTPAuthorizationCredentials +from alphapulse.core.config import get_settings +from alphapulse.core.db import engine +async def main(): + async with engine.connect() as connection: + busy = await connection.scalar(text("SELECT count(*) FROM trading_analyses WHERE status='running'")) + token, _ = await create_token(get_settings().admin_username) + principal = await require_session(HTTPAuthorizationCredentials(scheme="Bearer", credentials=token)) + try: + async with httpx.AsyncClient(trust_env=False) as client: + response = await client.get("http://localhost:8000/api/chat/session", headers={"Authorization":"Bearer "+token}) + response.raise_for_status() + busy += int(response.json()["data"]["generating"]) + finally: + await revoke_session(principal) + await engine.dispose() + print(json.dumps({"busy": busy})) +asyncio.run(main()) +''' + + +def validate_images(build): + require(build.get('verified') is True, 'Build has not passed isolated acceptance') + require(re.fullmatch('[0-9a-f]{40}', build['commit']), 'Invalid commit SHA') + require(re.fullmatch(r'\d{8}-[0-9a-f]{8}', build['version']), 'Invalid development version') + require(build['version'].endswith(build['commit'][:8]), 'Version does not identify this commit') + for part, repository in IMAGES.items(): + image = build['images'][part]['digest'] + require(re.fullmatch(re.escape(repository) + r'@sha256:[0-9a-f]{64}', image), 'Expected a development image digest') + run('docker', 'pull', image) + info = json.loads(run('docker', 'image', 'inspect', image))[0] + labels = info['Config'].get('Labels') or {} + require(labels.get('org.opencontainers.image.revision') == build['commit'], 'Image revision mismatch') + require(labels.get('org.opencontainers.image.version') == build['version'], 'Image version mismatch') + require(info['Architecture'] == 'amd64' and info['Os'] == 'linux', 'Unexpected image architecture') + + +def compose(directory, release): + images = directory / 'images.env' + images.write_text(''.join(f'ALPHAPULSE_{part.upper()}_IMAGE={release["images"][part]["digest"]}\n' for part in IMAGES)) + return ['docker', 'compose', '--project-name', 'alphapulse-dev', '--env-file', str(images), + '-f', str(directory / 'compose.yml')] + + +def inspect_services(command): + identifiers = run(*command, 'ps', '-aq').split() + return json.loads(run('docker', 'inspect', *identifiers)) if identifiers else [] + + +def health(command, build): + containers = inspect_services(command) + require({c['Config']['Labels']['com.docker.compose.service'] for c in containers} == set(SERVICES), 'Missing services') + for container in containers: + require(container['State']['Running'] and container['RestartCount'] == 0, 'Service exited or restarted') + part = 'web' if container['Name'] == '/AlphaPulse-dev' else 'backend' + expected = json.loads(run('docker', 'image', 'inspect', build['images'][part]['digest']))[0]['Id'] + require(container['Image'] == expected, 'Running image differs from the accepted digest') + payload = run('docker', 'exec', 'AlphaPulse-dev-api', 'python', '-c', + 'import json,urllib.request; print(json.load(urllib.request.urlopen("http://web/api/health"))["data"]["commit"])') + require(payload == build['commit'], 'Served API revision mismatch') + payload = run('docker', 'exec', 'AlphaPulse-dev-api', 'python', '-c', + 'import json,urllib.request; print(json.load(urllib.request.urlopen("http://web/version.json"))["commit"])') + require(payload == build['commit'], 'Served frontend revision mismatch') + + +def deploy(build, directory=INSTANCE): + directory = Path(directory).resolve(strict=True) + require(directory == INSTANCE, 'Unexpected instance directory') + with locked(directory / '.deploy.lock'): + require(not (directory / 'deployment.pending.json').exists(), 'Resolve the previous pending deployment with recover first') + if branch_head() != build['commit']: + print('Skipped: dev has advanced') + return + validate_images(build) + # Only this read-only schema guard receives the existing application environment. + print(run('docker', 'run', '--rm', '-i', '--network', 'dev_default', '--env-file', directory / '.env', + build['images']['backend']['digest'], 'python', '-', input=SCHEMA), flush=True) + previous_file = directory / 'deployment.json' + previous = json.loads(previous_file.read_text()) if previous_file.exists() else None + old_compose = (directory / 'compose.yml').read_text() if previous else None + previous_running = [] + if previous: + old_command = compose(directory, previous) + previous_running = [c['Config']['Labels']['com.docker.compose.service'] for c in inspect_services(old_command) if c['State']['Running']] + for _ in range(60): + idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE)) + if not idle['busy']: + break + time.sleep(5) + else: + raise RuntimeError('Chat or trading analysis did not become idle within five minutes') + # Recheck after pulls and waiting; only a current dev commit can mutate the instance. + if branch_head() != build['commit']: + print('Skipped: dev advanced while preparing deployment') + return + if previous and 'trader' in previous_running: + run('docker', 'pause', 'AlphaPulse-dev-trader') + try: + idle = json.loads(run('docker', 'exec', '-i', 'AlphaPulse-dev-api', 'python', '-', input=IDLE)) + require(not idle['busy'], 'Analysis became active during preparation; retry after it completes') + except BaseException: + run('docker', 'unpause', 'AlphaPulse-dev-trader') + raise + pending = {'build': build, 'previous': previous, 'previous_compose': old_compose, + 'previous_running': previous_running, 'started_at': time.time()} + save(directory / 'deployment.pending.json', pending) + try: + if previous: + # Docker unpauses a paused container before delivering the stop signal. + run(*old_command, 'stop', 'trader') + run(*old_command, 'stop', 'crawler', 'calendar', 'market', 'web', 'api') + (directory / 'compose.yml').write_text((ROOT / 'deploy/docker-compose.yml').read_text()) + command = compose(directory, build) + run(*command, 'config', '--quiet') + run(*command, 'up', '-d', '--no-build', '--pull', 'never', '--wait', '--wait-timeout', '120', 'api', 'web') + run(*command, 'up', '-d', '--no-build', '--pull', 'never', 'market', 'crawler', 'calendar', 'trader') + time.sleep(5) + health(command, build) + if previous: + save(directory / 'deployment.previous.json', previous) + (directory / 'compose.previous.yml').write_text(old_compose) + save(previous_file, build) + (directory / 'deployment.pending.json').unlink() + except BaseException: + # Never roll back or reset business data. A pending marker blocks subsequent jobs. + subprocess.run(['docker', 'compose', '--project-name', 'alphapulse-dev', '--env-file', str(directory / 'images.env'), + '-f', str(directory / 'compose.yml'), 'stop'], capture_output=True) + if previous: + restore(directory, pending) + raise + print('All six services now run the accepted development digests', flush=True) + + +def restore(directory, pending): + previous = pending['previous'] + require(previous is not None, 'No previous deployment exists; inspect the pending first-start failure') + validate_images(previous) + (directory / 'compose.yml').write_text(pending['previous_compose']) + command = compose(directory, previous) + run(*command, 'up', '-d', '--no-build', '--pull', 'never', '--wait', '--wait-timeout', '120', *pending['previous_running']) + if set(pending['previous_running']) == set(SERVICES): + health(command, previous) + save(directory / 'deployment.json', previous) + (directory / 'deployment.pending.json').unlink() + print('Restored previous images and configuration; business data was retained', flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('update', 'recover', 'rollback')) + parser.add_argument('--build', type=Path, default=ROOT / 'build.json') + args = parser.parse_args() + if args.command == 'update': + deploy(json.loads(args.build.read_text())) + else: + with locked(INSTANCE / '.deploy.lock'): + if args.command == 'recover': + restore(INSTANCE, json.loads((INSTANCE / 'deployment.pending.json').read_text())) + else: + require(not (INSTANCE / 'deployment.pending.json').exists(), 'Recover the pending deployment first') + previous = json.loads((INSTANCE / 'deployment.previous.json').read_text()) + pending = {'previous': previous, 'previous_compose': (INSTANCE / 'compose.previous.yml').read_text(), + 'previous_running': list(SERVICES)} + save(INSTANCE / 'deployment.pending.json', pending) + run(*compose(INSTANCE, json.loads((INSTANCE / 'deployment.json').read_text())), 'stop') + restore(INSTANCE, pending) + + +if __name__ == '__main__': + main() diff --git a/scripts/mirror.py b/scripts/mirror.py new file mode 100644 index 0000000..66e40e0 --- /dev/null +++ b/scripts/mirror.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Reconcile all business Git references from Gitea to GitHub under one lock.""" + +import argparse +import os +from pathlib import Path +import shlex +import tempfile +import time + +from ci import SOURCE, locked, request_json, require, run, save + +TARGET = 'git@github.com:eezd/AlphaPulse.git' +DIRECTORY = Path('/docker/gitea/mirrors/AlphaPulse') + + +def refs(repository, remote=None, env=None): + args = ['ls-remote', remote, 'refs/heads/*', 'refs/tags/*'] if remote else [ + 'for-each-ref', '--format=%(objectname)\t%(refname)', 'refs/heads/', 'refs/tags/'] + output = run('git', '-C', repository, *args, env=env) + return {ref: sha for line in output.splitlines() for sha, ref in [line.split()] if not ref.endswith('^{}')} + + +def synchronize(directory, source, target, env=None): + directory = Path(directory) + with locked(directory / '.mirror.lock'): + repository = directory / 'repository.git' + if not repository.exists(): + run('git', 'init', '--bare', repository) + for _ in range(3): + run('git', '-C', repository, 'fetch', '--prune', '--no-tags', source, + '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', env=env) + snapshot = refs(repository) + require('refs/heads/main' in snapshot, 'Source main is missing; refusing destructive synchronization') + if run('git', '-C', repository, 'log', '--all', '--format=', '--name-only', '--', '.gitattributes', '**/.gitattributes'): + run('git', '-C', repository, 'lfs', 'fetch', '--all', source, env=env) + run('git', '-C', repository, 'lfs', 'push', '--all', target, env=env) + run('git', '-C', repository, 'push', '--atomic', '--prune', target, + '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', env=env) + require(refs(repository, target, env) == snapshot, 'GitHub references differ from the selected source snapshot') + if refs(repository, source, env) == snapshot: + save(directory / 'last-sync.json', {'refs': snapshot, 'verified_at': time.time()}) + return + raise RuntimeError('Source kept changing; retry the mirror') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args() + # A write-enabled deploy key limits GitHub access to this repository. + with tempfile.TemporaryDirectory(prefix='ap-mirror-auth-') as temporary: + key = Path(temporary) / 'key' + key.write_text(os.environ['ALPHAPULSE_GITHUB_SSH_KEY']) + key.chmod(0o600) + hosts = Path(temporary) / 'known_hosts' + hosts.write_text(''.join('github.com ' + entry + '\n' for entry in request_json('https://api.github.com/meta')['ssh_keys'])) + command = ['ssh', '-i', str(key), '-o', 'IdentitiesOnly=yes', '-o', 'StrictHostKeyChecking=yes', + '-o', 'UserKnownHostsFile=' + str(hosts)] + env = {**os.environ, 'GIT_SSH_COMMAND': shlex.join(command), 'GIT_TERMINAL_PROMPT': '0'} + synchronize(DIRECTORY, SOURCE, TARGET, env) + print('GitHub business references and history verified') + + +if __name__ == '__main__': + main() diff --git a/scripts/smoke_app.py b/scripts/smoke_app.py new file mode 100644 index 0000000..027929b --- /dev/null +++ b/scripts/smoke_app.py @@ -0,0 +1,96 @@ +"""Exercise the real API in the internal, synthetic CI network only.""" + +import asyncio +from datetime import UTC, datetime +import json +import os + +import httpx +from sqlalchemy import select +from websockets.asyncio.client import connect + +from alphapulse.core.db import SessionLocal +from alphapulse.core.redis import redis +from alphapulse.models import EconomicRelease, NewsItem + + +async def main(): + assert os.environ['DATABASE_URL'].startswith('postgresql+asyncpg://ci:ci-only@ap-ci-') + now = datetime.now(UTC) + async with SessionLocal() as session: + if await session.scalar(select(NewsItem.id).where(NewsItem.external_id == 'ci-synthetic')) is None: + session.add(NewsItem(source_code='wscn_news', external_id='ci-synthetic', url='https://example.invalid/ci', + title='CI synthetic news', summary='Isolated fixture', content_text='CI synthetic news', + content_html='

CI synthetic news

', published_at=now, content_hash='0' * 64)) + session.add(EconomicRelease(source='bls', external_id='ci-synthetic', family='cpi', title='CI CPI', + period=now.strftime('%Y-%m'), scheduled_at=now, schedule_url='https://example.invalid/ci', + schedule_checked_at=now, next_check_at=now, + values={'cpi_yoy': {'actual': '2.0', 'previous': '2.1'}})) + await session.commit() + origin = 'http://' + os.environ['SMOKE_WEB'] + async with httpx.AsyncClient(base_url=origin, timeout=15, trust_env=False) as client: + response = await client.get('/api/health') + response.raise_for_status() + assert response.json()['data']['commit'] == os.environ['APP_COMMIT'] + assert (await client.get('/api/news')).status_code == 401 + response = await client.post('/api/auth/login', json={'username': 'ci', 'password': 'ci-only'}) + response.raise_for_status() + token = response.json()['data']['token'] + client.headers['Authorization'] = 'Bearer ' + token + try: + for path in ['/api/auth/me', '/api/news?q=CI', '/api/crawler/sources', '/api/market/symbols', + '/api/skills/tree', '/api/trading/status', '/api/chat/session', + '/api/calendar?start=' + now.strftime('%Y-%m-%d')]: + response = await client.get(path) + response.raise_for_status() + assert response.json()['code'] == 0, path + assert 'CI synthetic news' in (await client.get('/api/news?q=CI')).text + assert 'CI CPI' in (await client.get('/api/calendar?start=' + now.strftime('%Y-%m-%d'))).text + for path in ['/', '/news', '/calendar', '/crawler', '/market', '/trading', '/chat', '/skills', '/settings']: + response = await client.get(path) + assert response.status_code == 200 and '