Pre-generate ci for modules
This commit is contained in:
Cédric Krier
File diff suppressed because it is too large
Load Diff
+267
-13
@@ -8,23 +8,277 @@ workflow:
|
||||
|
||||
stages:
|
||||
- preparation
|
||||
- execution
|
||||
- check
|
||||
- test
|
||||
- build
|
||||
|
||||
prepare-job:
|
||||
packages:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
script:
|
||||
- python .gitlab-scripts/generate-ci.py gitlab-generated-ci.yml
|
||||
- .gitlab-scripts/generate-packages.sh packages
|
||||
artifacts:
|
||||
paths:
|
||||
- gitlab-generated-ci.yml
|
||||
- packages/
|
||||
expire_in: 1 week
|
||||
|
||||
trigger-job:
|
||||
stage: execution
|
||||
trigger:
|
||||
forward:
|
||||
pipeline_variables: true
|
||||
strategy: depend
|
||||
include:
|
||||
- artifact: gitlab-generated-ci.yml
|
||||
job: prepare-job
|
||||
documentation:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
script:
|
||||
- .gitlab-scripts/generate-doc.sh docs
|
||||
artifacts:
|
||||
paths:
|
||||
- docs/
|
||||
expire_in: 1 week
|
||||
|
||||
.check:
|
||||
stage: check
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
|
||||
.check-doc:
|
||||
extends: .check
|
||||
needs:
|
||||
- documentation
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
DOC_DIR: doc
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- python -m sphinx -T -E -W -n -b html ${DOC_DIR} _build/html
|
||||
- python -m sphinx -T -E -W -n -b linkcheck ${DOC_DIR} _build
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/${DOC_DIR}/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
|
||||
check-doc-readthedocs:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
DOC_DIR: .
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/**/*
|
||||
|
||||
.check-code:
|
||||
extends: .check
|
||||
rules:
|
||||
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
|
||||
when: always
|
||||
|
||||
check-flake8:
|
||||
extends: .check-code
|
||||
script:
|
||||
- hg diff --rev s0 | flake8 --diff
|
||||
|
||||
check-isort:
|
||||
extends: .check-code
|
||||
script:
|
||||
- isort -m VERTICAL_GRID -p trytond -c `hg status --no-status --added --modified --rev s0`
|
||||
|
||||
check-dist:
|
||||
stage: check
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- twine check packages/*
|
||||
|
||||
check-ci-modules:
|
||||
extends: .check
|
||||
script:
|
||||
- python .gitlab-scripts/generate-ci.py $TMPDIR/gitab-ci-modules.yml
|
||||
- cmp .gitlab-ci-modules.yml $TMPDIR/gitab-ci-modules.yml
|
||||
|
||||
.test:
|
||||
stage: test
|
||||
rules:
|
||||
- if: $CI_COMMIT_SHA
|
||||
changes:
|
||||
- ${PACKAGE}/**/*
|
||||
when: on_success
|
||||
- if: '$CI_COMMIT_TAG =~ $TAG_PATTERN'
|
||||
when: on_success
|
||||
- when: never
|
||||
|
||||
.test-tox:
|
||||
extends: .test
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/python:${PYTHON_VERSION}
|
||||
variables:
|
||||
PIP_FIND_LINKS: file://${CI_PROJECT_DIR}/packages/
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
cache:
|
||||
paths:
|
||||
- .cache/pip/http
|
||||
before_script:
|
||||
- pip install tox
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
parallel:
|
||||
matrix:
|
||||
- PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
.test-db:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
DB_CACHE: /tmp/cache
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}-${DATABASE}" -c "${PACKAGE}"
|
||||
|
||||
.test-sqlite:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: sqlite
|
||||
|
||||
.test-postgresql:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgresql
|
||||
POSTGRESQL_URI: postgresql://postgres@postgresql:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgres
|
||||
alias: postgresql
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-postgis:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgis
|
||||
POSTGIS_URI: postgis://postgres@postgis:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgis/postgis
|
||||
alias: postgis
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-npm:
|
||||
extends: .test
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/node
|
||||
variables:
|
||||
NODE_CACHE_DIR: "$CI_PROJECT_DIR/.cache/npm"
|
||||
cache:
|
||||
paths:
|
||||
- .cache/npm
|
||||
before_script:
|
||||
- sh .gitlab-scripts/bower_allow_root.sh
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- npm -s install --cache ${NODE_CACHE_DIR} --unsafe-perm --legacy-peer-deps
|
||||
- npm test
|
||||
|
||||
.trigger-doc-build:
|
||||
stage: build
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- .gitlab-scripts/trigger-readthedocs.py -c "${RTD_CONFIG}" -p "${PACKAGE}" -b "${CI_COMMIT_BRANCH#branch/}"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/doc/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
allow_failure: true
|
||||
|
||||
trigger-doc-build-readthedocs:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/**/*
|
||||
allow_failure: true
|
||||
|
||||
check-doc-trytond:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: trytond
|
||||
|
||||
trigger-doc-build-trytond:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: trytond
|
||||
|
||||
test-trytond-sqlite:
|
||||
extends: .test-sqlite
|
||||
variables:
|
||||
PACKAGE: trytond
|
||||
TAG_PATTERN: '/^trytond-.*/'
|
||||
|
||||
test-trytond-postgresql:
|
||||
extends: .test-postgresql
|
||||
variables:
|
||||
PACKAGE: trytond
|
||||
TAG_PATTERN: '/^trytond-.*/'
|
||||
|
||||
check-doc-trytond-gis:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: trytond-gis
|
||||
|
||||
trigger-doc-build-trytond-gis:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: trytond-gis
|
||||
|
||||
test-trytond-gis-postgis:
|
||||
extends: .test-postgis
|
||||
variables:
|
||||
PACKAGE: trytond-gis
|
||||
TAG_PATTERN: '/^trytond-gis-.*/'
|
||||
|
||||
check-doc-proteus:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: proteus
|
||||
|
||||
trigger-doc-build-proteus:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: proteus
|
||||
|
||||
test-proteus:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
PACKAGE: proteus
|
||||
TAG_PATTERN: '/^proteus-.*/'
|
||||
|
||||
check-doc-tryton:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: tryton
|
||||
|
||||
trigger-doc-build-tryton:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: tryton
|
||||
|
||||
test-tryton:
|
||||
extends: .test-tox
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/tryton-test:${PYTHON_VERSION}
|
||||
variables:
|
||||
PACKAGE: tryton
|
||||
TAG_PATTERN: '/^tryton-.*/'
|
||||
|
||||
test-sao:
|
||||
extends: .test-npm
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/sao-test
|
||||
variables:
|
||||
PACKAGE: sao
|
||||
TAG_PATTERN: '/^sao-.*/'
|
||||
|
||||
include: '.gitlab-ci-modules.yml'
|
||||
|
||||
@@ -6,7 +6,7 @@ from itertools import chain
|
||||
from string import Template
|
||||
|
||||
BASE_DIR = os.path.dirname(__file__)
|
||||
TEMPLATE_DOC = Template("""
|
||||
TEMPLATE = Template("""
|
||||
check-doc-${name}:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
@@ -17,28 +17,6 @@ trigger-doc-build-${name}:
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
TAG_PATTERN: '/^${tag_name}-.*/'
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_IMAGE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-tox
|
||||
image: $${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/${package}-test:$${PYTHON_VERSION}
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
TAG_PATTERN: '/^${tag_name}-.*/'
|
||||
|
||||
""") # noqa: E501
|
||||
|
||||
TEMPLATE_DB = Template("""
|
||||
test-${name}-sqlite:
|
||||
extends: .test-sqlite
|
||||
variables:
|
||||
@@ -50,33 +28,8 @@ test-${name}-postgresql:
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
TAG_PATTERN: '/^${tag_name}-.*/'
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_GIS = Template("""
|
||||
test-${name}-postgis:
|
||||
extends: .test-postgis
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
TAG_PATTERN: '/^${tag_name}-.*/'
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_NPM_IMAGE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-npm
|
||||
image: $${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/${package}-test
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
TAG_PATTERN: '/^${tag_name}-.*/'
|
||||
|
||||
""")
|
||||
|
||||
NO_DB = {'proteus'}
|
||||
CUSTOM_IMAGE = {'tryton'}
|
||||
CUSTOM_NPM_IMAGE = {'sao'}
|
||||
GIS = {'trytond-gis'}
|
||||
|
||||
|
||||
def main(filename):
|
||||
if filename == '-':
|
||||
@@ -87,8 +40,7 @@ def main(filename):
|
||||
|
||||
|
||||
def render(file):
|
||||
file.write(open(os.path.join(BASE_DIR, 'gitlab-ci.yml'), 'r').read())
|
||||
packages = list(all_packages())
|
||||
packages = sorted(all_packages())
|
||||
for package in packages:
|
||||
name = tag_name = package.replace('/', '-')
|
||||
if tag_name.startswith('modules-'):
|
||||
@@ -98,26 +50,13 @@ def render(file):
|
||||
'package': package,
|
||||
'tag_name': tag_name,
|
||||
}
|
||||
if os.path.exists(os.path.join(package, 'doc')):
|
||||
file.write(TEMPLATE_DOC.substitute(mapping))
|
||||
if package in NO_DB:
|
||||
file.write(TEMPLATE.substitute(mapping))
|
||||
elif package in CUSTOM_IMAGE:
|
||||
file.write(TEMPLATE_IMAGE.substitute(mapping))
|
||||
elif package in CUSTOM_NPM_IMAGE:
|
||||
file.write(TEMPLATE_NPM_IMAGE.substitute(mapping))
|
||||
elif package in GIS:
|
||||
file.write(TEMPLATE_GIS.substitute(mapping))
|
||||
else:
|
||||
file.write(TEMPLATE_DB.substitute(mapping))
|
||||
|
||||
|
||||
def all_packages():
|
||||
root_dir = os.path.dirname(BASE_DIR)
|
||||
for filename in chain(
|
||||
glob.glob('*/setup.py', root_dir=root_dir),
|
||||
glob.glob('modules/*/setup.py', root_dir=root_dir),
|
||||
glob.glob('*/package.json', root_dir=root_dir)):
|
||||
glob.glob('modules/*/setup.py', root_dir=root_dir)):
|
||||
yield os.path.dirname(filename)
|
||||
|
||||
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
stages:
|
||||
- preparation
|
||||
- check
|
||||
- test
|
||||
- build
|
||||
|
||||
packages:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
script:
|
||||
- .gitlab-scripts/generate-packages.sh packages
|
||||
artifacts:
|
||||
paths:
|
||||
- packages/
|
||||
expire_in: 1 week
|
||||
|
||||
documentation:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
script:
|
||||
- .gitlab-scripts/generate-doc.sh docs
|
||||
artifacts:
|
||||
paths:
|
||||
- docs/
|
||||
expire_in: 1 week
|
||||
|
||||
.check:
|
||||
stage: check
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
|
||||
.check-doc:
|
||||
extends: .check
|
||||
needs:
|
||||
- documentation
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
DOC_DIR: doc
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- python -m sphinx -T -E -W -n -b html ${DOC_DIR} _build/html
|
||||
- python -m sphinx -T -E -W -n -b linkcheck ${DOC_DIR} _build
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/${DOC_DIR}/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
|
||||
check-doc-readthedocs:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
DOC_DIR: .
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/**/*
|
||||
|
||||
.check-code:
|
||||
extends: .check
|
||||
rules:
|
||||
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
|
||||
when: always
|
||||
|
||||
check-flake8:
|
||||
extends: .check-code
|
||||
script:
|
||||
- hg diff --rev s0 | flake8 --diff
|
||||
|
||||
check-isort:
|
||||
extends: .check-code
|
||||
script:
|
||||
- isort -m VERTICAL_GRID -p trytond -c `hg status --no-status --added --modified --rev s0`
|
||||
|
||||
check-dist:
|
||||
stage: check
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- twine check packages/*
|
||||
|
||||
.test:
|
||||
stage: test
|
||||
rules:
|
||||
- if: $CI_COMMIT_SHA
|
||||
changes:
|
||||
- ${PACKAGE}/**/*
|
||||
when: on_success
|
||||
- if: '$CI_COMMIT_TAG =~ $TAG_PATTERN'
|
||||
when: on_success
|
||||
- when: never
|
||||
|
||||
.test-tox:
|
||||
extends: .test
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/python:${PYTHON_VERSION}
|
||||
variables:
|
||||
PIP_FIND_LINKS: file://${CI_PROJECT_DIR}/packages/
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
cache:
|
||||
paths:
|
||||
- .cache/pip/http
|
||||
before_script:
|
||||
- pip install tox
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
parallel:
|
||||
matrix:
|
||||
- PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
.test-db:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
DB_CACHE: /tmp/cache
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}-${DATABASE}" -c "${PACKAGE}"
|
||||
|
||||
.test-sqlite:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: sqlite
|
||||
|
||||
.test-postgresql:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgresql
|
||||
POSTGRESQL_URI: postgresql://postgres@postgresql:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgres
|
||||
alias: postgresql
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-postgis:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgis
|
||||
POSTGIS_URI: postgis://postgres@postgis:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgis/postgis
|
||||
alias: postgis
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-npm:
|
||||
extends: .test
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/node
|
||||
variables:
|
||||
NODE_CACHE_DIR: "$CI_PROJECT_DIR/.cache/npm"
|
||||
cache:
|
||||
paths:
|
||||
- .cache/npm
|
||||
before_script:
|
||||
- sh .gitlab-scripts/bower_allow_root.sh
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- npm -s install --cache ${NODE_CACHE_DIR} --unsafe-perm --legacy-peer-deps
|
||||
- npm test
|
||||
|
||||
.trigger-doc-build:
|
||||
stage: build
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- .gitlab-scripts/trigger-readthedocs.py -c "${RTD_CONFIG}" -p "${PACKAGE}" -b "${CI_COMMIT_BRANCH#branch/}"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/doc/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
allow_failure: true
|
||||
|
||||
trigger-doc-build-readthedocs:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/**/*
|
||||
allow_failure: true
|
||||
Reference in New Issue
Block a user