Prepare migration to monorepo
This commit is contained in:
Cédric Krier
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
echo '{ "allow_root": true }' > ~/.bowerrc
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/bin/env python
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from itertools import chain
|
||||
from string import Template
|
||||
|
||||
BASE_DIR = os.path.dirname(__file__)
|
||||
TEMPLATE_DOC = Template("""
|
||||
check-doc-${name}:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
|
||||
trigger-doc-build-${name}:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_IMAGE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-tox
|
||||
image: $${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/${package}-test:$${PYTHON_VERSION}
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
""") # noqa: E501
|
||||
|
||||
TEMPLATE_DB = Template("""
|
||||
test-${name}-sqlite:
|
||||
extends: .test-sqlite
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
test-${name}-postgresql:
|
||||
extends: .test-postgresql
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_GIS = Template("""
|
||||
test-${name}-postgis:
|
||||
extends: .test-postgis
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
""")
|
||||
|
||||
TEMPLATE_NPM_IMAGE = Template("""
|
||||
test-${name}:
|
||||
extends: .test-npm
|
||||
image: $${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/${package}-test
|
||||
variables:
|
||||
PACKAGE: ${package}
|
||||
${stage}
|
||||
${when}
|
||||
|
||||
""")
|
||||
|
||||
STAGE_MANUAL = "stage: manual"
|
||||
WHEN_MANUAL = """
|
||||
allow_failure: true
|
||||
rules:
|
||||
- when: manual
|
||||
"""
|
||||
NO_DB = {'proteus'}
|
||||
CUSTOM_IMAGE = {'tryton'}
|
||||
CUSTOM_NPM_IMAGE = {'sao'}
|
||||
GIS = {'trytond-gis'}
|
||||
|
||||
|
||||
def main(filename):
|
||||
if filename == '-':
|
||||
render(sys.stdout)
|
||||
else:
|
||||
with open(filename, 'w') as file:
|
||||
render(file)
|
||||
|
||||
|
||||
def render(file):
|
||||
file.write(open(os.path.join(BASE_DIR, 'gitlab-ci.yml'), 'r').read())
|
||||
packages = list(all_packages())
|
||||
modified = set(modified_packages(packages))
|
||||
for package in packages:
|
||||
mapping = {
|
||||
'name': package.replace('/', '-'),
|
||||
'package': package,
|
||||
'stage': STAGE_MANUAL if package not in modified else '',
|
||||
'when': WHEN_MANUAL if package not in modified else '',
|
||||
}
|
||||
if os.path.exists(os.path.join(package, 'doc')):
|
||||
file.write(TEMPLATE_DOC.substitute(mapping))
|
||||
if package in NO_DB:
|
||||
file.write(TEMPLATE.substitute(mapping))
|
||||
elif package in CUSTOM_IMAGE:
|
||||
file.write(TEMPLATE_IMAGE.substitute(mapping))
|
||||
elif package in CUSTOM_NPM_IMAGE:
|
||||
file.write(TEMPLATE_NPM_IMAGE.substitute(mapping))
|
||||
elif package in GIS:
|
||||
file.write(TEMPLATE_GIS.substitute(mapping))
|
||||
else:
|
||||
file.write(TEMPLATE_DB.substitute(mapping))
|
||||
|
||||
|
||||
def all_packages():
|
||||
root_dir = os.path.dirname(BASE_DIR)
|
||||
for filename in chain(
|
||||
glob.glob('**/setup.py', recursive=True, root_dir=root_dir),
|
||||
glob.glob('**/package.json', recursive=True, root_dir=root_dir)):
|
||||
yield os.path.dirname(filename)
|
||||
|
||||
|
||||
def modified_packages(packages):
|
||||
env = os.environ.copy()
|
||||
env['HGPLAIN'] = '1'
|
||||
if env.get('CI_COMMIT_TAG'):
|
||||
tagged = env['CI_COMMIT_TAG'].split('-')[0]
|
||||
for package in packages:
|
||||
if tagged in package:
|
||||
yield package
|
||||
else:
|
||||
if not subprocess.run(
|
||||
['hg', 'topic', '--current'],
|
||||
env=env, stdout=subprocess.DEVNULL).returncode:
|
||||
cmd = ['hg', 'status', '--rev', 's0', '--no-status']
|
||||
else:
|
||||
cmd = ['hg', 'status', '--change', '.', '--no-status']
|
||||
proc = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, text=True)
|
||||
proc.check_returncode()
|
||||
modified_files = proc.stdout.splitlines()
|
||||
|
||||
modified = set()
|
||||
for filename in modified_files:
|
||||
parts = filename.split(os.sep)
|
||||
modified.add(parts[0])
|
||||
modified.add(os.sep.join(parts[:2]))
|
||||
for package in packages:
|
||||
if package in modified:
|
||||
yield package
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
filename = sys.argv[1]
|
||||
except IndexError:
|
||||
filename = '-'
|
||||
main(filename)
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
OUTPUTDIR=`realpath "${1}"`
|
||||
mkdir -p "${OUTPUTDIR}"
|
||||
|
||||
find . -path '*/doc/conf.py' | while read path; do
|
||||
path=`dirname "${path}"`
|
||||
path=`dirname "${path}"`
|
||||
package=`basename "${path}"`
|
||||
package=$(echo ${package} | sed "s/_/-/g")
|
||||
if [ "${package}" = "trytond" ]; then
|
||||
package="server"
|
||||
elif [ "${package}" = "tryton" ]; then
|
||||
package="client-desktop"
|
||||
elif [ "${package}" = "proteus" ]; then
|
||||
package="client-library"
|
||||
elif [ "${package}" = "trytond-gis" ]; then
|
||||
package="backend-gis"
|
||||
else
|
||||
package="modules-${package}"
|
||||
fi
|
||||
(cd "${path}" && python -m sphinx -T -E -b html doc "${OUTPUTDIR}/${package}")
|
||||
done
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PACKAGES=`realpath "${1}"`
|
||||
mkdir -p "${PACKAGES}"
|
||||
|
||||
find . -name setup.py | while read path; do
|
||||
(cd "`dirname ${path}`" && python setup.py sdist && cp dist/* "${PACKAGES}")
|
||||
done
|
||||
@@ -0,0 +1,174 @@
|
||||
stages:
|
||||
- preparation
|
||||
- check
|
||||
- test
|
||||
- build
|
||||
- manual
|
||||
|
||||
packages:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
script:
|
||||
- .gitlab-scripts/generate-packages.sh packages
|
||||
artifacts:
|
||||
paths:
|
||||
- packages/
|
||||
expire_in: 1 week
|
||||
|
||||
documentation:
|
||||
stage: preparation
|
||||
rules:
|
||||
- when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
script:
|
||||
- .gitlab-scripts/generate-doc.sh docs
|
||||
artifacts:
|
||||
paths:
|
||||
- docs/
|
||||
expire_in: 1 week
|
||||
|
||||
.check:
|
||||
stage: check
|
||||
rules:
|
||||
- if: $CI_MERGE_REQUEST_ID != null
|
||||
when: always
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
|
||||
.check-doc:
|
||||
extends: .check
|
||||
needs:
|
||||
- documentation
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/sphinxdoc/sphinx
|
||||
variables:
|
||||
DOC_BASE_URL: ${CI_PROJECT_DIR}/docs
|
||||
DOC_DIR: doc
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- python -m sphinx -T -E -W -n -b html ${DOC_DIR} _build/html
|
||||
- python -m sphinx -T -E -W -n -b linkcheck ${DOC_DIR} _build
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/${DOC_DIR}/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
|
||||
check-doc-readthedocs:
|
||||
extends: .check-doc
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
DOC_DIR: .
|
||||
rules:
|
||||
- changes:
|
||||
- ${PACKAGE}/**/*
|
||||
|
||||
check-flake8:
|
||||
extends: .check
|
||||
script:
|
||||
- hg diff --rev s0 | flake8 --diff
|
||||
|
||||
check-isort:
|
||||
extends: .check
|
||||
script:
|
||||
- isort -m VERTICAL_GRID -p trytond -c `hg status --no-status --added --modified --rev s0`
|
||||
|
||||
check-dist:
|
||||
stage: check
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- twine check packages/*
|
||||
|
||||
.test:
|
||||
stage: test
|
||||
rules:
|
||||
- when: always
|
||||
|
||||
.test-tox:
|
||||
extends: .test
|
||||
needs:
|
||||
- packages
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/python:${PYTHON_VERSION}
|
||||
variables:
|
||||
CFLAGS: -O0
|
||||
PIP_FIND_LINKS: file://${CI_PROJECT_DIR}/packages/
|
||||
before_script:
|
||||
- pip install tox
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
parallel:
|
||||
matrix:
|
||||
- PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
.test-db:
|
||||
extends: .test-tox
|
||||
variables:
|
||||
DB_CACHE: /tmp/cache
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}-${DATABASE}" -c "${PACKAGE}"
|
||||
|
||||
.test-sqlite:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: sqlite
|
||||
|
||||
.test-postgresql:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgresql
|
||||
POSTGRESQL_URI: postgresql://postgres@postgresql:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgres
|
||||
alias: postgresql
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-postgis:
|
||||
extends: .test-db
|
||||
variables:
|
||||
DATABASE: postgis
|
||||
POSTGIS_URI: postgis://postgres@postgis:5432/
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
script:
|
||||
- tox -e "py${PYTHON_VERSION/./}" -c "${PACKAGE}"
|
||||
services:
|
||||
- name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/postgis/postgis
|
||||
alias: postgis
|
||||
command: ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
|
||||
.test-npm:
|
||||
extends: .test
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/node
|
||||
before_script:
|
||||
- sh .gitlab-scripts/bower_allow_root.sh
|
||||
script:
|
||||
- cd "${PACKAGE}"
|
||||
- npm -s install --unsafe-perm --legacy-peer-deps
|
||||
- npm test
|
||||
|
||||
.trigger-doc-build:
|
||||
stage: build
|
||||
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/tryton/ci
|
||||
script:
|
||||
- .gitlab-scripts/trigger-readthedocs.py -c "${RTD_CONFIG}" -p "${PACKAGE}" -b "${CI_COMMIT_BRANCH#branch/}"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/doc/**/*
|
||||
- ${PACKAGE}/tryton.cfg
|
||||
- ${PACKAGE}/CHANGELOG
|
||||
allow_failure: true
|
||||
|
||||
trigger-doc-build-readthedocs:
|
||||
extends: .trigger-doc-build
|
||||
variables:
|
||||
PACKAGE: readthedocs
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH =~ /^branch\/.*/
|
||||
changes:
|
||||
- ${PACKAGE}/**/*
|
||||
allow_failure: true
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Requires $RTD_CONFIG to point to a file providing the webhook data for each
|
||||
# package. The file is in Python cfg file format.
|
||||
# Each section defines on package. Example::
|
||||
#
|
||||
# [modules/party]
|
||||
# url: https://readthedocs.org/api/v2/webhook/modules-party/12345/
|
||||
# token: 93afe1ce2d1107597f69774009964990c0643dad
|
||||
|
||||
import configparser
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from urllib import request
|
||||
from urllib.parse import urlencode
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('--config', '-c', required=True)
|
||||
parser.add_argument('--package', '-p', required=True)
|
||||
parser.add_argument('--branches', '-b', nargs='*')
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(args.config)
|
||||
|
||||
url = cfg.get(args.package, 'url')
|
||||
token = cfg.get(args.package, 'token')
|
||||
if not url:
|
||||
sys.exit(f"Missing url for {args.package!r}")
|
||||
elif not token:
|
||||
sys.exit(f"Missing token for {args.package!r}")
|
||||
|
||||
data = [
|
||||
('token', token),
|
||||
*(('branches', branch) for branch in args.branches),
|
||||
]
|
||||
|
||||
req = request.Request(
|
||||
url,
|
||||
data=urlencode(data).encode(),
|
||||
headers={
|
||||
'User-Agent': 'python-urllib/%s' % request.__version__,
|
||||
})
|
||||
with request.urlopen(req) as resp:
|
||||
print(resp.read())
|
||||
Reference in New Issue
Block a user