[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 5 commits: Add script and cronjob to update debian-sso CA certificate and revocation list
Mattia Rizzolo
gitlab at salsa.debian.org
Wed May 16 11:36:18 BST 2018
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
c09f6b9f by Mattia Rizzolo at 2018-05-16T11:58:07+02:00
Add script and cronjob to update debian-sso CA certificate and revocation list
More information on https://wiki.debian.org/DebianSingleSignOn#Documentation_for_web_application_owners
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
a9a895f4 by Mattia Rizzolo at 2018-05-16T12:02:10+02:00
reproducible: add a cgi script to schedule builds over HTTPS
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
93d449dd by Mattia Rizzolo at 2018-05-16T12:07:34+02:00
Configure apache to require authentication on https://tests.reproducible-builds.org/cgi-bin/schedule
More info on https://wiki.debian.org/DebianSingleSignOn#Documentation_for_web_application_owners
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
a06daad4 by Mattia Rizzolo at 2018-05-16T12:31:21+02:00
reproducible: Add jenkins job to build the diffoscope.org website
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
66eac719 by Mattia Rizzolo at 2018-05-16T12:35:48+02:00
apache: configure to serve http://diffoscope.org (no ssl for now)
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
6 changed files:
- + bin/cgi-bin/schedule
- hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
- hosts/jenkins/etc/cron.d/dsa
- + hosts/jenkins/usr/local/bin/update-debsso-ca
- job-cfg/reproducible.yaml
- update_jdn.sh
Changes:
=====================================
bin/cgi-bin/schedule
=====================================
--- /dev/null
+++ b/bin/cgi-bin/schedule
@@ -0,0 +1,136 @@
+#!/usr/bin/python3
+
+# Copyright © 2018 Mattia Rizzolo <mattia at debian.org>
+# Licensed under GPL-2
+
+import os
+import re
+import cgi
+import cgitb
+import subprocess
+cgitb.enable()
+
+
+class ValidationError(Exception):
+ def __init__(self, message):
+ super().__init__(message)
+ print('Status: 400 Bad Request')
+ print('Content-Type: text/html; charset="utf-8"')
+ print()
+ print(message)
+
+
+def sanify_field(field_name, field_text):
+ sane_re = re.compile(r'^[a-zA-Z0-9_.+-]+$')
+ if not sane_re.match(field_text):
+ err = '"{}" is not sane (does not match {})'.format(field_name, sane_re)
+ raise ValidationError(err)
+
+def validate(form):
+ if 'pkg' not in form:
+ raise ValidationError('no packages specified')
+ for pkg in form.getlist('pkg'):
+ sanify_field('pkg', pkg)
+
+ known_opts = (
+ 'dry-run',
+ 'keep-artifacts',
+ 'notify',
+ 'notify-on-start',
+ )
+ known_opts2 = (
+ 'message',
+ 'status',
+ 'issue',
+ 'after',
+ 'before',
+ )
+ args = []
+ for opt in known_opts:
+ value = form.getvalue(opt)
+ if value:
+ if value in ('yes', 'true'):
+ args.append('--{}'.format(opt))
+ for opt in known_opts2:
+ value = form.getvalue(opt)
+ if value:
+ sanify_field(opt, value)
+ args.append('--{} {}'.format(opt, value))
+ for f in ('suite', 'architecture'):
+ for i in form.getlist(f):
+ sanify_field(f, i)
+ return args
+
+
+def main(args):
+ processes = []
+ failure = False
+ for s in form.getlist('suite'):
+ for a in form.getlist('architecture'):
+ cmd = (
+ '/srv/jenkins/bin/reproducible_remote_scheduler.py',
+ *args,
+ '--dry-run',
+ '--architecture', a,
+ '--suite', s,
+ *form.getlist('pkg'),
+ )
+ print('Executing: ', cmd)
+ try:
+ p = subprocess.run(
+ cmd,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env={'LC_USER': user},
+ )
+ processes.append(p)
+ except subprocess.CalledProcessError as e:
+ processes.append(e)
+ failure = True
+
+ if failure:
+ print('Status: 520 Unknown Error')
+ print('Content-Type: text/plain; charset="utf-8"')
+ print()
+ print('Failed to schedule packages. Please try again later or contact us for support.')
+ print()
+ print('Error log:')
+ else:
+ print('Status: 200 OK')
+ print('Content-Type: text/plain; charset="utf-8"')
+ print()
+ print('Successfully scheduled the requested packages.')
+ print()
+ print('Scheduling log:')
+ for p in processes:
+ print()
+ print('Command ran: ', p.args)
+ print(p.stdout.decode('utf-8', errors='ignore').strip())
+ print('Return code: ', p.returncode)
+
+
+# Check whether the user has successfully authenticated
+try:
+ user = os.environ['SSL_CLIENT_S_DN_CN']
+except KeyError:
+ user = None
+ print('Status: 496 SSL Certificate Required')
+ print('Content-Type: text/plain; charset="utf-8"')
+ print()
+ print('You need to authenticate with a Debian SSO certificate to use this service')
+else:
+ try:
+ form = cgi.FieldStorage()
+ main(validate(form))
+ except Exception:
+ print('Status: 500 Internal Server Error')
+ print('Content-Type: text/html; charset="utf-8"')
+ print()
+ cgitb.handler()
+
+print()
+
+print('WARNING: This endpoint is still a WORK IN PROGRESS. This means that the option details can change, and it will not commit any change.')
+print('User: {}'.format(user))
+print(form)
=====================================
hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
=====================================
--- a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
+++ b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
@@ -60,6 +60,17 @@ Use https-redirect tests.reproducible-builds.org
Use https-redirect reproducible-builds.org
Use https-redirect www.reproducible-builds.org
+<VirtualHost *:80>
+ Use common-directives diffoscope.org
+ DocumentRoot /srv/diffoscope.org/wwww
+ AddDefaultCharset utf-8
+
+ <Directory /srv/reproducible-builds.org/www>
+ AllowOverride None
+ Require all granted
+ </Directory>
+</VirtualHost>
+
<VirtualHost *:443>
Use common-directives jenkins.debian.net
Use common-directives-ssl jenkins.debian.net
@@ -139,6 +150,18 @@ Use https-redirect www.reproducible-builds.org
Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
Require all granted
</Directory>
+ # Use the sso.debian.org CA to validate client certificates
+ # Keep these files up to date with update-debsso-ca
+ SSLCACertificateFile /etc/apache2/ssl/debsso/debsso.crt
+ SSLCARevocationCheck chain
+ SSLCARevocationFile /etc/apache2/ssl/debsso/debsso.crl
+ <Location /cgi-bin/schedule>
+ # Export data about the certificate to the environment
+ SSLOptions +StdEnvVars
+ # Allow access if one does not have a valid certificate,
+ # so we can show a decent error message
+ SSLVerifyClient optional
+ </Location>
<Proxy *>
Require all granted
=====================================
hosts/jenkins/etc/cron.d/dsa
=====================================
--- a/hosts/jenkins/etc/cron.d/dsa
+++ b/hosts/jenkins/etc/cron.d/dsa
@@ -9,3 +9,4 @@ MAILTO=root
0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel
2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages
30 * * * * root ( /usr/sbin/service ntp stop ; sleep 2 ; /usr/sbin/ntpdate de.pool.ntp.org ; /usr/sbin/service ntp start ) > /dev/null
+0 0 * * * root mkdir -p /etc/apache2/ssl/debsso && /usr/local/bin/update-debsso-ca --destdir /etc/apache2/ssl/debsso
=====================================
hosts/jenkins/usr/local/bin/update-debsso-ca
=====================================
--- /dev/null
+++ b/hosts/jenkins/usr/local/bin/update-debsso-ca
@@ -0,0 +1,113 @@
+#!/usr/bin/python3
+
+# Originally downloaded from https://anonscm.debian.org/cgit/debian-sso/debian-sso.git/tree/update-debsso-ca
+
+# Download new versions of the CA certificate and Certificate Revocation List
+# from sso.debian.org and write them out atomically.
+
+import requests
+import tempfile
+import argparse
+import os
+import subprocess
+import ssl
+
+class atomic_writer(object):
+ """
+ Atomically write to a file
+ """
+ def __init__(self, fname, mode, osmode=0o644, sync=True, **kw):
+ self.fname = fname
+ self.osmode = osmode
+ self.sync = sync
+ dirname = os.path.dirname(self.fname)
+ self.fd, self.abspath = tempfile.mkstemp(dir=dirname, text="b" not in mode)
+ self.outfd = open(self.fd, mode, closefd=True, **kw)
+
+ def __enter__(self):
+ return self.outfd
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if exc_type is None:
+ self.outfd.flush()
+ if self.sync: os.fdatasync(self.fd)
+ os.fchmod(self.fd, self.osmode)
+ os.rename(self.abspath, self.fname)
+ else:
+ os.unlink(self.abspath)
+ self.outfd.close()
+ return False
+
+
+def get_url(url):
+ """
+ Fetch a URL and return the raw result as bytes
+ """
+ bundle='/etc/ssl/ca-debian/ca-certificates.crt'
+ if os.path.exists(bundle):
+ res = requests.get(url, verify=bundle)
+ else:
+ res = requests.get(url)
+ res.raise_for_status()
+ return res.content
+
+
+def update_file(pathname, content, validate=None):
+ """
+ Write content on pathname atomically, and do nothing if pathname exists and
+ has the same content as `content`.
+
+ Returns True if the file has been updated, else False.
+ """
+ try:
+ with open(pathname, "rb") as fd:
+ existing = fd.read()
+ except OSError:
+ existing = None
+
+ if existing == content: return False
+
+ # Validate the contents
+ if validate:
+ validate(content)
+
+ with atomic_writer(pathname, "wb", osmode=0o644) as out:
+ out.write(content)
+ return True
+
+def validate_crt(data):
+ ssl.PEM_cert_to_DER_cert(data.decode("utf-8"))
+
+def validate_crl(data):
+ if not data.startswith(b"-----BEGIN X509 CRL-----"):
+ raise RuntimeError("Data does not begin with a CRL signature")
+ if not data.endswith(b"-----END X509 CRL-----\n"):
+ raise RuntimeError("Data does not end with a CRL footer")
+
+def update(destdir):
+ # Fetch the certificate and the CRL
+ cert = get_url("https://sso.debian.org/ca/ca.pem")
+ crl = get_url("https://sso.debian.org/ca/ca.crl")
+
+ # Write them out atomically
+
+ updated = False
+ updated = update_file(os.path.join(destdir, "debsso.crt"), cert, validate=validate_crt) or updated
+ updated = update_file(os.path.join(destdir, "debsso.crl"), crl, validate=validate_crl) or updated
+ return updated
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--destdir", default=".", help="destination directory. Default: .")
+ parser.add_argument("--onupdate", help="command to run if the file has been updated. Default: do not run anything.")
+ args = parser.parse_args()
+
+ if update(args.destdir):
+ if args.onupdate:
+ subprocess.check_call(["sh", "-c", args.onupdate])
+
+
+
+if __name__ == "__main__":
+ main()
=====================================
job-cfg/reproducible.yaml
=====================================
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -739,6 +739,13 @@
my_gitrepo: 'https://salsa.debian.org/reproducible-builds/reproducible-website.git'
my_gitbranches: 'origin/master'
my_shell: 'jekyll build -s . -d /srv/reproducible-builds.org/www'
+ - 'builds_diffoscope_website':
+ my_description: 'Build https://diffoscope.org/ on every git commit to https://salsa.debian.org/reproducible-builds/diffoscope-website.git'
+ my_timed: ''
+ my_hname: ''
+ my_gitrepo: 'https://salsa.debian.org/reproducible-builds/diffoscope-website.git'
+ my_gitbranches: 'origin/master'
+ my_shell: 'mkdir -pv /srv/diffoscope.org/www && rsync -rptHCv --dry-run ./ /srv/diffoscope.org/www/'
my_gitbranches: 'master'
my_recipients: 'jenkins+debian-reproducible qa-jenkins-scm at lists.alioth.debian.org'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
=====================================
update_jdn.sh
=====================================
--- a/update_jdn.sh
+++ b/update_jdn.sh
@@ -447,6 +447,7 @@ if [ -f /etc/debian_version ] ; then
procmail
python3-debian
python3-pystache
+ python3-requests
python3-sqlalchemy
python3-xdg
python3-yaml
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6b82fdc6ed653fa1fd384196578e2b11328a8bd6...66eac7196788e017046939eb2d502f0f01053798
---
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6b82fdc6ed653fa1fd384196578e2b11328a8bd6...66eac7196788e017046939eb2d502f0f01053798
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180516/3a7a0d2b/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list