[libvirt] [libvirt-jenkins-ci PATCH v2 01/16] ansible: Initial support

This commit adds just enough to bootstrap Jenkins workers and install packages required to build any project, such as gcc and make. Ubuntu 12-16 are supported, even though they're not actually used in libvirt CI, because there's a lot of overlap between this and Travis CI, which only supports Ubuntu instead. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ansible/.gitignore | 3 + ansible/Makefile | 12 +++ ansible/README.markdown | 60 ++++++++++++ ansible/ansible.cfg | 8 ++ ansible/bootstrap.yml | 15 +++ ansible/group_vars/all/main.yml | 8 ++ ansible/host_vars/libvirt-centos-6/main.yml | 3 + ansible/host_vars/libvirt-centos-7/main.yml | 3 + ansible/host_vars/libvirt-debian-8/main.yml | 3 + ansible/host_vars/libvirt-debian-9/main.yml | 3 + ansible/host_vars/libvirt-fedora-25/main.yml | 3 + ansible/host_vars/libvirt-fedora-26/main.yml | 3 + ansible/host_vars/libvirt-fedora-rawhide/main.yml | 3 + ansible/host_vars/libvirt-freebsd-10/main.yml | 10 ++ ansible/host_vars/libvirt-freebsd-11/main.yml | 10 ++ ansible/host_vars/libvirt-ubuntu-12/main.yml | 3 + ansible/host_vars/libvirt-ubuntu-14/main.yml | 3 + ansible/host_vars/libvirt-ubuntu-16/main.yml | 3 + ansible/inventory | 9 ++ ansible/site.yml | 19 ++++ ansible/tasks/base.yml | 108 ++++++++++++++++++++++ ansible/tasks/bootstrap.yml | 22 +++++ ansible/tasks/packages.yml | 66 +++++++++++++ ansible/vars/mappings.yml | 88 ++++++++++++++++++ ansible/vars/projects/base.yml | 17 ++++ 25 files changed, 485 insertions(+) create mode 100644 ansible/.gitignore create mode 100644 ansible/Makefile create mode 100644 ansible/README.markdown create mode 100644 ansible/ansible.cfg create mode 100644 ansible/bootstrap.yml create mode 100644 ansible/group_vars/all/main.yml create mode 100644 ansible/host_vars/libvirt-centos-6/main.yml create mode 100644 ansible/host_vars/libvirt-centos-7/main.yml create mode 100644 ansible/host_vars/libvirt-debian-8/main.yml create mode 100644 ansible/host_vars/libvirt-debian-9/main.yml create mode 100644 ansible/host_vars/libvirt-fedora-25/main.yml create mode 100644 ansible/host_vars/libvirt-fedora-26/main.yml create mode 100644 ansible/host_vars/libvirt-fedora-rawhide/main.yml create mode 100644 ansible/host_vars/libvirt-freebsd-10/main.yml create mode 100644 ansible/host_vars/libvirt-freebsd-11/main.yml create mode 100644 ansible/host_vars/libvirt-ubuntu-12/main.yml create mode 100644 ansible/host_vars/libvirt-ubuntu-14/main.yml create mode 100644 ansible/host_vars/libvirt-ubuntu-16/main.yml create mode 100644 ansible/inventory create mode 100644 ansible/site.yml create mode 100644 ansible/tasks/base.yml create mode 100644 ansible/tasks/bootstrap.yml create mode 100644 ansible/tasks/packages.yml create mode 100644 ansible/vars/mappings.yml create mode 100644 ansible/vars/projects/base.yml diff --git a/ansible/.gitignore b/ansible/.gitignore new file mode 100644 index 0000000..4a271f2 --- /dev/null +++ b/ansible/.gitignore @@ -0,0 +1,3 @@ +*.retry +*.swp +log diff --git a/ansible/Makefile b/ansible/Makefile new file mode 100644 index 0000000..39ebe52 --- /dev/null +++ b/ansible/Makefile @@ -0,0 +1,12 @@ +all: + +site: + @ansible-playbook site.yml + +bootstrap: + @ansible-playbook --ask-pass bootstrap.yml + +clean: + @rm -f *.retry log + +.PHONY: all site bootstrap clean diff --git a/ansible/README.markdown b/ansible/README.markdown new file mode 100644 index 0000000..4d464e1 --- /dev/null +++ b/ansible/README.markdown @@ -0,0 +1,60 @@ +Ansible playbooks for libvirt CI +================================ + +These can be used to turn a freshly installed machine into a worker for +the Jenkins-based libvirt CI. + +There are two main playbooks: + +* `bootstrap.yml`, used to perform the bootstrapping phase, that is, getting + guests to the point where Ansible can manage them fully and prompting the + user for a password is no longer required; + +* `site.yml`, used for the remaining configuration steps. + +Although you can use the playbooks directly, it's much more convenient to +call either `make bootstrap` or `make site` instead. + +Each guest only needs to be bootstrapped once; that said, both playbooks are +idempotent so there's no harm in applying them over and over again. + + +Requirements +------------ + +SSH must be running in the guest, and root login must be permitted. + + +CI use +------ + +After you have reinstalled a Jenkins worker, run `make bootstrap` followed +by `make site` and a reboot to get it ready for CI use. No further action +should be necessary. + +Adding new workers will require tweaking the inventory and host variables, +but it should be very easy to eg. use the Fedora 26 configuration to come +up with a working Fedora 27 configuration. + + +Development use +--------------- + +If you are a developer trying to reproduce a bug on some OS you don't have +easy access to, you can use these playbooks to create a suitable test +environment. + +Since the playbooks are intended mainly for CI use, you'll have to tweak them +a bit first, including: + +* trimming down the `inventory` file to just the guest you're interested in; + +* removing any references to the `jenkins` pseudo-project from + `host_vars/$guest/main.yml`, along with any references to projects you're + not interested to (this will cut down on the number of packages installed) + and any references to `jenkins_secret`; + +* deleting `host_vars/$guest/vault.yml` altogether. + +After performing these tweaks, you should be able to just run `make bootstrap` +followed by `make site` as usual. diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..ca02677 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +display_skipped_hosts = False +forks = 16 +inventory = ./inventory +log_path = ./log +nocows = 1 +pipelining = True +squash_actions = package diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml new file mode 100644 index 0000000..544dd9d --- /dev/null +++ b/ansible/bootstrap.yml @@ -0,0 +1,15 @@ +--- +- hosts: all + gather_facts: no + + tasks: + + # Bootstrap Ansible itself + - include: tasks/bootstrap.yml + +- hosts: all + + tasks: + + # Prepare the base environment + - include: tasks/base.yml diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml new file mode 100644 index 0000000..e8d3cb6 --- /dev/null +++ b/ansible/group_vars/all/main.yml @@ -0,0 +1,8 @@ +--- +ansible_user: root + +# Paths to various command. Can be overridden on a per-host basis +bash: /bin/bash +java: /usr/bin/java +make: /usr/bin/make +sudo: /usr/bin/sudo diff --git a/ansible/host_vars/libvirt-centos-6/main.yml b/ansible/host_vars/libvirt-centos-6/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-centos-6/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-centos-7/main.yml b/ansible/host_vars/libvirt-centos-7/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-centos-7/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-debian-8/main.yml b/ansible/host_vars/libvirt-debian-8/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-debian-8/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-debian-9/main.yml b/ansible/host_vars/libvirt-debian-9/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-debian-9/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-fedora-25/main.yml b/ansible/host_vars/libvirt-fedora-25/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-fedora-25/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-fedora-26/main.yml b/ansible/host_vars/libvirt-fedora-26/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-fedora-26/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-fedora-rawhide/main.yml b/ansible/host_vars/libvirt-fedora-rawhide/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-fedora-rawhide/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-freebsd-10/main.yml b/ansible/host_vars/libvirt-freebsd-10/main.yml new file mode 100644 index 0000000..d405d58 --- /dev/null +++ b/ansible/host_vars/libvirt-freebsd-10/main.yml @@ -0,0 +1,10 @@ +--- +ansible_python_interpreter: /usr/local/bin/python2 + +bash: /usr/local/bin/bash +java: /usr/local/bin/java +make: /usr/local/bin/gmake +sudo: /usr/local/bin/sudo + +projects: + - base diff --git a/ansible/host_vars/libvirt-freebsd-11/main.yml b/ansible/host_vars/libvirt-freebsd-11/main.yml new file mode 100644 index 0000000..d405d58 --- /dev/null +++ b/ansible/host_vars/libvirt-freebsd-11/main.yml @@ -0,0 +1,10 @@ +--- +ansible_python_interpreter: /usr/local/bin/python2 + +bash: /usr/local/bin/bash +java: /usr/local/bin/java +make: /usr/local/bin/gmake +sudo: /usr/local/bin/sudo + +projects: + - base diff --git a/ansible/host_vars/libvirt-ubuntu-12/main.yml b/ansible/host_vars/libvirt-ubuntu-12/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-ubuntu-12/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-ubuntu-14/main.yml b/ansible/host_vars/libvirt-ubuntu-14/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-ubuntu-14/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/host_vars/libvirt-ubuntu-16/main.yml b/ansible/host_vars/libvirt-ubuntu-16/main.yml new file mode 100644 index 0000000..1602406 --- /dev/null +++ b/ansible/host_vars/libvirt-ubuntu-16/main.yml @@ -0,0 +1,3 @@ +--- +projects: + - base diff --git a/ansible/inventory b/ansible/inventory new file mode 100644 index 0000000..d9d6f5f --- /dev/null +++ b/ansible/inventory @@ -0,0 +1,9 @@ +libvirt-centos-6 +libvirt-centos-7 +libvirt-debian-8 +libvirt-debian-9 +libvirt-fedora-25 +libvirt-fedora-26 +libvirt-fedora-rawhide +libvirt-freebsd-10 +libvirt-freebsd-11 diff --git a/ansible/site.yml b/ansible/site.yml new file mode 100644 index 0000000..c6e6152 --- /dev/null +++ b/ansible/site.yml @@ -0,0 +1,19 @@ +--- +- hosts: all + + vars_files: + - vars/mappings.yml + + tasks: + + # Prepare the base environment + - include: tasks/base.yml + + # Install build dependencies for each project + - include: tasks/packages.yml + with_items: + '{{ projects }}' + loop_control: + loop_var: project + when: + - projects is defined diff --git a/ansible/tasks/base.yml b/ansible/tasks/base.yml new file mode 100644 index 0000000..dd8d306 --- /dev/null +++ b/ansible/tasks/base.yml @@ -0,0 +1,108 @@ +--- +- name: Set additional facts (OS) + set_fact: + os_name: '{{ ansible_distribution }}' + os_version: '{{ ansible_distribution_major_version }}' + when: + - ansible_distribution_release != 'Rawhide' + +- name: Set additional facts (OS) + set_fact: + os_name: '{{ ansible_distribution }}' + os_version: Rawhide + when: + - ansible_distribution_release == 'Rawhide' + +- name: Set additional facts (package format) + set_fact: + package_format: deb + when: + - ( os_name == 'Debian' or + os_name == 'Ubuntu' ) + +- name: Set additional facts (package format) + set_fact: + package_format: pkg + when: + - os_name == 'FreeBSD' + +- name: Set additional facts (package format) + set_fact: + package_format: rpm + when: + - ( os_name == 'CentOS' or + os_name == 'Fedora' ) + +- name: Bootstrap the package module + command: apt-get install -y python-apt + args: + creates: /usr/lib/python2*/*-packages/apt + when: + - package_format == 'deb' + +- name: Bootstrap the package module + command: dnf install -y python2-dnf + args: + creates: /usr/lib*/python2*/*-packages/dnf + when: + - os_name == 'Fedora' + +- name: Update installed packages + package: + name: '*' + state: latest + when: + - package_format == 'rpm' + +- name: Update installed packages + apt: + upgrade: dist + update_cache: yes + when: + - package_format == 'deb' + +- name: Update installed packages + shell: pkg update && pkg upgrade -y + when: + - package_format == 'pkg' + +- name: Install base packages + package: + name: '{{ item }}' + state: present + with_items: + - bash + - git + - screen + - sudo + - vim + +- name: Remove unwanted packages + package: + name: '{{ item }}' + state: absent + with_items: + - nano + +- name: Permit file editing on SELinux-enabled systems + package: + name: libselinux-python + state: present + when: + - ( os_name == 'CentOS' or + os_name == 'Fedora' ) + +- name: Configure hostname + hostname: + name: '{{ inventory_hostname }}' + +- name: Configure root shell + user: + name: root + shell: '{{ bash }}' + +- name: Configure ssh access for the root user + authorized_key: + user: root + key: '{{ lookup("file", lookup("env", "HOME") + "/.ssh/id_rsa.pub") }}' + state: present diff --git a/ansible/tasks/bootstrap.yml b/ansible/tasks/bootstrap.yml new file mode 100644 index 0000000..24848c8 --- /dev/null +++ b/ansible/tasks/bootstrap.yml @@ -0,0 +1,22 @@ +--- +- name: Bootstrap the pkgng package manager + raw: env ASSUME_ALWAYS_YES=YES pkg bootstrap + when: + - inventory_hostname|search('freebsd') + +- name: Bootstrap Ansible + raw: yum install -y python2 + when: + - ( inventory_hostname|search('centos') or + inventory_hostname|search('fedora') ) + +- name: Bootstrap Ansible + raw: apt-get install -y python + when: + - ( inventory_hostname|search('debian') or + inventory_hostname|search('ubuntu') ) + +- name: Bootstrap Ansible + raw: pkg install -y python2 + when: + - inventory_hostname|search('freebsd') diff --git a/ansible/tasks/packages.yml b/ansible/tasks/packages.yml new file mode 100644 index 0000000..807b5c4 --- /dev/null +++ b/ansible/tasks/packages.yml @@ -0,0 +1,66 @@ +--- +- name: '{{ project }}: Load variables' + include_vars: + file: 'vars/projects/{{ project }}.yml' + +- set_fact: + temp: {} + +- name: '{{ project }}: Verify mappings' + fail: + msg: 'No mappings defined for {{ item }}' + with_items: + '{{ packages }}' + when: + - mappings[item] is undefined + +- name: '{{ project }}: Look up mappings (default)' + set_fact: + temp: '{{ temp|combine({ item: mappings[item]["default"] }) }}' + with_items: + '{{ packages }}' + when: + - mappings[item]["default"] is defined + +- name: '{{ project }}: Look up mappings (package format)' + set_fact: + temp: '{{ temp|combine({ item: mappings[item][package_format] }) }}' + with_items: + '{{ packages }}' + when: + - mappings[item][package_format] is defined + +- name: '{{ project }}: Look up mappings (OS name)' + set_fact: + temp: '{{ temp|combine({ item: mappings[item][os_name] }) }}' + with_items: + '{{ packages }}' + when: + - mappings[item][os_name] is defined + +- name: '{{ project }}: Look up mappings (OS version)' + set_fact: + temp: '{{ temp|combine({ item: mappings[item][os_name + os_version] }) }}' + with_items: + '{{ packages }}' + when: + - mappings[item][os_name + os_version] is defined + +- set_fact: + flattened: [] + +- name: '{{ project }}: Flatten package list' + set_fact: + flattened: '{{ flattened }} + [ "{{ temp[item] }}" ]' + with_items: + '{{ temp }}' + when: + - temp[item] != None + - temp[item] not in flattened + +- name: '{{ project }}: Install packages' + package: + name: '{{ item }}' + state: present + with_items: + '{{ flattened|sort }}' diff --git a/ansible/vars/mappings.yml b/ansible/vars/mappings.yml new file mode 100644 index 0000000..e9639eb --- /dev/null +++ b/ansible/vars/mappings.yml @@ -0,0 +1,88 @@ +--- +# Mappings are used to map the generic packages name used to define projects +# to the specific package names used by each OS. They implement inheritance, +# so you can define them without repeating yourself too much. +# +# Inheritance works the way you'd expect, that is, in order of increasing +# priority: +# +# - default +# - package format (deb, pkg, rpm) +# - OS name (CentOS, Debian, Fedora, FreeBSD, Ubuntu) +# - OS version (CentOS6, Debian9, FedoraRawhide, Ubuntu14 and so on) +# +# So something like +# +# make: +# default: make +# FreeBSD: gmake +# +# will result in the 'make' package being installed everywhere except +# for FreeBSD, where 'gmake' will be used instead. Clearing out existing +# mappings is possible as well, so +# +# ccache: +# default: ccache +# CentOS: +# +# will result in the 'ccache' package being installed everywhere except +# for CentOS, where nothing will be installed. + +mappings: + + autoconf: + default: autoconf + + automake: + default: automake + + autopoint: + deb: autopoint + pkg: gettext-tools + rpm: gettext-devel + + ccache: + default: ccache + CentOS: + + cppi: + Fedora: cppi + FreeBSD: cppi + + gcc: + default: gcc + FreeBSD: + + gettext: + default: gettext + + glibc: + deb: libc6-dev + rpm: glibc-devel + + libtool: + default: libtool + Debian: libtool-bin + Ubuntu16: libtool-bin + + libtoolize: + default: libtool + + make: + default: make + FreeBSD: gmake + + patch: + default: patch + + perl: + default: perl + FreeBSD: perl5 + + pkg-config: + default: pkgconf + rpm: pkgconfig + Ubuntu12: pkg-config + + rpmbuild: + rpm: rpm-build diff --git a/ansible/vars/projects/base.yml b/ansible/vars/projects/base.yml new file mode 100644 index 0000000..d82f6b9 --- /dev/null +++ b/ansible/vars/projects/base.yml @@ -0,0 +1,17 @@ +--- +packages: + - autoconf + - automake + - autopoint + - ccache + - cppi + - gcc + - gettext + - glibc + - libtool + - libtoolize + - make + - patch + - perl + - pkg-config + - rpmbuild -- 2.13.6