Apache Lucene open-source search software
script to run microbenchmarks across different ec2 instance types (#12787)
Robert Muirc28d174cd7ca
diff --git a/dev-tools/aws-jmh/Makefile b/dev-tools/aws-jmh/Makefilenew file mode 100644index 00000000000..e1e761b9357--- /dev/null+++ b/dev-tools/aws-jmh/Makefile@@ -0,0 +1,45 @@+.PHONY: test clean env+.DELETE_ON_ERROR:++# python interpreter to use+export PYTHON=python3+# improve ansible output+export ANSIBLE_CALLBACK_RESULT_FORMAT=yaml+# reasonable default verbosity, crank to debug issues+export ANSIBLE_VERBOSITY=0+# pipeline operations over ssh+export ANSIBLE_PIPELINING=true+# show breakdown of task times+export ANSIBLE_CALLBACKS_ENABLED=profile_tasks+# not useful since they are throwaways+export ANSIBLE_HOST_KEY_CHECKING=false+# run all machines in parallel+export ANSIBLE_FORKS=10++ifeq ($(AWS_ACCESS_KEY_ID),)+ $(error AWS_ACCESS_KEY_ID is not set)+endif++ifeq ($(AWS_SECRET_ACCESS_KEY),)+ $(error AWS_SECRET_ACCESS_KEY is not set)+endif++ifeq ($(PATCH_BRANCH),)+ $(error PATCH_BRANCH is not set)+endif++# runs benchmark across AWS instances+# always destroys anything created out there+test: env+ bash -c "set -ex; trap 'build/.env/bin/ansible-playbook destroy.yml' EXIT; build/.env/bin/ansible-playbook playbook.yml"++clean:+ rm -rf build++# keep venv in sync with requirements.txt+env: build/.env/bin/activate+build/.env/bin/activate: requirements.txt+ rm -rf build/.env+ $(PYTHON) -m venv build/.env+ build/.env/bin/pip install --no-input -U -r requirements.txt+ touch build/.env/bin/activatediff --git a/dev-tools/aws-jmh/README.txt b/dev-tools/aws-jmh/README.txtnew file mode 100644index 00000000000..ee2d4bb322f--- /dev/null+++ b/dev-tools/aws-jmh/README.txt@@ -0,0 +1,13 @@+Runs lucene microbenchmarks across a variety of CPUs in EC2.++Example:++export AWS_ACCESS_KEY_ID=xxxxx+export AWS_SECRET_ACCESS_KEY=yyyy+make PATCH_BRANCH=rmuir:some-speedup++Results file will be in build/report.txt++You can also pass additional JMH args if you want:++make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'diff --git a/dev-tools/aws-jmh/cloud-init.yml.j2 b/dev-tools/aws-jmh/cloud-init.yml.j2new file mode 100644index 00000000000..4463d79ef2b--- /dev/null+++ b/dev-tools/aws-jmh/cloud-init.yml.j2@@ -0,0 +1,8 @@+#cloud-config+package_upgrade: false+package_update: false+system_info:+ default_user:+ name: {{ user }}+ ssh_authorized_keys:+ - {{ lookup('file', ssh_key + '.pub') }}diff --git a/dev-tools/aws-jmh/cloudformation.yml.j2 b/dev-tools/aws-jmh/cloudformation.yml.j2new file mode 100644index 00000000000..82ac8e55978--- /dev/null+++ b/dev-tools/aws-jmh/cloudformation.yml.j2@@ -0,0 +1,49 @@+AWSTemplateFormatVersion: 2010-09-09+Resources:++ SecurityGroupIngress:+ Type: AWS::EC2::SecurityGroup+ Properties:+ VpcId: {{ vpc_info.vpcs[0].id }}+ GroupDescription: Allow inbound SSH traffic+ SecurityGroupIngress:+ - IpProtocol: "tcp"+ FromPort: 22+ ToPort: 22+ CidrIp: "0.0.0.0/0"+ Tags:+ - Key: Name+ Value: {{ stack_name }}-inbound-ssh++ SecurityGroupEgress:+ Type: AWS::EC2::SecurityGroup+ Properties:+ VpcId: {{ vpc_info.vpcs[0].id }}+ GroupDescription: Allow all outbound traffic+ SecurityGroupEgress:+ - IpProtocol: "-1"+ CidrIp: "0.0.0.0/0"+ Tags:+ - Key: Name+ Value: {{ stack_name }}-outbound-all++{% for instance in instances %}+ {{ instance.name }}Instance:+ Type: AWS::EC2::Instance+ DependsOn:+ - SecurityGroupIngress+ - SecurityGroupEgress+ Properties:+ InstanceType: {{ instance.type }}+ InstanceInitiatedShutdownBehavior: terminate+ ImageId: {{ instance.ami }}+ SecurityGroupIds:+ - !Ref SecurityGroupIngress+ - !Ref SecurityGroupEgress+ UserData: {{ lookup('template', 'cloud-init.yml.j2') | b64encode }}+ Tags:+ - Key: Name+ Value: {{ stack_name }}-{{ instance.name }}+ - Key: ShortName+ Value: {{ instance.name }}+{% endfor %}diff --git a/dev-tools/aws-jmh/destroy.yml b/dev-tools/aws-jmh/destroy.ymlnew file mode 100644index 00000000000..38beda2276c--- /dev/null+++ b/dev-tools/aws-jmh/destroy.yml@@ -0,0 +1,9 @@+- hosts: localhost+ gather_facts: false+ tasks:+ - name: Destroy cloudformation stack+ cloudformation:+ stack_name: "{{ stack_name }}"+ region: "{{ region }}"+ state: absent+ delegate_to: localhostdiff --git a/dev-tools/aws-jmh/group_vars/all.yml b/dev-tools/aws-jmh/group_vars/all.ymlnew file mode 100644index 00000000000..23c14c70b1e--- /dev/null+++ b/dev-tools/aws-jmh/group_vars/all.yml@@ -0,0 +1,65 @@+# cloudformation stack name+stack_name: lucene-jmh++# region to launch instances, make sure they are supported!+region: us-east-1++# ssh keypair to use for accessing instances+ssh_key: ~/.ssh/id_ed25519++# user to create remotely+user: lucene++# ami to use+x86_ami: ami-04cb4ca688797756f+arm_ami: ami-06f9c0b2ce386dda7++# instance types to spin up+instances:+ - name: haswell+ type: c4.large+ ami: "{{ x86_ami }}"+ - name: cascadelake+ type: c5.large+ ami: "{{ x86_ami }}"+ - name: icelake+ type: c6i.large+ ami: "{{ x86_ami }}"+ - name: sapphirerapids+ type: c7i.large+ ami: "{{ x86_ami }}"+ - name: zen2+ type: c5a.large+ ami: "{{ x86_ami }}"+ - name: zen3+ type: c6a.large+ ami: "{{ x86_ami }}"+ - name: zen4+ type: c7a.large+ ami: "{{ x86_ami }}"+ - name: graviton2+ type: c6g.large+ ami: "{{ arm_ami }}"+ - name: graviton3+ type: c7g.large+ ami: "{{ arm_ami }}"++jdk_version: 21.0.1+x86_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-x64_bin.tar.gz"+arm_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-aarch64_bin.tar.gz"+jdk_url: "{{ arm_jdk_url if ansible_architecture == 'aarch64' else x86_jdk_url }}"++main_branch: "apache:main"+patch_branch: "{{ lookup('env', 'PATCH_BRANCH') }}"++assemble_command: ./gradlew -p lucene/benchmark-jmh assemble+jmh_command: java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-10.0.0-SNAPSHOT.jar -rf text+jmh_args: "{{ lookup('env', 'JMH_ARGS', default='-p size=1024') }}"++# services that cause noise and don't help us+disable_services:+ - atd.service+ - logrotate.timer+ - sysstat-collect.timer+ - sysstat-summary.timer+ - systemd-tmpfiles-clean.timerdiff --git a/dev-tools/aws-jmh/playbook.yml b/dev-tools/aws-jmh/playbook.ymlnew file mode 100644index 00000000000..197a415c23e--- /dev/null+++ b/dev-tools/aws-jmh/playbook.yml@@ -0,0 +1,155 @@++# spin up aws hardware+- hosts: localhost+ gather_facts: false+ tasks:+ - name: Lookup default VPC+ ec2_vpc_net_info:+ region: "{{ region }}"+ filters:+ isDefault: true+ register: vpc_info++ - name: Create cloudformation stack+ cloudformation:+ stack_name: "{{ stack_name }}"+ region: "{{ region }}"+ template_body: "{{ lookup('template', 'cloudformation.yml.j2') }}"++ - name: Gather instance details+ ec2_instance_info:+ region: "{{ region }}"+ filters:+ instance-state-name: ["pending", "running"]+ "tag:aws:cloudformation:stack-name": "{{ stack_name }}"+ register: ec2_instances++ - name: Add instances to inventory+ add_host:+ name: "{{ item['tags']['ShortName'] }}"+ ansible_host: "{{ item['public_ip_address'] }}"+ ansible_user: "{{ user }}"+ ansible_ssh_private_key_file: "{{ ssh_key }}"+ args: "{{ item }}"+ with_items: "{{ ec2_instances.instances }}"++- hosts: all+ gather_facts: false+ tasks:+ - name: Wait for connection+ wait_for_connection:++ - name: Gather facts+ setup:++ - name: Configure kernel+ command:+ argv:+ - grubby+ - --remove-args=selinux_1 security=selinux quiet+ - --args=mitigations=0 random.trust_cpu=1 loglevel=7 selinux=0+ - --update-kernel=ALL+ become: true++ - name: Install packages+ package:+ name:+ - git+ become: true++ - name: Disable noisy services+ command:+ cmd: "systemctl disable --now {{ item }}"+ loop: "{{ disable_services }}"+ failed_when: false+ become: true++ - name: Reboot machine+ reboot:+ become: true++ - name: Download JDK+ unarchive:+ src: "{{ jdk_url }}"+ dest: .+ remote_src: true++ - name: Configure JDK+ blockinfile:+ path: .bashrc+ block: |+ export JAVA_HOME=/home/{{ user }}/jdk-{{ jdk_version }}+ export PATH=$JAVA_HOME/bin:$PATH++ - name: Configure Gradle+ blockinfile:+ path: .gradle/gradle.properties+ create: true+ block: |+ org.gradle.daemon=false++ - name: Checkout main+ git:+ repo: "[email protected]:{{ (main_branch | split(':'))[0] }}/lucene.git"+ version: "{{ (main_branch | split(':'))[1] }}"+ dest: main+ accept_newhostkey: true+ depth: 1++ - name: Checkout patch+ git:+ repo: "[email protected]:{{ (patch_branch | split(':'))[0] }}/lucene.git"+ version: "{{ (patch_branch | split(':'))[1] }}"+ dest: patch+ accept_newhostkey: true+ depth: 1++ - name: Assemble Sources+ command:+ cmd: "{{ assemble_command }}"+ chdir: "{{ item }}"+ loop:+ - main+ - patch++ - name: Run benchmark+ command:+ cmd: "{{ jmh_command }} -rff ~/{{ item }}.txt {{ jmh_args }}"+ chdir: "{{ item }}"+ loop:+ - main+ - patch++ - name: Read main results+ command:+ cmd: "cat main.txt"+ register: main_out++ - name: Read patch results+ command:+ cmd: "cat patch.txt"+ register: patch_out++ - name: Write Report+ copy:+ dest: "build/{{ inventory_hostname }}.log"+ content: |++ {{ inventory_hostname }}: `{{ ansible_processor }}`++ main+ ```+ {{ main_out.stdout }}+ ```+ patch+ ```+ {{ patch_out.stdout }}+ ```+ delegate_to: localhost++- hosts: localhost+ gather_facts: false+ tasks:+ - name: Create combined report+ shell:+ cmd: "cat build/*.log > build/report.txt"diff --git a/dev-tools/aws-jmh/requirements.txt b/dev-tools/aws-jmh/requirements.txtnew file mode 100644index 00000000000..09175f717fa--- /dev/null+++ b/dev-tools/aws-jmh/requirements.txt@@ -0,0 +1,4 @@+# automate botnet with ansible+ansible<9+# aws api+boto3<2diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradleindex 4d9759188cb..3c215c35223 100644--- a/gradle/validation/rat-sources.gradle+++ b/gradle/validation/rat-sources.gradle@@ -82,6 +82,9 @@ allprojects {// projects. Include their sources in the scan.include "buildSrc/src/**"include "dev-tools/missing-doclet/src/**"++ // do not let RAT attempt to scan a python venv, it gets lost and confused...+ exclude "dev-tools/aws-jmh/build/**"breakcase ":lucene:analysis:morfologik":