Adopting a Red Hat OpenStack Platform 17.1 deployment

[Default]

local_config_dir=/tmp/
service_config_file=config.yaml

[Tripleo]

ssh_cmd=ssh -F ssh.config (1)
director_host=standalone (2)
container_engine=podman
connection=ssh
remote_config_path=/tmp/tripleo
local_config_path=/tmp/

[Openshift]

ocp_local_config_path=/tmp/ocp
connection=local
ssh_cmd=""

Host *
    IdentitiesOnly yes

Host virthost
    Hostname virthost
    IdentityFile ~/.ssh/id_rsa
    User root
    StrictHostKeyChecking no
    UserKnownHostsFile=/dev/null


Host standalone
    Hostname standalone
    IdentityFile ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa
    User root
    StrictHostKeyChecking no
    UserKnownHostsFile=/dev/null

Host crc
    Hostname crc
    IdentityFile ~/.ssh/id_rsa
    User stack
    StrictHostKeyChecking no
    UserKnownHostsFile=/dev/null

$ os-diff configure -i tripleo-ansible-inventory.yaml -o ssh.config --yaml

export PULL_OPENSTACK_CONFIGURATION_DATABASES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo "$PULL_OPENSTACK_CONFIGURATION_DATABASES"

export PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root -p"$SOURCE_DB_ROOT_PASSWORD" | grep -v OK)
echo "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"

export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" nova_api -e \
    'select uuid,name,transport_url,database_connection,disabled from cell_mappings;')
echo "$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS"

export PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" nova_api -e \
    "select host from nova.services where services.binary='nova-compute';")
echo "$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES"

export PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER1_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells)
echo "$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS"

SRIOV_AGENTS=$ oc run mariadb-client mysql -rsh "$SOURCE_MARIADB_IP" \
-uroot -p"$SOURCE_DB_ROOT_PASSWORD" ovs_neutron -e \
"select host, configurations from agents where agents.binary='neutron-sriov-nic-agent';"

$ cat >~/.source_cloud_exported_variables <<EOF
PULL_OPENSTACK_CONFIGURATION_DATABASES="$PULL_OPENSTACK_CONFIGURATION_DATABASES"
PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK="$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"
PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS"
PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES="$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES"
PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS"
SRIOV_AGENTS="$SRIOV_AGENTS"
EOF

$ oc project openstack

# in install_yamls
make input

$ oc set data secret/osp-secret "AdminPassword=$ADMIN_PASSWORD"

$ oc set data secret/osp-secret "AodhPassword=$AODH_PASSWORD"
$ oc set data secret/osp-secret "BarbicanPassword=$BARBICAN_PASSWORD"
$ oc set data secret/osp-secret "CeilometerPassword=$CEILOMETER_PASSWORD"
$ oc set data secret/osp-secret "CinderPassword=$CINDER_PASSWORD"
$ oc set data secret/osp-secret "GlancePassword=$GLANCE_PASSWORD"
$ oc set data secret/osp-secret "HeatAuthEncryptionKey=$HEAT_AUTH_ENCRYPTION_KEY"
$ oc set data secret/osp-secret "HeatPassword=$HEAT_PASSWORD"
$ oc set data secret/osp-secret "HeatStackDomainAdminPassword=$HEAT_STACK_DOMAIN_ADMIN_PASSWORD"
$ oc set data secret/osp-secret "IronicPassword=$IRONIC_PASSWORD"
$ oc set data secret/osp-secret "IronicInspectorPassword=$IRONIC_PASSWORD"
$ oc set data secret/osp-secret "ManilaPassword=$MANILA_PASSWORD"
$ oc set data secret/osp-secret "MetadataSecret=$METADATA_SECRET"
$ oc set data secret/osp-secret "NeutronPassword=$NEUTRON_PASSWORD"
$ oc set data secret/osp-secret "NovaPassword=$NOVA_PASSWORD"
$ oc set data secret/osp-secret "OctaviaPassword=$OCTAVIA_PASSWORD"
$ oc set data secret/osp-secret "PlacementPassword=$PLACEMENT_PASSWORD"
$ oc set data secret/osp-secret "SwiftPassword=$SWIFT_PASSWORD"

apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  tls:
    podLevel:
      enabled: true
      internal:
        ca:
          customIssuer: rootca-internal
      libvirt:
        ca:
          customIssuer: rootca-internal
      ovn:
        ca:
          customIssuer: rootca-internal
    ingress:
      ca:
        customIssuer: rootca-internal
      enabled: true

apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  tls:
    podLevel:
      enabled: false
    ingress:
      enabled: false

oc apply -f - <<EOF
apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  secret: osp-secret
  storageClass: local-storage (1)

  barbican:
    enabled: false
    template:
      barbicanAPI: {}
      barbicanWorker: {}
      barbicanKeystoneListener: {}

  cinder:
    enabled: false
    template:
      cinderAPI: {}
      cinderScheduler: {}
      cinderBackup: {}
      cinderVolumes: {}

  dns:
    template:
      override:
        service:
          metadata:
            annotations:

              metallb.universe.tf/address-pool: ctlplane
              metallb.universe.tf/allow-shared-ip: ctlplane
              metallb.universe.tf/loadBalancerIPs: 192.168.122.80
          spec:
            type: LoadBalancer
      options:
      - key: server
        values:
        - 192.168.122.1
      replicas: 1

  glance:
    enabled: false
    template:
      glanceAPIs: {}

  heat:
    enabled: false
    template: {}

  horizon:
    enabled: false
    template: {}

  ironic:
    enabled: false
    template:
      ironicConductors: []

  keystone:
    enabled: false
    template: {}

  manila:
    enabled: false
    template:
      manilaAPI: {}
      manilaScheduler: {}
      manilaShares: {}

  mariadb:
    enabled: false
    templates: {}

  galera:
    enabled: true
    templates:
      openstack:
        secret: osp-secret
        replicas: 3
        storageRequest: 500M
      openstack-cell1:
        secret: osp-secret
        replicas: 3
        storageRequest: 500M

  memcached:
    enabled: true
    templates:
      memcached:
        replicas: 3

  neutron:
    enabled: false
    template: {}

  nova:
    enabled: false
    template: {}

  ovn:
    enabled: false
    template:
      ovnController:
        networkAttachment: tenant
        nodeSelector:
          node: non-existing-node-name
      ovnNorthd:
        replicas: 0
      ovnDBCluster:
        ovndbcluster-nb:
          dbType: NB
          networkAttachment: internalapi
        ovndbcluster-sb:
          dbType: SB
          networkAttachment: internalapi
  placement:
    enabled: false
    template: {}

  rabbitmq:
    templates:
      rabbitmq:
        override:
          service:
            metadata:
              annotations:
                metallb.universe.tf/address-pool: internalapi
                metallb.universe.tf/loadBalancerIPs: 172.17.0.85
            spec:
              type: LoadBalancer
      rabbitmq-cell1:
        override:
          service:
            metadata:
              annotations:
                metallb.universe.tf/address-pool: internalapi
                metallb.universe.tf/loadBalancerIPs: 172.17.0.86
            spec:
              type: LoadBalancer

  telemetry:
    enabled: false

  swift:
    enabled: false
    template:
      swiftRing:
        ringReplicas: 1
      swiftStorage:
        replicas: 0
      swiftProxy:
        replicas: 1
EOF

$ oc apply -f - <<EOF
apiVersion: v1
data:
  ceph.client.openstack.keyring: $CEPH_KEY
  ceph.conf: $CEPH_CONF
kind: Secret
metadata:
  name: ceph-conf-files
  namespace: openstack
type: Opaque
EOF

apiVersion: v1
kind: Secret
metadata:
  name: ceph-conf-files
  namespace: openstack
stringData:
  ceph.client.openstack.keyring: |
    [client.openstack]
        key = <secret key>
        caps mgr = "allow *"
        caps mon = "allow r, profile rbd"
        caps osd = "pool=vms, profile rbd pool=volumes, profile rbd pool=images, allow rw pool manila_data'
  ceph.conf: |
    [global]
    fsid = 7a1719e8-9c59-49e2-ae2b-d7eb08c695d4
    mon_host = 10.1.1.2,10.1.1.3,10.1.1.4

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  extraMounts:
    - name: v1
      region: r1
      extraVol:
        - propagation:
          - CinderVolume
          - CinderBackup
          - GlanceAPI
          - ManilaShare
          extraVolType: Ceph
          volumes:
          - name: ceph
            projected:
              sources:
              - secret:
                  name: ceph-conf-files
          mounts:
          - name: ceph
            mountPath: "/etc/ceph"
            readOnly: true
'

# check the co-location and ordering constraints concerning "manila-share"
sudo pcs constraint list --full

# remove these constraints
sudo pcs constraint remove colocation-openstack-manila-share-ceph-nfs-INFINITY
sudo pcs constraint remove order-ceph-nfs-openstack-manila-share-Optional

# Update the services list to be stopped
ServicesToStop=("tripleo_aodh_api.service"
                "tripleo_aodh_api_cron.service"
                "tripleo_aodh_evaluator.service"
                "tripleo_aodh_listener.service"
                "tripleo_aodh_notifier.service"
                "tripleo_ceilometer_agent_central.service"
                "tripleo_ceilometer_agent_notification.service"
                "tripleo_horizon.service"
                "tripleo_keystone.service"
                "tripleo_barbican_api.service"
                "tripleo_barbican_worker.service"
                "tripleo_barbican_keystone_listener.service"
                "tripleo_cinder_api.service"
                "tripleo_cinder_api_cron.service"
                "tripleo_cinder_scheduler.service"
                "tripleo_cinder_volume.service"
                "tripleo_cinder_backup.service"
                "tripleo_collectd.service"
                "tripleo_glance_api.service"
                "tripleo_gnocchi_api.service"
                "tripleo_gnocchi_metricd.service"
                "tripleo_gnocchi_statsd.service"
                "tripleo_manila_api.service"
                "tripleo_manila_api_cron.service"
                "tripleo_manila_scheduler.service"
                "tripleo_neutron_api.service"
                "tripleo_placement_api.service"
                "tripleo_nova_api_cron.service"
                "tripleo_nova_api.service"
                "tripleo_nova_conductor.service"
                "tripleo_nova_metadata.service"
                "tripleo_nova_scheduler.service"
                "tripleo_nova_vnc_proxy.service"
                "tripleo_aodh_api.service"
                "tripleo_aodh_api_cron.service"
                "tripleo_aodh_evaluator.service"
                "tripleo_aodh_listener.service"
                "tripleo_aodh_notifier.service"
                "tripleo_ceilometer_agent_central.service"
                "tripleo_ceilometer_agent_compute.service"
                "tripleo_ceilometer_agent_ipmi.service"
                "tripleo_ceilometer_agent_notification.service"
                "tripleo_ovn_cluster_northd.service"
                "tripleo_ironic_neutron_agent.service"
                "tripleo_ironic_api.service"
                "tripleo_ironic_inspector.service"
                "tripleo_ironic_conductor.service")

PacemakerResourcesToStop=("openstack-cinder-volume"
                          "openstack-cinder-backup"
                          "openstack-manila-share")

echo "Stopping systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Stopping the $service in controller $i"
            if ${!SSH_CMD} sudo systemctl is-active $service; then
                ${!SSH_CMD} sudo systemctl stop $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive >/dev/null; then
                echo "ERROR: Service $service still running on controller $i"
            else
                echo "OK: Service $service is not running on controller $i"
            fi
        fi
    done
done

echo "Stopping pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                echo "Stopping $resource"
                ${!SSH_CMD} sudo pcs resource disable $resource
            else
                echo "Service $resource not present"
            fi
        done
        break
    fi
done

echo "Checking pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                if ! ${!SSH_CMD} sudo pcs resource status $resource | grep Started; then
                    echo "OK: Service $resource is stopped"
                else
                    echo "ERROR: Service $resource is started"
                fi
            fi
        done
        break
    fi
done

for i in "${!SOURCE_GALERA_MEMBERS[@]}"; do
  echo "Checking for the database node $i WSREP status Synced"
  oc rsh mariadb-copy-data mysql \
    -h "${SOURCE_GALERA_MEMBERS[$i]}" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" \
    -e "show global status like 'wsrep_local_state_comment'" | \
    grep -qE "\bSynced\b"
done

$ oc rsh mariadb-copy-data mysql -h "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" -e "SHOW databases;"

. ~/.source_cloud_exported_variables
test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"  || [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = " " ] && echo "OK" || echo "CHECK FAILED"

$ oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
    mysql -rsh "$PODIFIED_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
$ oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
    mysql -rsh "$PODIFIED_CELL1_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'

$ oc rsh mariadb-copy-data << EOF
  mysql -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \
  -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \
  while read dbname; do
    echo "Dumping \${dbname}";
    mysqldump -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \
      --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \
      "\${dbname}" > /backup/"\${dbname}".sql;
   done
EOF

$ oc rsh mariadb-copy-data << EOF
  # db schemas to rename on import
  declare -A db_name_map
  db_name_map['nova']='nova_cell1'
  db_name_map['ovs_neutron']='neutron'
  db_name_map['ironic-inspector']='ironic_inspector'

  # db servers to import into
  declare -A db_server_map
  db_server_map['default']=${PODIFIED_MARIADB_IP}
  db_server_map['nova_cell1']=${PODIFIED_CELL1_MARIADB_IP}

  # db server root password map
  declare -A db_server_password_map
  db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD}
  db_server_password_map['nova_cell1']=${PODIFIED_DB_ROOT_PASSWORD}

  cd /backup
  for db_file in \$(ls *.sql); do
    db_name=\$(echo \${db_file} | awk -F'.' '{ print \$1; }')
    if [[ -v "db_name_map[\${db_name}]" ]]; then
      echo "renaming \${db_name} to \${db_name_map[\${db_name}]}"
      db_name=\${db_name_map[\${db_name}]}
    fi
    db_server=\${db_server_map["default"]}
    if [[ -v "db_server_map[\${db_name}]" ]]; then
      db_server=\${db_server_map[\${db_name}]}
    fi
    db_password=\${db_server_password_map['default']}
    if [[ -v "db_server_password_map[\${db_name}]" ]]; then
      db_password=\${db_server_password_map[\${db_name}]}
    fi
    echo "creating \${db_name} in \${db_server}"
    mysql -h"\${db_server}" -uroot "-p\${db_password}" -e \
      "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \
      CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};"
    echo "importing \${db_name} into \${db_server}"
    mysql -h "\${db_server}" -uroot "-p\${db_password}" "\${db_name}" < "\${db_file}"
  done

  mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \
    "update nova_api.cell_mappings set name='cell1' where name='default';"
  mysql -h "\${db_server_map['nova_cell1']}" -uroot -p"\${db_server_password_map['nova_cell1']}" -e \
    "delete from nova_cell1.services where host not like '%nova-cell1-%' and services.binary != 'nova-compute';"
EOF

. ~/.source_cloud_exported_variables

# use 'oc exec' and 'mysql -rs' to maintain formatting
dbs=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo $dbs | grep -Eq '\bkeystone\b' && echo "OK" || echo "CHECK FAILED"

# ensure neutron db is renamed from ovs_neutron
echo $dbs | grep -Eq '\bneutron\b'
echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED"

# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1
c1dbs=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo $c1dbs | grep -Eq '\bnova_cell1\b' && echo "OK" || echo "CHECK FAILED"

# ensure default cell renamed to cell1, and the cell UUIDs retained intact
novadb_mapped_cells=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \
  nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;')
uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}'
left_behind=$(comm -23 \
  <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \
  <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+"))
changed=$(comm -13 \
  <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \
  <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+"))
test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 && echo "OK" || echo "CHECK FAILED"
default=$(grep -E ' default$' <<<$left_behind)
test $(grep -Ec ' \S+$' <<<$changed) -eq 1 && echo "OK" || echo "CHECK FAILED"
grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed && echo "OK" || echo "CHECK FAILED"

# ensure the registered Compute service name has not changed
novadb_svc_records=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \
  nova_cell1 -e "select host from services where services.binary='nova-compute' order by host asc;")
diff -Z <(echo $novadb_svc_records) <(echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES) && echo "OK" || echo "CHECK FAILED"

$ oc delete pod mariadb-copy-data
$ oc delete pvc mariadb-data

$ oc wait --for=condition=Ready pod/ovn-copy-data --timeout=30s

$ $CONTROLLER1_SSH sudo iptables -I INPUT -s {PODIFIED_INTERNALAPI_NETWORK} -p tcp -m tcp --dport 6641 -m conntrack --ctstate NEW -j ACCEPT
$ $CONTROLLER1_SSH sudo iptables -I INPUT -s {PODIFIED_INTERNALAPI_NETWORK} -p tcp -m tcp --dport 6642 -m conntrack --ctstate NEW -j ACCEPT

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnDBCluster:
        ovndbcluster-nb:
          dbType: NB
          storageRequest: 10G
          networkAttachment: internalapi
        ovndbcluster-sb:
          dbType: SB
          storageRequest: 10G
          networkAttachment: internalapi
      ovnNorthd:
        replicas: 0
      ovnController:
        networkAttachment: tenant
        nodeSelector:
'

$ oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-nb
$ oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-sb

PODIFIED_OVSDB_NB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-nb-0" -ojsonpath='{.items[0].spec.clusterIP}')
PODIFIED_OVSDB_SB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-sb-0" -ojsonpath='{.items[0].spec.clusterIP}')

$ oc exec -it ovsdbserver-nb-0 -- ovn-nbctl show
$ oc exec -it ovsdbserver-sb-0 -- ovn-sbctl list Chassis

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnNorthd:
        replicas: 1
'

$ oc patch openstackcontrolplane openstack --type=json -p="[{'op': 'remove', 'path': '/spec/ovn/template/ovnController/nodeSelector'}]"

$ oc delete --ignore-not-found=true pod ovn-copy-data
$ oc delete --ignore-not-found=true pvc ovn-data

ServicesToStop=("tripleo_ovn_cluster_north_db_server.service"
                "tripleo_ovn_cluster_south_db_server.service")

echo "Stopping systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Stopping the $service in controller $i"
            if ${!SSH_CMD} sudo systemctl is-active $service; then
                ${!SSH_CMD} sudo systemctl stop $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive >/dev/null; then
                echo "ERROR: Service $service still running on controller $i"
            else
                echo "OK: Service $service is not running on controller $i"
            fi
        fi
    done
done

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  keystone:
    enabled: true
    apiOverride:
      route: {}
    template:
      override:
        service:
          internal:
            metadata:
              annotations:
                metallb.universe.tf/address-pool: internalapi
                metallb.universe.tf/allow-shared-ip: internalapi
                metallb.universe.tf/loadBalancerIPs: 172.17.0.80
            spec:
              type: LoadBalancer
      databaseInstance: openstack
      secret: osp-secret
'

$ alias openstack="oc exec -t openstackclient -- openstack"

$ openstack endpoint list | grep keystone | awk '/admin/{ print $2; }' | xargs ${BASH_ALIASES[openstack]} endpoint delete || true

for service in aodh heat heat-cfn barbican cinderv3 glance gnocchi manila manilav2 neutron nova placement swift ironic-inspector ironic; do
  openstack service list | awk "/ $service /{ print \$2; }" | xargs -r ${BASH_ALIASES[openstack]} service delete || true
done

$ openstack endpoint list | grep keystone

$ oc set data secret/osp-secret "BarbicanSimpleCryptoKEK=$($CONTROLLER1_SSH "python3 -c \"import configparser; c = configparser.ConfigParser(); c.read('/var/lib/config-data/puppet-generated/barbican/etc/barbican/barbican.conf'); print(c['simple_crypto_plugin']['kek'])\"")"

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  barbican:
    enabled: true
    apiOverride:
      route: {}
    template:
      databaseInstance: openstack
      databaseAccount: barbican
      rabbitMqClusterName: rabbitmq
      secret: osp-secret
      simpleCryptoBackendSecret: osp-secret
      serviceAccount: barbican
      serviceUser: barbican
      passwordSelectors:
        service: BarbicanPassword
        simplecryptokek: BarbicanSimpleCryptoKEK
      barbicanAPI:
        replicas: 1
        override:
          service:
            internal:
              metadata:
                annotations:
                  metallb.universe.tf/address-pool: internalapi
                  metallb.universe.tf/allow-shared-ip: internalapi
                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80
              spec:
                type: LoadBalancer
      barbicanWorker:
        replicas: 1
      barbicanKeystoneListener:
        replicas: 1
'

$ oc apply -f - <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: swift-conf
  namespace: openstack
type: Opaque
data:
  swift.conf: $($CONTROLLER1_SSH sudo cat /var/lib/config-data/puppet-generated/swift/etc/swift/swift.conf | base64 -w0)
EOF

$ oc apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: swift-ring-files
binaryData:
  swiftrings.tar.gz: $($CONTROLLER1_SSH "cd /var/lib/config-data/puppet-generated/swift/etc/swift && tar cz *.builder *.ring.gz backups/ | base64 -w0")
  account.ring.gz: $($CONTROLLER1_SSH "base64 -w0 /var/lib/config-data/puppet-generated/swift/etc/swift/account.ring.gz")
  container.ring.gz: $($CONTROLLER1_SSH "base64 -w0 /var/lib/config-data/puppet-generated/swift/etc/swift/container.ring.gz")
  object.ring.gz: $($CONTROLLER1_SSH "base64 -w0 /var/lib/config-data/puppet-generated/swift/etc/swift/object.ring.gz")
EOF

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  swift:
    enabled: true
    template:
      memcachedInstance: memcached
      swiftRing:
        ringReplicas: 1
      swiftStorage:
        replicas: 0
        networkAttachments:
        - storage
        storageClass: local-storage (1)
        storageRequest: 10Gi
      swiftProxy:
        secret: osp-secret
        replicas: 1
        passwordSelectors:
          service: SwiftPassword
        serviceUser: swift
        override:
          service:
            internal:
              metadata:
                annotations:
                  metallb.universe.tf/address-pool: internalapi
                  metallb.universe.tf/allow-shared-ip: internalapi
                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80
              spec:
                type: LoadBalancer
        networkAttachments: (2)
        - storage
'

$ oc patch openstackcontrolplane openstack -n openstack --type=merge --patch '
spec:
  nova:
    enabled: true
    apiOverride:
      route: {}
    template:
      secret: osp-secret
      apiServiceTemplate:
        override:
          service:
            internal:
              metadata:
                annotations:
                  metallb.universe.tf/address-pool: internalapi
                  metallb.universe.tf/allow-shared-ip: internalapi
                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80
              spec:
                type: LoadBalancer
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=true
      metadataServiceTemplate:
        enabled: true # deploy single nova metadata on the top level
        override:
          service:
            metadata:
              annotations:
                metallb.universe.tf/address-pool: internalapi
                metallb.universe.tf/allow-shared-ip: internalapi
                metallb.universe.tf/loadBalancerIPs: 172.17.0.80
            spec:
              type: LoadBalancer
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=true
      schedulerServiceTemplate:
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=true
      cellTemplates:
        cell0:
          conductorServiceTemplate:
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=true
        cell1:
          metadataServiceTemplate:
            enabled: false # enable here to run it in a cell instead
            override:
                service:
                  metadata:
                    annotations:
                      metallb.universe.tf/address-pool: internalapi
                      metallb.universe.tf/allow-shared-ip: internalapi
                      metallb.universe.tf/loadBalancerIPs: 172.17.0.80
                  spec:
                    type: LoadBalancer
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=true
          conductorServiceTemplate:
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=true
'

        cell1:
          novaComputeTemplates:
            standalone:
              customServiceConfig: |
                [DEFAULT]
                host = <hostname>
                [workarounds]
                disable_compute_service_check_for_ffu=true

$ oc wait --for condition=Ready --timeout=300s Nova/nova

$ oc patch openstackcontrolplane openstack --type=merge --patch-file=<patch_name>

$ openstack volume service list

+------------------+------------------------+------+---------+-------+----------------------------+
| Binary           | Host                   | Zone | Status  | State | Updated At                 |
+------------------+------------------------+------+---------+-------+----------------------------+
| cinder-scheduler | standalone.localdomain | nova | enabled | down  | 2024-11-04T17:47:14.000000 |
| cinder-backup    | standalone.localdomain | nova | enabled | down  | 2024-11-04T17:47:14.000000 |
| cinder-volume    | hostgroup@tripleo_ceph | nova | enabled | down  | 2024-11-04T17:47:14.000000 |
+------------------+------------------------+------+---------+-------+----------------------------+

$ oc exec -t cinder-api-0 -c cinder-api -- cinder-manage service remove <service_binary> <service_host>

$ openstack volume service list

+------------------+------------------------+------+---------+-------+----------------------------+
| Binary           | Host                   | Zone | Status  | State | Updated At                 |
+------------------+------------------------+------+---------+-------+----------------------------+
| cinder-volume    | hostgroup@tripleo_ceph | nova | enabled | up    | 2023-06-28T17:00:03.000000 |
| cinder-scheduler | cinder-scheduler-0     | nova | enabled | up    | 2023-06-28T17:00:02.000000 |
| cinder-backup    | cinder-backup-0        | nova | enabled | up    | 2023-06-28T17:00:01.000000 |
+------------------+------------------------+------+---------+-------+----------------------------+

$ oc exec -it cinder-scheduler-0 -- cinder-manage db online_data_migrations

$ alias openstack="oc exec -t openstackclient -- openstack"

$ openstack endpoint list --service <endpoint>

$ openstack volume service list

$ openstack volume type list
$ openstack volume list
$ openstack volume snapshot list
$ openstack volume backup list

$ oc get horizon

PUBLIC_URL=$(oc get horizon horizon -o jsonpath='{.status.endpoint}')
curl --silent --output /dev/stderr --head --write-out "%{http_code}" "$PUBLIC_URL/dashboard/auth/login/?next=/dashboard/" -k | grep 200

[stack@rhosp17 ~]$ grep -E 'HeatPassword|HeatAuth|HeatStackDomainAdmin' ~/overcloud-deploy/overcloud/overcloud-passwords.yaml
  HeatAuthEncryptionKey: Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2
  HeatPassword: dU2N0Vr2bdelYH7eQonAwPfI3
  HeatStackDomainAdminPassword: dU2N0Vr2bdelYH7eQonAwPfI3

[stack@rhosp17 ~]$ ansible -i overcloud-deploy/overcloud/config-download/overcloud/tripleo-ansible-inventory.yaml overcloud-controller-0 -m shell -a "grep auth_encryption_key /var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf | grep -Ev '^#|^$'" -b
overcloud-controller-0 | CHANGED | rc=0 >>
auth_encryption_key=Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2

$ echo Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2 | base64
UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK

$ oc patch secret osp-secret --type='json' -p='[{"op" : "replace" ,"path" : "/data/HeatAuthEncryptionKey" ,"value" : "UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK"}]'
secret/osp-secret patched

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  heat:
    enabled: true
    apiOverride:
      route: {}
    template:
      databaseInstance: openstack
      databaseAccount: heat
      secret: osp-secret
      memcachedInstance: memcached
      passwordSelectors:
        authEncryptionKey: HeatAuthEncryptionKey
        service: HeatPassword
        stackDomainAdminPassword: HeatStackDomainAdminPassword
'

$ oc get Heat,HeatAPI,HeatEngine,HeatCFNAPI
NAME                           STATUS   MESSAGE
heat.heat.openstack.org/heat   True     Setup complete

NAME                                  STATUS   MESSAGE
heatapi.heat.openstack.org/heat-api   True     Setup complete

NAME                                        STATUS   MESSAGE
heatengine.heat.openstack.org/heat-engine   True     Setup complete

NAME                                        STATUS   MESSAGE
heatcfnapi.heat.openstack.org/heat-cfnapi   True     Setup complete

$ oc exec -it openstackclient -- openstack service list -c Name -c Type
+------------+----------------+
| Name       | Type           |
+------------+----------------+
| heat       | orchestration  |
| glance     | image          |
| heat-cfn   | cloudformation |
| ceilometer | Ceilometer     |
| keystone   | identity       |
| placement  | placement      |
| cinderv3   | volumev3       |
| nova       | compute        |
| neutron    | network        |
+------------+----------------+

$ oc exec -it openstackclient -- openstack endpoint list --service=heat -f yaml
- Enabled: true
  ID: 1da7df5b25b94d1cae85e3ad736b25a5
  Interface: public
  Region: regionOne
  Service Name: heat
  Service Type: orchestration
  URL: http://heat-api-public-openstack-operators.apps.okd.bne-shift.net/v1/%(tenant_id)s
- Enabled: true
  ID: 414dd03d8e9d462988113ea0e3a330b0
  Interface: internal
  Region: regionOne
  Service Name: heat
  Service Type: orchestration
  URL: http://heat-api-internal.openstack-operators.svc:8004/v1/%(tenant_id)s

$ oc exec -it openstackclient -- openstack orchestration service list -f yaml
- Binary: heat-engine
  Engine ID: b16ad899-815a-4b0c-9f2e-e6d9c74aa200
  Host: heat-engine-6d47856868-p7pzz
  Hostname: heat-engine-6d47856868-p7pzz
  Status: up
  Topic: engine
  Updated At: '2023-10-11T21:48:01.000000'
- Binary: heat-engine
  Engine ID: 887ed392-0799-4310-b95c-ac2d3e6f965f
  Host: heat-engine-6d47856868-p7pzz
  Hostname: heat-engine-6d47856868-p7pzz
  Status: up
  Topic: engine
  Updated At: '2023-10-11T21:48:00.000000'
- Binary: heat-engine
  Engine ID: 26ed9668-b3f2-48aa-92e8-2862252485ea
  Host: heat-engine-6d47856868-p7pzz
  Hostname: heat-engine-6d47856868-p7pzz
  Status: up
  Topic: engine
  Updated At: '2023-10-11T21:48:00.000000'
- Binary: heat-engine
  Engine ID: 1011943b-9fea-4f53-b543-d841297245fd
  Host: heat-engine-6d47856868-p7pzz
  Hostname: heat-engine-6d47856868-p7pzz
  Status: up
  Topic: engine
  Updated At: '2023-10-11T21:48:01.000000'

$ openstack stack list -f yaml
- Creation Time: '2023-10-11T22:03:20Z'
  ID: 20f95925-7443-49cb-9561-a1ab736749ba
  Project: 4eacd0d1cab04427bc315805c28e66c9
  Stack Name: test-networks
  Stack Status: CREATE_COMPLETE
  Updated Time: null

$ oc create -f - <<EOF
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: cluster-observability-operator
  namespace: openshift-operators
spec:
  channel: development
  installPlanApproval: Automatic
  name: cluster-observability-operator
  source: redhat-operators
  sourceNamespace: openshift-marketplace
EOF

$ oc wait --for jsonpath="{.status.phase}"=Succeeded csv --namespace=openshift-operators -l operators.coreos.com/cluster-observability-operator.openshift-operators

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  telemetry:
    enabled: true
    template:
      ceilometer:
        passwordSelector:
          ceilometerService: CeilometerPassword
        enabled: true
        secret: osp-secret
        serviceUser: ceilometer
'

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  telemetry:
    template:
      metricStorage:
        enabled: true
        monitoringStack:
          alertingEnabled: true
          scrapeInterval: 30s
          storage:
            strategy: persistent
            retention: 24h
            persistent:
              pvcStorageRequest: 20G
'

$ oc get pods -l alertmanager=metric-storage -n openstack
NAME                            READY   STATUS    RESTARTS   AGE
alertmanager-metric-storage-0   2/2     Running   0          46s
alertmanager-metric-storage-1   2/2     Running   0          46s

$ oc get pods -l prometheus=metric-storage -n openstack
NAME                          READY   STATUS    RESTARTS   AGE
prometheus-metric-storage-0   3/3     Running   0          46s

CEILOMETETR_POD=`oc get pods -l service=ceilometer -n openstack | tail -n 1 | cut -f 1 -d' '`
oc exec -t $CEILOMETETR_POD -c ceilometer-central-agent -- cat /etc/ceilometer/ceilometer.conf

$ oc get secret ceilometer-config-data -o jsonpath="{.data['polling\.yaml\.j2']}"  | base64 -d

$ oc patch openstackcontrolplane controlplane --type=merge --patch '
spec:
  telemetry:
    template:
      ceilometer:
          defaultConfigOverwrite:
            polling.yaml.j2: |
              ---
              sources:
                - name: pollsters
                  interval: 100
                  meters:
                    - volume.*
                    - image.size
          enabled: true
          secret: osp-secret
'

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  telemetry:
    template:
      logging:
      enabled: false
      ipaddr: 172.17.0.80
      port: 10514
      cloNamespace: openshift-logging
'

$ oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  telemetry:
    enabled: true
    template:
      autoscaling:
        enabled: true
        aodh:
          passwordSelector:
            aodhService: AodhPassword
          databaseAccount: aodh
          databaseInstance: openstack
          secret: osp-secret
          serviceUser: aodh
        heatInstance: heat
'

$ AODH_POD=`oc get pods -l service=aodh -n openstack | tail -n 1 | cut -f 1 -d' '`
$ oc exec -t $AODH_POD -c aodh-api -- cat /etc/aodh/aodh.conf

$ openstack endpoint list | grep aodh
| d05d120153cd4f9b8310ac396b572926 | regionOne | aodh  | alarming  | True    | internal  | http://aodh-internal.openstack.svc:8042  |
| d6daee0183494d7a9a5faee681c79046 | regionOne | aodh  | alarming  | True    | public    | http://aodh-public.openstack.svc:8042    |

$ openstack alarm create --name high_cpu_alarm \
--type prometheus \
--query "(rate(ceilometer_cpu{resource_name=~'cirros'})) * 100" \
--alarm-action 'log://' \
--granularity 15 \
--evaluation-periods 3 \
--comparison-operator gt \
--threshold 7000000000

ssh_cmd=ssh -F ssh.config standalone
container_engine=podman
connection=ssh
remote_config_path=/tmp/tripleo

$ chown ospng:ospng /etc/os-diff/config.yaml

# service name and file location
services:
  # Service name
  keystone:
    # Bool to enable/disable a service (not implemented yet)
    enable: true
    # Pod name, in both OCP and podman context.
    # It could be strict match or will only just grep the podman_name
    # and work with all the pods which matched with pod_name.
    # To enable/disable use strict_pod_name_match: true/false
    podman_name: keystone
    pod_name: keystone
    container_name: keystone-api
    # pod options
    # strict match for getting pod id in TripleO and podman context
    strict_pod_name_match: false
    # Path of the config files you want to analyze.
    # It could be whatever path you want:
    # /etc/<service_name> or /etc or /usr/share/<something> or even /
    # @TODO: need to implement loop over path to support multiple paths such as:
    # - /etc
    # - /usr/share
    path:
      - /etc/
      - /etc/keystone
      - /etc/keystone/keystone.conf
      - /etc/keystone/logging.conf

services:
  ovs_external_ids:
    hosts: (1)
      - standalone
    service_command: "ovs-vsctl list Open_vSwitch . | grep external_ids | awk -F ': ' '{ print $2; }'" (2)
    cat_output: true (3)
    path:
      - ovs_external_ids.json
    config_mapping: (4)
      ovn-bridge-mappings: edpm_ovn_bridge_mappings (5)
      ovn-bridge: edpm_ovn_bridge
      ovn-encap-type: edpm_ovn_encap_type
      ovn-monitor-all: ovn_monitor_all
      ovn-remote-probe-interval: edpm_ovn_remote_probe_interval
      ovn-ofctrl-wait-before-clear: edpm_ovn_ofctrl_wait_before_clear

# will only update the /etc/os-diff/config.yaml
os-diff pull --update-only

# will update the /etc/os-diff/config.yaml and pull configuration
os-diff pull --update

# will update the /etc/os-diff/config.yaml and pull configuration
os-diff pull

/tmp/tripleo/

ServicesToStart=("tripleo_horizon.service"
                 "tripleo_keystone.service"
                 "tripleo_barbican_api.service"
                 "tripleo_barbican_worker.service"
                 "tripleo_barbican_keystone_listener.service"
                 "tripleo_cinder_api.service"
                 "tripleo_cinder_api_cron.service"
                 "tripleo_cinder_scheduler.service"
                 "tripleo_cinder_volume.service"
                 "tripleo_cinder_backup.service"
                 "tripleo_glance_api.service"
                 "tripleo_manila_api.service"
                 "tripleo_manila_api_cron.service"
                 "tripleo_manila_scheduler.service"
                 "tripleo_neutron_api.service"
                 "tripleo_placement_api.service"
                 "tripleo_nova_api_cron.service"
                 "tripleo_nova_api.service"
                 "tripleo_nova_conductor.service"
                 "tripleo_nova_metadata.service"
                 "tripleo_nova_scheduler.service"
                 "tripleo_nova_vnc_proxy.service"
                 "tripleo_aodh_api.service"
                 "tripleo_aodh_api_cron.service"
                 "tripleo_aodh_evaluator.service"
                 "tripleo_aodh_listener.service"
                 "tripleo_aodh_notifier.service"
                 "tripleo_ceilometer_agent_central.service"
                 "tripleo_ceilometer_agent_compute.service"
                 "tripleo_ceilometer_agent_ipmi.service"
                 "tripleo_ceilometer_agent_notification.service"
                 "tripleo_ovn_cluster_north_db_server.service"
                 "tripleo_ovn_cluster_south_db_server.service"
                 "tripleo_ovn_cluster_northd.service")

PacemakerResourcesToStart=("galera-bundle"
                           "haproxy-bundle"
                           "rabbitmq-bundle"
                           "openstack-cinder-volume"
                           "openstack-cinder-backup"
                           "openstack-manila-share")

echo "Starting systemd OpenStack services"
for service in ${ServicesToStart[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ${!SSH_CMD} sudo systemctl is-enabled $service &> /dev/null; then
                echo "Starting the $service in controller $i"
                ${!SSH_CMD} sudo systemctl start $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStart[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ${!SSH_CMD} sudo systemctl is-enabled $service &> /dev/null; then
                if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=active >/dev/null; then
                    echo "ERROR: Service $service is not running on controller $i"
                else
                    echo "OK: Service $service is running in controller $i"
                fi
            fi
        fi
    done
done

echo "Starting pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStart[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                echo "Starting $resource"
                ${!SSH_CMD} sudo pcs resource enable $resource
            else
                echo "Service $resource not present"
            fi
        done
        break
    fi
done

echo "Checking pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                if ${!SSH_CMD} sudo pcs resource status $resource | grep Started >/dev/null; then
                    echo "OK: Service $resource is started"
                else
                    echo "ERROR: Service $resource is stopped"
                fi
            fi
        done
        break
    fi
done

$ sudo pcs constraint order start ceph-nfs then openstack-manila-share kind=Optional id=order-ceph-nfs-openstack-manila-share-Optional
$ sudo pcs constraint colocation add openstack-manila-share with ceph-nfs score=INFINITY id=colocation-openstack-manila-share-ceph-nfs-INFINITY

$ oc delete --ignore-not-found=true --wait=false openstackcontrolplane/openstack
$ oc patch openstackcontrolplane openstack --type=merge --patch '
metadata:
  finalizers: []
' || true

while oc get pod | grep rabbitmq-server-0; do
    sleep 2
done
while oc get pod | grep openstack-galera-0; do
    sleep 2
done

$ oc delete --ignore-not-found=true --wait=false pod mariadb-copy-data
$ oc delete --ignore-not-found=true --wait=false pvc mariadb-data
$ oc delete --ignore-not-found=true --wait=false pod ovn-copy-data
$ oc delete --ignore-not-found=true secret osp-secret

$ oc apply -f - <<EOF
apiVersion: v1
kind: Secret
metadata:
    name: dataplane-adoption-secret
    namespace: openstack
data:
    ssh-privatekey: |
$(cat ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa | base64 | sed 's/^/        /')
EOF

$ cd "$(mktemp -d)"
ssh-keygen -f ./id -t ecdsa-sha2-nistp521 -N ''
oc get secret nova-migration-ssh-key || oc create secret generic nova-migration-ssh-key \
  -n openstack \
  --from-file=ssh-privatekey=id \
  --from-file=ssh-publickey=id.pub \
  --type kubernetes.io/ssh-auth
rm -f id*
cd -

$ oc apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: nova-extra-config
  namespace: openstack
data:
  19-nova-compute-cell1-workarounds.conf: |
    [workarounds]
    disable_compute_service_check_for_ffu=true
EOF

  tlsCerts:
    contents:
      - dnsnames
      - ips
    networks:
      - ctlplane
    issuer: osp-rootca-issuer-internal
  caCerts: combined-ca-bundle
  edpmServiceType: nova

$ oc apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: nova-extra-config
  namespace: openstack
data:
  19-nova-compute-cell1-workarounds.conf: |
    [workarounds]
    disable_compute_service_check_for_ffu=true
  03-ceph-nova.conf: |
    [libvirt]
    images_type=rbd
    images_rbd_pool=vms
    images_rbd_ceph_conf=/etc/ceph/ceph.conf
    images_rbd_glance_store_name=default_backend
    images_rbd_glance_copy_poll_interval=15
    images_rbd_glance_copy_timeout=600
    rbd_user=openstack
    rbd_secret_uuid=$CEPH_FSID
EOF

$ oc apply -f - <<EOF
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneNodeSet
metadata:
  name: openstack
spec:
  tlsEnabled: false (1)
  networkAttachments:
      - ctlplane
  preProvisioned: true
  services:
    - bootstrap
    - download-cache
    - configure-network
    - validate-network
    - install-os
    - configure-os
    - ssh-known-hosts
    - run-os
    - reboot-os
    - install-certs
    - libvirt
    - nova
    - ovn
    - neutron-metadata
    - telemetry
  env:
    - name: ANSIBLE_CALLBACKS_ENABLED
      value: "profile_tasks"
    - name: ANSIBLE_FORCE_COLOR
      value: "True"
  nodes:
    standalone:
      hostName: standalone (2)
      ansible:
        ansibleHost: ${computes[standalone.localdomain]}
      networks:
      - defaultRoute: true
        fixedIP: ${computes[standalone.localdomain]}
        name: ctlplane
        subnetName: subnet1
      - name: internalapi
        subnetName: subnet1
      - name: storage
        subnetName: subnet1
      - name: tenant
        subnetName: subnet1
  nodeTemplate:
    ansibleSSHPrivateKeySecret: dataplane-adoption-secret
    ansible:
      ansibleUser: root
      ansibleVars:
        edpm_bootstrap_release_version_package: []
        # edpm_network_config
        # Default nic config template for a EDPM node
        # These vars are edpm_network_config role vars
        edpm_network_config_template: |
           ---
           {% set mtu_list = [ctlplane_mtu] %}
           {% for network in nodeset_networks %}
           {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
           {%- endfor %}
           {% set min_viable_mtu = mtu_list | max %}
           network_config:
           - type: ovs_bridge
             name: {{ neutron_physical_bridge_name }}
             mtu: {{ min_viable_mtu }}
             use_dhcp: false
             dns_servers: {{ ctlplane_dns_nameservers }}
             domain: {{ dns_search_domains }}
             addresses:
             - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
             routes: {{ ctlplane_host_routes }}
             members:
             - type: interface
               name: nic1
               mtu: {{ min_viable_mtu }}
               # force the MAC address of the bridge to this interface
               primary: true
           {% for network in nodeset_networks %}
             - type: vlan
               mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
               vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
               addresses:
               - ip_netmask:
                   {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
               routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
           {% endfor %}

        edpm_network_config_hide_sensitive_logs: false
        #
        # These vars are for the network config templates themselves and are
        # considered EDPM network defaults.
        neutron_physical_bridge_name: br-ctlplane
        neutron_public_interface_name: eth0

        # edpm_nodes_validation
        edpm_nodes_validation_validate_controllers_icmp: false
        edpm_nodes_validation_validate_gateway_icmp: false

        # edpm ovn-controller configuration
        edpm_ovn_bridge_mappings: <bridge_mappings> (3)
        edpm_ovn_bridge: br-int
        edpm_ovn_encap_type: geneve
        ovn_monitor_all: true
        edpm_ovn_remote_probe_interval: 60000
        edpm_ovn_ofctrl_wait_before_clear: 8000

        timesync_ntp_servers:
        - hostname: pool.ntp.org

        edpm_bootstrap_command: |
          # This is a hack to deploy RDO Delorean repos to RHEL as if it were Centos 9 Stream
          set -euxo pipefail
          curl -sL https://github.com/openstack-k8s-operators/repo-setup/archive/refs/heads/main.tar.gz | tar -xz
          python3 -m venv ./venv
          PBR_VERSION=0.0.0 ./venv/bin/pip install ./repo-setup-main
          # This is required for FIPS enabled until trunk.rdoproject.org
          # is not being served from a centos7 host, tracked by
          # https://issues.redhat.com/browse/RHOSZUUL-1517
          dnf -y install crypto-policies
          update-crypto-policies --set FIPS:NO-ENFORCE-EMS
          # FIXME: perform dnf upgrade for other packages in EDPM ansible
          # here we only ensuring that decontainerized libvirt can start
          ./venv/bin/repo-setup current-podified -b antelope -d centos9 --stream
          dnf -y upgrade openstack-selinux
          rm -f /run/virtlogd.pid
          rm -rf repo-setup-main

        gather_facts: false
        # edpm firewall, change the allowed CIDR if needed
        edpm_sshd_configure_firewall: true
        edpm_sshd_allowed_ranges: ['192.168.122.0/24']

        # Do not attempt OVS major upgrades here
        edpm_ovs_packages:
        - openvswitch3.3
EOF

ovs-vsctl list Open .
...
external_ids        : {hostname=standalone.localdomain, ovn-bridge=br-int, ovn-bridge-mappings=<bridge_mappings>, ovn-chassis-mac-mappings="datacentre:1e:0a:bb:e6:7c:ad", ovn-encap-ip="172.19.0.100", ovn-encap-tos="0", ovn-encap-type=geneve, ovn-match-northd-version=False, ovn-monitor-all=True, ovn-ofctrl-wait-before-clear="8000", ovn-openflow-probe-interval="60", ovn-remote="tcp:ovsdbserver-sb.openstack.svc:6642", ovn-remote-probe-interval="60000", rundir="/var/run/openvswitch", system-id="2eec68e6-aa21-4c95-a868-31aeafc11736"}
...

$ oc patch osdpns/openstack --type=merge --patch "
spec:
  services:
    - bootstrap
    - download-cache
    - configure-network
    - validate-network
    - install-os
    - configure-os
    - ssh-known-hosts
    - run-os
    - reboot-os
    - ceph-client
    - install-certs
    - ovn
    - neutron-metadata
    - libvirt
    - nova
    - telemetry
  nodeTemplate:
    extraMounts:
    - extraVolType: Ceph
      volumes:
      - name: ceph
        secret:
          secretName: ceph-conf-files
      mounts:
      - name: ceph
        mountPath: "/etc/ceph"
        readOnly: true
"

$ oc patch openstackdataplanenodeset openstack --type='json' --patch='[
  {
    "op": "add",
    "path": "/spec/services/-",
    "value": "neutron-sriov"
  }, {
    "op": "add",
    "path": "/spec/nodeTemplate/ansible/ansibleVars/edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings",
    "value": "dummy_sriov_net:dummy-dev"
  }, {
    "op": "add",
    "path": "/spec/nodeTemplate/ansible/ansibleVars/edpm_neutron_sriov_agent_SRIOV_NIC_resource_provider_bandwidths",
    "value": "dummy-dev:40000000:40000000"
  }, {
    "op": "add",
    "path": "/spec/nodeTemplate/ansible/ansibleVars/edpm_neutron_sriov_agent_SRIOV_NIC_resource_provider_hypervisors",
    "value": "dummy-dev:standalone.localdomain"
  }
]'

$ oc patch openstackdataplanenodeset openstack --type='json' --patch='[
  {
    "op": "add",
    "path": "/spec/services/-",
    "value": "neutron-dhcp"
  }]'

$ oc apply -f - <<EOF
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneDeployment
metadata:
  name: openstack
spec:
  nodeSets:
  - openstack
EOF

$ watch oc get pod -l app=openstackansibleee

$ oc logs -l app=openstackansibleee -f --max-log-requests 20

$ oc wait --for condition=Ready osdpns/openstack --timeout=30m

$ oc exec openstackclient -- openstack network agent list
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+
| ID                                   | Agent Type                   | Host                   | Availability Zone | Alive | State | Binary                     |
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+
| 174fc099-5cc9-4348-b8fc-59ed44fcfb0e | DHCP agent                   | standalone.localdomain | nova              | :-)   | UP    | neutron-dhcp-agent         |
| 10482583-2130-5b0d-958f-3430da21b929 | OVN Metadata agent           | standalone.localdomain |                   | :-)   | UP    | neutron-ovn-metadata-agent |
| a4f1b584-16f1-4937-b2b0-28102a3f6eaa | OVN Controller agent         | standalone.localdomain |                   | :-)   | UP    | ovn-controller             |
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+

$ oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot -p$PODIFIED_DB_ROOT_PASSWORD \
    -e "select a.version from nova_cell1.services a join nova_cell1.services b where a.version!=b.version and a.binary='nova-compute';"

$ oc patch openstackcontrolplane openstack -n openstack --type=merge --patch '
spec:
  nova:
    template:
      cellTemplates:
        cell0:
          conductorServiceTemplate:
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=false
        cell1:
          metadataServiceTemplate:
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=false
          conductorServiceTemplate:
            customServiceConfig: |
              [workarounds]
              disable_compute_service_check_for_ffu=false
      apiServiceTemplate:
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=false
      metadataServiceTemplate:
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=false
      schedulerServiceTemplate:
        customServiceConfig: |
          [workarounds]
          disable_compute_service_check_for_ffu=false
'

$ oc wait --for condition=Ready --timeout=300s Nova/nova

$ oc apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: nova-extra-config
  namespace: openstack
data:
  20-nova-compute-cell1-workarounds.conf: |
    [workarounds]
    disable_compute_service_check_for_ffu=false
---
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneDeployment
metadata:
  name: openstack-nova-compute-ffu
  namespace: openstack
spec:
  nodeSets:
    - openstack
  servicesOverride:
    - nova
EOF

$ oc wait --for condition=Ready openstackdataplanedeployment/openstack-nova-compute-ffu --timeout=5m

$ oc exec -it nova-cell0-conductor-0 -- nova-manage db online_data_migrations
$ oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations

$ oc rsh nova-cell0-conductor-0 nova-manage cell_v2 discover_hosts --verbose

${BASH_ALIASES[openstack]} server --os-compute-api-version 2.48 show --diagnostics test 2>&1 || echo FAIL

${BASH_ALIASES[openstack]} server list -c Name -c Status -f value | grep -qF "test ACTIVE" && ${BASH_ALIASES[openstack]} server stop test || echo PASS
${BASH_ALIASES[openstack]} server list -c Name -c Status -f value | grep -qF "test SHUTOFF" || echo FAIL
${BASH_ALIASES[openstack]} server --os-compute-api-version 2.48 show --diagnostics test 2>&1 || echo PASS

${BASH_ALIASES[openstack]} server list -c Name -c Status -f value | grep -qF "test SHUTOFF" && ${BASH_ALIASES[openstack]} server start test || echo PASS
${BASH_ALIASES[openstack]} server list -c Name -c Status -f value | grep -qF "test ACTIVE" && \
  ${BASH_ALIASES[openstack]} server --os-compute-api-version 2.48 show --diagnostics test --fit-width -f json | jq -r '.state' | grep running || echo FAIL

$ oc apply -f - <<EOF
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneNodeSet
metadata:
  name: openstack-networker
spec:
  tlsEnabled: false (1)
  networkAttachments:
      - ctlplane
  preProvisioned: true
  services:
    - bootstrap
    - download-cache
    - configure-network
    - validate-network
    - install-os
    - configure-os
    - ssh-known-hosts
    - run-os
    - install-certs
    - ovn
  env:
    - name: ANSIBLE_CALLBACKS_ENABLED
      value: "profile_tasks"
    - name: ANSIBLE_FORCE_COLOR
      value: "True"
  nodes:
    standalone:
      hostName: standalone
      ansible:
        ansibleHost: ${networkers[standalone.localdomain]}
      networks:
      - defaultRoute: true
        fixedIP: ${networkers[standalone.localdomain]}
        name: ctlplane
        subnetName: subnet1
      - name: internalapi
        subnetName: subnet1
      - name: storage
        subnetName: subnet1
      - name: tenant
        subnetName: subnet1
  nodeTemplate:
    ansibleSSHPrivateKeySecret: dataplane-adoption-secret
    ansible:
      ansibleUser: root
      ansibleVars:
        edpm_bootstrap_release_version_package: []
        # edpm_network_config
        # Default nic config template for a EDPM node
        # These vars are edpm_network_config role vars
        edpm_network_config_template: |
           ---
           {% set mtu_list = [ctlplane_mtu] %}
           {% for network in nodeset_networks %}
           {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
           {%- endfor %}
           {% set min_viable_mtu = mtu_list | max %}
           network_config:
           - type: ovs_bridge
             name: {{ neutron_physical_bridge_name }}
             mtu: {{ min_viable_mtu }}
             use_dhcp: false
             dns_servers: {{ ctlplane_dns_nameservers }}
             domain: {{ dns_search_domains }}
             addresses:
             - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
             routes: {{ ctlplane_host_routes }}
             members:
             - type: interface
               name: nic1
               mtu: {{ min_viable_mtu }}
               # force the MAC address of the bridge to this interface
               primary: true
           {% for network in nodeset_networks %}
             - type: vlan
               mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
               vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
               addresses:
               - ip_netmask:
                   {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
               routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
           {% endfor %}

        edpm_network_config_hide_sensitive_logs: false
        #
        # These vars are for the network config templates themselves and are
        # considered EDPM network defaults.
        neutron_physical_bridge_name: br-ctlplane
        neutron_public_interface_name: eth0

        # edpm_nodes_validation
        edpm_nodes_validation_validate_controllers_icmp: false
        edpm_nodes_validation_validate_gateway_icmp: false

        # edpm ovn-controller configuration
        edpm_ovn_bridge_mappings: <bridge_mappings> (2)
        edpm_ovn_bridge: br-int
        edpm_ovn_encap_type: geneve
        ovn_monitor_all: true
        edpm_ovn_remote_probe_interval: 60000
        edpm_ovn_ofctrl_wait_before_clear: 8000

        # serve as a OVN gateway
        edpm_enable_chassis_gw: true (3)

        timesync_ntp_servers:
        - hostname: pool.ntp.org

        edpm_bootstrap_command: |
          # This is a hack to deploy RDO Delorean repos to RHEL as if it were Centos 9 Stream
          set -euxo pipefail
          curl -sL https://github.com/openstack-k8s-operators/repo-setup/archive/refs/heads/main.tar.gz | tar -xz
          python3 -m venv ./venv
          PBR_VERSION=0.0.0 ./venv/bin/pip install ./repo-setup-main
          # This is required for FIPS enabled until trunk.rdoproject.org
          # is not being served from a centos7 host, tracked by
          # https://issues.redhat.com/browse/RHOSZUUL-1517
          dnf -y install crypto-policies
          update-crypto-policies --set FIPS:NO-ENFORCE-EMS
          ./venv/bin/repo-setup current-podified -b antelope -d centos9 --stream
          rm -rf repo-setup-main

        gather_facts: false
        enable_debug: false
        # edpm firewall, change the allowed CIDR if needed
        edpm_sshd_configure_firewall: true
        edpm_sshd_allowed_ranges: ['192.168.122.0/24']
        # SELinux module
        edpm_selinux_mode: enforcing

        # Do not attempt OVS major upgrades here
        edpm_ovs_packages:
        - openvswitch3.3
EOF

ovs-vsctl list Open .
...
external_ids        : {hostname=standalone.localdomain, ovn-bridge=br-int, ovn-bridge-mappings=<bridge_mappings>, ovn-chassis-mac-mappings="datacentre:1e:0a:bb:e6:7c:ad", ovn-cms-options=enable-chassis-as-gw, ovn-encap-ip="172.19.0.100", ovn-encap-tos="0", ovn-encap-type=geneve, ovn-match-northd-version=False, ovn-monitor-all=True, ovn-ofctrl-wait-before-clear="8000", ovn-openflow-probe-interval="60", ovn-remote="tcp:ovsdbserver-sb.openstack.svc:6642", ovn-remote-probe-interval="60000", rundir="/var/run/openvswitch", system-id="2eec68e6-aa21-4c95-a868-31aeafc11736"}
...

$ oc patch openstackdataplanenodeset <networker_CR_name> --type='json' --patch='[
  {
    "op": "add",
    "path": "/spec/services/-",
    "value": "neutron-metadata"
  }]'

$ oc patch openstackdataplanenodeset <networker_CR_name> --type='json' --patch='[
  {
    "op": "add",
    "path": "/spec/services/-",
    "value": "neutron-dhcp"
  }]'

$ oc apply -f - <<EOF
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneDeployment
metadata:
  name: openstack-networker
spec:
  nodeSets:
  - openstack-networker
EOF

$ watch oc get pod -l app=openstackansibleee

$ oc logs -l app=openstackansibleee -f --max-log-requests 20

$ oc wait --for condition=Ready osdpns/<networker_CR_name> --timeout=30m

$ oc exec openstackclient -- openstack network agent list
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+
| ID                                   | Agent Type                   | Host                   | Availability Zone | Alive | State | Binary                     |
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+
| 174fc099-5cc9-4348-b8fc-59ed44fcfb0e | DHCP agent                   | standalone.localdomain | nova              | :-)   | UP    | neutron-dhcp-agent         |
| 10482583-2130-5b0d-958f-3430da21b929 | OVN Metadata agent           | standalone.localdomain |                   | :-)   | UP    | neutron-ovn-metadata-agent |
| a4f1b584-16f1-4937-b2b0-28102a3f6eaa | OVN Controller Gateway agent | standalone.localdomain |                   | :-)   | UP    | ovn-controller             |
+--------------------------------------+------------------------------+------------------------+-------------------+-------+-------+----------------------------+

$ oc patch openstackcontrolplane openstack --type=merge -p='{"spec":{"swift":{"template":{"swiftStorage":{"replicas": 3}}}}}'

$ oc wait pods --for condition=Ready -l component=swift-storage

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c 'swift-ring-tool get && swift-ring-builder object.builder' | tail -n +7 | awk '{print $4}' | sort -u

172.20.0.100:6200
swift-storage-0.swift-storage.openstack.svc:6200
swift-storage-1.swift-storage.openstack.svc:6200
swift-storage-2.swift-storage.openstack.svc:6200

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c '
swift-ring-tool get
swift-ring-tool drain 172.20.0.100
swift-ring-tool rebalance
swift-ring-tool push'

$ oc extract --confirm cm/swift-ring-files
$CONTROLLER1_SSH "tar -C /var/lib/config-data/puppet-generated/swift/etc/swift/ -xzf -" < swiftrings.tar.gz
$CONTROLLER1_SSH "systemctl restart tripleo_swift_*"

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c "swift-ring-tool get && swift-dispersion-report"

Queried 1024 containers for dispersion reporting, 6s, 0 retries
There were 5 partitions missing 1 copy.
99.84% of container copies found (3067 of 3072)
Sample represents 100.00% of the container partition space
Queried 1024 objects for dispersion reporting, 7s, 0 retries
There were 739 partitions missing 1 copy.
There were 285 partitions missing 0 copies.
75.94% of object copies found (2333 of 3072)
Sample represents 100.00% of the object partition space

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c '
swift-ring-tool get
swift-ring-tool rebalance
swift-ring-tool push'

$ oc extract --confirm cm/swift-ring-files
$CONTROLLER1_SSH "tar -C /var/lib/config-data/puppet-generated/swift/etc/swift/ -xzf -" < swiftrings.tar.gz
$CONTROLLER1_SSH "systemctl restart tripleo_swift_*"

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c '
swift-ring-tool get
swift-ring-tool remove 172.20.0.100
swift-ring-tool rebalance
swift-ring-tool push'

$ oc debug --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c 'swift-ring-tool get && swift-recon -d'

$CONTROLLER1_SSH "find /srv/node/ -type f -name '*.db' -o -name '*.data' | wc -l"

$ sudo cephadm shell -- ceph fs ls
name: cephfs, metadata pool: manila_metadata, data pools: [manila_data ]

$ sudo cephadm shell -- ceph mds stat
cephfs:1 {0=mds.controller-2.oebubl=up:active} 2 up:standby

$ sudo cephadm shell -- ceph fs status cephfs

cephfs - 0 clients
======
RANK  STATE         	MDS           	ACTIVITY 	DNS	INOS   DIRS   CAPS
 0	active  mds.controller-2.oebubl  Reqs:	0 /s   696	196	173  	0
  	POOL     	TYPE 	USED  AVAIL
manila_metadata  metadata   152M   141G
  manila_data  	data	3072M   141G
  	STANDBY MDS
mds.controller-0.anwiwd
mds.controller-1.cwzhog

$ sudo cephadm shell -- ceph fs dump

e8
enable_multiple, ever_enabled_multiple: 1,1
default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2}
legacy client fscid: 1

Filesystem 'cephfs' (1)
fs_name cephfs
epoch   5
flags   12 joinable allow_snaps allow_multimds_snaps
created 2024-01-18T19:04:01.633820+0000
modified    	2024-01-18T19:04:05.393046+0000
tableserver 	0
root	0
session_timeout 60
session_autoclose   	300
max_file_size   1099511627776
required_client_features    	{}
last_failure	0
last_failure_osd_epoch  0
compat  compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2}
max_mds 1
in  	0
up  	{0=24553}
failed
damaged
stopped
data_pools  	[7]
metadata_pool   9
inline_data 	disabled
balancer
standby_count_wanted	1
[mds.mds.controller-2.oebubl{0:24553} state up:active seq 2 addr [v2:172.17.3.114:6800/680266012,v1:172.17.3.114:6801/680266012] compat {c=[1],r=[1],i=[7ff]}]


Standby daemons:

[mds.mds.controller-0.anwiwd{-1:14715} state up:standby seq 1 addr [v2:172.17.3.20:6802/3969145800,v1:172.17.3.20:6803/3969145800] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.controller-1.cwzhog{-1:24566} state up:standby seq 1 addr [v2:172.17.3.43:6800/2227381308,v1:172.17.3.43:6801/2227381308] compat {c=[1],r=[1],i=[7ff]}]
dumped fsmap epoch 8

$ sudo cephadm shell -- ceph osd blocklist ls
..
..
for item in $(sudo cephadm shell -- ceph osd blocklist ls | awk '{print $1}'); do
     sudo cephadm shell -- ceph osd blocklist rm $item;
done

[ceph: root@controller-0 /]# ceph orch host ls
HOST                        ADDR           LABELS          STATUS
cephstorage-0.redhat.local  192.168.24.25  osd
cephstorage-1.redhat.local  192.168.24.50  osd
cephstorage-2.redhat.local  192.168.24.47  osd
controller-0.redhat.local   192.168.24.24  _admin mgr mon
controller-1.redhat.local   192.168.24.42  mgr _admin mon
controller-2.redhat.local   192.168.24.37  mgr _admin mon
6 hosts in cluster

for item in $(sudo cephadm shell --  ceph orch host ls --format json | jq -r '.[].hostname'); do
    sudo cephadm shell -- ceph orch host label add  $item mds;
done

$ sudo cephadm shell -- ceph orch host ls

HOST                    	ADDR       	   LABELS
cephstorage-0.redhat.local  192.168.24.11  osd mds
cephstorage-1.redhat.local  192.168.24.12  osd mds
cephstorage-2.redhat.local  192.168.24.47  osd mds
controller-0.redhat.local   192.168.24.35  _admin mon mgr mds
controller-1.redhat.local   192.168.24.53  mon _admin mgr mds
controller-2.redhat.local   192.168.24.10  mon _admin mgr mds

$ SPEC_DIR=${SPEC_DIR:-"$PWD/ceph_specs"}
$ mkdir -p ${SPEC_DIR}
$ sudo cephadm shell -- ceph orch ls --export mds > ${SPEC_DIR}/mds

service_type: mds
service_id: mds
service_name: mds.mds
placement:
  label: mds

$ SPEC_DIR=${SPEC_DIR:-"$PWD/ceph_specs"}
$ sudo cephadm shell -m ${SPEC_DIR}/mds -- ceph orch apply -i /mnt/mds

Scheduling new mds deployment ...

$ sudo cephadm shell -- ceph fs dump

Active

standby_count_wanted    1
[mds.mds.controller-0.awzplm{0:463158} state up:active seq 307 join_fscid=1 addr [v2:172.17.3.20:6802/51565420,v1:172.17.3.20:6803/51565420] compat {c=[1],r=[1],i=[7ff]}]


Standby daemons:

[mds.mds.cephstorage-1.jkvomp{-1:463800} state up:standby seq 1 join_fscid=1 addr [v2:172.17.3.135:6820/2075903648,v1:172.17.3.135:6821/2075903648] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.controller-2.gfrqvc{-1:475945} state up:standby seq 1 addr [v2:172.17.3.114:6800/2452517189,v1:172.17.3.114:6801/2452517189] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.cephstorage-0.fqcshx{-1:476503} state up:standby seq 1 join_fscid=1 addr [v2:172.17.3.92:6820/4120523799,v1:172.17.3.92:6821/4120523799] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.cephstorage-2.gnfhfe{-1:499067} state up:standby seq 1 addr [v2:172.17.3.79:6820/2448613348,v1:172.17.3.79:6821/2448613348] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.controller-1.tyiziq{-1:499136} state up:standby seq 1 addr [v2:172.17.3.43:6800/3615018301,v1:172.17.3.43:6801/3615018301] compat {c=[1],r=[1],i=[7ff]}]

$ sudo cephadm shell -- ceph config set mds.mds.cephstorage-0.fqcshx mds_join_fs cephfs

$ for i in 0 1 2; do sudo cephadm shell -- ceph orch host label rm "controller-$i.redhat.local" mds; done

Removed label mds from host controller-0.redhat.local
Removed label mds from host controller-1.redhat.local
Removed label mds from host controller-2.redhat.local

$ sudo cephadm shell -- ceph fs dump
…
…
standby_count_wanted    1
[mds.mds.cephstorage-0.fqcshx{0:476503} state up:active seq 168 join_fscid=1 addr [v2:172.17.3.92:6820/4120523799,v1:172.17.3.92:6821/4120523799] compat {c=[1],r=[1],i=[7ff]}]


Standby daemons:

[mds.mds.cephstorage-2.gnfhfe{-1:499067} state up:standby seq 1 addr [v2:172.17.3.79:6820/2448613348,v1:172.17.3.79:6821/2448613348] compat {c=[1],r=[1],i=[7ff]}]
[mds.mds.cephstorage-1.jkvomp{-1:499760} state up:standby seq 1 join_fscid=1 addr [v2:172.17.3.135:6820/452139733,v1:172.17.3.135:6821/452139733] compat {c=[1],r=[1],i=[7ff]}]


$ sudo cephadm shell -- ceph orch ls

NAME                     PORTS   RUNNING  REFRESHED  AGE  PLACEMENT
crash                                6/6  10m ago    10d  *
mds.mds                          3/3  10m ago    32m  label:mds


$ sudo cephadm shell -- ceph orch ps | grep mds


mds.mds.cephstorage-0.fqcshx  cephstorage-0.redhat.local                     running (79m)     3m ago  79m    27.2M        -  17.2.6-100.el9cp  1af7b794f353  2a2dc5ba6d57
mds.mds.cephstorage-1.jkvomp  cephstorage-1.redhat.local                     running (79m)     3m ago  79m    21.5M        -  17.2.6-100.el9cp  1af7b794f353  7198b87104c8
mds.mds.cephstorage-2.gnfhfe  cephstorage-2.redhat.local                     running (79m)     3m ago  79m    24.2M        -  17.2.6-100.el9cp  1af7b794f353  f3cb859e2a15

mgr    advanced  mgr/dashboard/controller-0.ycokob/server_addr  172.17.3.33
mgr    advanced  mgr/dashboard/controller-1.lmzpuc/server_addr  172.17.3.147
mgr    advanced  mgr/dashboard/controller-2.xpdgfl/server_addr  172.17.3.138

$ sudo cephadm shell
$ ceph orch ps | awk '/mgr./ {print $1}'

$ ceph config set mgr mgr/dashboard/<>/server_addr/<ip addr>