cloudstack下libvirtd服务无响应问题

2019-10-13 21:14:50王振洲

并未获得致命错误和更多线索。(该日志配置选项还是很有必要打开的,很多问题都可以通过它来定位)

解决过程

解决思路

尝试和找到终止进程、重启服务的方法
提交bug,等待补丁升级
分析源代码,再现问题,解决问题(投入研发和时间)
由于不能再现问题,还是从简入繁吧。触发这些子进程的元凶是谁?还是cloudstack-agent的嫌疑最大,但之前重启过该服务并没有解决问题,那么agent服务是怎么一回事呢?

看下启动脚本可以基本了解,

[root@node5 libvirt]# cat /etc/rc.d/init.d/cloudstack-agent

#!/bin/bash

# chkconfig: 35 99 10
# description: Cloud Agent

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# WARNING: if this script is changed, then all other initscripts MUST BE changed to match it as well

. /etc/rc.d/init.d/functions

# set environment variables

SHORTNAME=$(basename $0 | sed -e 's/^[SK][0-9][0-9]//')
PIDFILE=/var/run/"$SHORTNAME".pid
LOCKFILE=/var/lock/subsys/"$SHORTNAME"
LOGDIR=/var/log/cloudstack/agent
LOGFILE=${LOGDIR}/agent.log
PROGNAME="Cloud Agent"
CLASS="com.cloud.agent.AgentShell"
JSVC=`which jsvc 2>/dev/null`;

# exit if we don't find jsvc
if [ -z "$JSVC" ]; then
  echo no jsvc found in path;
  exit 1;
fi

unset OPTIONS
[ -r /etc/sysconfig/"$SHORTNAME" ] && source /etc/sysconfig/"$SHORTNAME"

# The first existing directory is used for JAVA_HOME (if JAVA_HOME is not defined in $DEFAULT)
JDK_DIRS="/usr/lib/jvm/jre /usr/lib/jvm/java-7-openjdk /usr/lib/jvm/java-7-openjdk-i386 /usr/lib/jvm/java-7-openjdk-amd64 /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-openjdk-i386 /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-sun"

for jdir in $JDK_DIRS; do
  if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then
    JAVA_HOME="$jdir"
  fi
done
export JAVA_HOME

ACP=`ls /usr/share/cloudstack-agent/lib/*.jar | tr 'n' ':' | sed s'/.$//'`
PCP=`ls /usr/share/cloudstack-agent/plugins/*.jar 2>/dev/null | tr 'n' ':' | sed s'/.$//'`

# We need to append the JSVC daemon JAR to the classpath
# AgentShell implements the JSVC daemon methods
export CLASSPATH="/usr/share/java/commons-daemon.jar:$ACP:$PCP:/etc/cloudstack/agent:/usr/share/cloudstack-common/scripts"

start() {
  echo -n $"Starting $PROGNAME: "
  if hostname --fqdn >/dev/null 2>&1 ; then
    $JSVC -Xms256m -Xmx2048m -cp "$CLASSPATH" -pidfile "$PIDFILE" 
      -errfile $LOGDIR/cloudstack-agent.err -outfile $LOGDIR/cloudstack-agent.out $CLASS
    RETVAL=$?
    echo
  else
    failure
    echo
    echo The host name does not resolve properly to an IP address. Cannot start "$PROGNAME". > /dev/stderr
    RETVAL=9
  fi
  [ $RETVAL = 0 ] && touch ${LOCKFILE}
  return $RETVAL
}

stop() {
  echo -n $"Stopping $PROGNAME: "
  $JSVC -pidfile "$PIDFILE" -stop $CLASS
  RETVAL=$?
  echo
  [ $RETVAL = 0 ] && rm -f ${LOCKFILE} ${PIDFILE}
}

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  status)
    status -p ${PIDFILE} $SHORTNAME
    RETVAL=$?
    ;;
  restart)
    stop
    sleep 3
    start
    ;;
  condrestart)
    if status -p ${PIDFILE} $SHORTNAME >&/dev/null; then
      stop
      sleep 3
      start
    fi
    ;;
  *)
  echo $"Usage: $SHORTNAME {start|stop|restart|condrestart|status|help}"
  RETVAL=3
esac

exit $RETVAL