最近折腾了一个简单的自动化压测，监控日志，生成报告平台，分享下，想得到大家更多的意见谢谢

关于压测客户端

netty nio 压测端

package com.nio.test;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequestEncoder;
import io.netty.handler.codec.http.HttpResponseDecoder;
import io.netty.handler.codec.http.HttpVersion;
import java.net.URI;
import java.nio.charset.StandardCharsets;
public class HttpClient {
    public void connect(String host, int port) throws Exception {
        EventLoopGroup workerGroup = new NioEventLoopGroup();
        try {
            Bootstrap b = new Bootstrap();
            b.group(workerGroup);
            b.channel(NioSocketChannel.class);
            b.option(ChannelOption.SO_KEEPALIVE, true);
            b.handler(new ChannelInitializer<SocketChannel>() {
                @Override
                public void initChannel(SocketChannel ch) throws Exception {
                    // 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
                    ch.pipeline().addLast(new HttpResponseDecoder());
                    // 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
                    ch.pipeline().addLast(new HttpRequestEncoder());
                    ch.pipeline().addLast(new HttpClientInboundHandler()); 
                }
            });
                 ChannelFuture f = b.connect(host, port).sync();

               URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
                 String msg = "Are you ok?";
                 DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET,
                         uri.toASCIIString(), Unpooled.wrappedBuffer(msg.getBytes("UTF-8")));                             
                 // 构建http请求
                 request.headers().set(HttpHeaders.Names.HOST, host);
                 request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
                 request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, request.content().readableBytes());
                 // 发送http请求
                 f.channel().write(request);
                 f.channel().flush();
                 f.channel().closeFuture().sync();                
//            }

        } finally {
            workerGroup.shutdownGracefully();
        }
    }
    public void connect_post(String host, int port) throws Exception {
        EventLoopGroup workerGroup = new NioEventLoopGroup();

        try {
            Bootstrap b = new Bootstrap();
            b.group(workerGroup);
            b.channel(NioSocketChannel.class);
            b.option(ChannelOption.SO_KEEPALIVE, true);
            b.handler(new ChannelInitializer<SocketChannel>() {
                @Override
                public void initChannel(SocketChannel ch) throws Exception {
                    // 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
                    ch.pipeline().addLast(new HttpResponseDecoder());
                    // 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
                    ch.pipeline().addLast(new HttpRequestEncoder());
                    ch.pipeline().addLast(new HttpClientInboundHandler()); 
                }
            });

                 ChannelFuture f = b.connect(host, port).sync();

               URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
               FullHttpRequest request = new DefaultFullHttpRequest(
                       HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath());                        
                 // 构建http请求
               request.headers().set(HttpHeaders.Names.HOST, host);
               request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); // or HttpHeaders.Values.CLOSE
               request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);
               request.headers().add(HttpHeaders.Names.CONTENT_TYPE, "application/json");
               ByteBuf bbuf = Unpooled.copiedBuffer("{\"jsonrpc\":\"2.0\",\"method\":\"calc.add\",\"params\":[1,2],\"id\":1}", StandardCharsets.UTF_8);
               request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, bbuf.readableBytes());
                 // 发送http请求
                 f.channel().write(request);
                 f.channel().flush();
                 f.channel().closeFuture().sync();

//            }

        } finally {
            workerGroup.shutdownGracefully();
        }
    }
    public static void main(String[] args) throws Exception {
        HttpClient client = new HttpClient();
            //请自行修改成服务端的IP
        for(int k=0; k<1;k++){
          client.connect("http://gc.ditu.aliyun.com/", 80);

            System.out.println(k);
            }
    }
}
监听代码
package com.nio.test;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpResponse;
public class HttpClientInboundHandler extends ChannelInboundHandlerAdapter {
    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
        System.out.print("success!~!~");
        if (msg instanceof HttpResponse) 
        {
            HttpResponse response = (HttpResponse) msg;
//            System.out.println("CONTENT_TYPE:" + response.headers().get(HttpHeaders.Names.CONTENT_TYPE));
            System.out.println("HTTP_CODE:" + response.getStatus());

        }
        if(msg instanceof HttpContent)
        {
            HttpContent content = (HttpContent)msg;
            ByteBuf buf = content.content();

            System.out.println(buf.toString(io.netty.util.CharsetUtil.UTF_8));
            buf.release();
        }
        ctx.close();
    }
}

python tornado 异步框架压测

import tornado.ioloop
from tornado.httpclient import AsyncHTTPClient
def handle_request(response):
  if response.error:
      print ("Error:", response.error)
  else:
      print(response.body)
param = {"msg":"111"}
param["img_msg"] = open("t.jpg",'r',encoding='utf-8')
url = 'http://gc.ditu.aliyun.com/geocoding?a=苏州市'
i = 1
print(param)
req = tornado.httpclient.HTTPRequest(url, 'POST', body=str(param))
http_client = AsyncHTTPClient()
while i<10000:
  i += 1
  http_client.fetch(req, handle_request)
tornado.ioloop.IOLoop.instance().start()

python 协程压测端

#-*- coding: utf-8 -*-
import urllib.request
# url = "http://r.qzone.qq.com/cgi-bin/user/cgi_personal_card?uin=284772894"
url = "http://gc.ditu.aliyun.com/geocoding?a="+urllib.request.quote("苏州市")
HEADER = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:29.0) Gecko/20100101 Firefox/29.0'}

# 自定义请求的方法,get post
def f(url):
    import urllib
    import urllib.request
    data=urllib.request.urlopen(url).read()
    z_data=data.decode('UTF-8')
    print(z_data)

# 我自己封装了gevent 的方法,重载了run
from gevent import Greenlet
class MyGreenlet(Greenlet):
    def __init__(self, func):
        Greenlet.__init__(self)
        self.func = func
    def _run(self):
        # gevent.sleep(self.n)
        self.func


count = 3
green_let = []
for i in range(0, count):
    green_let.append(MyGreenlet(f(url)))
for j in range(0, count):
    green_let[j].start()
for k in range(0, count):
    green_let[k].join()

当然也可以用多线程，apache ab 等，如果是大量并发可以使用 netty nio 压测 + 虚拟机的方式（其实原理差不多，协程，tornado 等）

远程监控

把此 shell 脚本上传到服务器

#!/bin/bash
if (( $# != 1))
then
echo "please input sum times:"
exit 1
fi
sar -u 1 $1 | sed '1,3d'|sed '$d' > sar_cpu_1.txt
sar -r 1 $1 |sed '1,3d'|sed '$d' > sar_men_1.txt
sar -b 1 $1 |sed '1,3d'|sed '$d' > sar_io_1.txt

远程上传 shell 脚本，执行，下载日志

import paramiko
import paramiko
server_ip = '192.168.1.1'
server_user = 'root'
server_passwd = ''
server_port = 22
ssh = paramiko.SSHClient()
def ssh_connect():
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.load_system_host_keys()
    ssh.connect(server_ip, server_port,server_user, server_passwd)
    return ssh
def client_connect():
    client = paramiko.Transport((server_ip, server_port))
    client.connect(username = server_user, password = server_passwd)
    return client
def ssh_disconnect(client):
    client.close()

def exec_cmd(command, ssh):
    '''
    windows客户端远程执行linux服务器上命令
    '''
    stdin, stdout, stderr = ssh.exec_command(command)
    err = stderr.readline()
    out = stdout.readline()
    print(stdout.read())

def win_to_linux(localpath, remotepath,client):
    '''
    windows向linux服务器上传文件.
    localpath  为本地文件的绝对路径。如：D:  est.py
    remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
    '''

    sftp = paramiko.SFTPClient.from_transport(client)
    sftp.put(localpath,remotepath)
    client.close()

def linux_to_win(localpath, remotepath,client):
    '''
    从linux服务器下载文件到本地
    localpath  为本地文件的绝对路径。如：D:  est.py
    remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
    '''
    sftp = paramiko.SFTPClient.from_transport(client)
    sftp.get(remotepath, localpath)
    client.close()

class AllowAllKeys(paramiko.MissingHostKeyPolicy):
   def missing_host_key(self, client, hostname, key):
       return

def muit_exec_cmd(ssh,cmd):
    '''
    ssh ssh连接
    cmd 多命名
    '''
    ssh.set_missing_host_key_policy(AllowAllKeys())
    channel = ssh.invoke_shell()
    stdin = channel.makefile('wb')
    stdout = channel.makefile('rb')

    stdin.write(cmd)
    print(stdout.read())

    stdout.close()
    stdin.close()

cl = client_connect()
sh = ssh_connect()
win_to_linux("get_men_cpu.sh","/data/get_men_cpu.sh",cl)
exec_cmd("/data/get_men_cpu.sh 100",sh)
# 要和服务器那边的监控时间结束同步，可以设置下sleep时间
linux_to_win("d:/cpu.txt","/data/sar_cpu_1.txt",cl)
linux_to_win("d:/men.txt","/data/sar_men_1.txt",cl)
linux_to_win("d:/io.txt","/data/sar_io_1.txt",cl)
cl.close()
sh.close()

js 解析日志

js 分析日志文件.cpu,内存,io 都是基于这样分析用的 highcharts 插件，自己去学习吧

<script>
    var idle= [];
    var iowait = [];
    var categories = [];
    var temp;
    $(function () {
        $.get( "txt/sar_cpu.txt", function( data ) {
            var resourceContent = data.toString(); // can be a global variable too...
            var rc = resourceContent.split("\n");
            for(var i=0; i<rc.length; i++){
             temp = getNotNullArr(rc[i].split(" "));
                console.log(temp);
                idle.push(parseFloat(temp[temp.length-1]));
                categories.push(temp[0]);
                iowait.push(parseFloat(temp[6]));
            }
            set_h_title("cpu使用情况");
            set_h_sub_title("test.com");
            set_h_xaxis_setp(1);
            set_h_tooltip("%");
            set_h_yaxis_title("cpu使用情况");
            set_h_xaxis_categories(categories);
            set_h_series( [{name: 'idle',data:idle}, {name: 'iowait', data: iowait}]);
            highchartsinit("#container",get_h_title(),get_h_sub_title(),get_h_xaxis_setp(),get_h_xaxis_categories(),get_h_yaxis_title(),get_h_tooltip(),get_h_series());
    });
    });
</script>

关于压测客户端

netty nio 压测端

python tornado 异步框架压测

python 协程压测端

远程监控

js 解析日志

更多参考