移动性能测试 python 压测 +paramiko 远程监下载日志 +js 测试报告
- 最近折腾了一个简单的自动化压测，监控日志，生成报告平台，分享下，想得到大家更多的意见谢谢
关于压测客户端
netty nio 压测端
package com.nio.test;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequestEncoder;
import io.netty.handler.codec.http.HttpResponseDecoder;
import io.netty.handler.codec.http.HttpVersion;
import java.net.URI;
import java.nio.charset.StandardCharsets;
public class HttpClient {
public void connect(String host, int port) throws Exception {
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(workerGroup);
b.channel(NioSocketChannel.class);
b.option(ChannelOption.SO_KEEPALIVE, true);
b.handler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
// 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
ch.pipeline().addLast(new HttpResponseDecoder());
// 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
ch.pipeline().addLast(new HttpRequestEncoder());
ch.pipeline().addLast(new HttpClientInboundHandler());
}
});
ChannelFuture f = b.connect(host, port).sync();
URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
String msg = "Are you ok?";
DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET,
uri.toASCIIString(), Unpooled.wrappedBuffer(msg.getBytes("UTF-8")));
// 构建http请求
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, request.content().readableBytes());
// 发送http请求
f.channel().write(request);
f.channel().flush();
f.channel().closeFuture().sync();
// }
} finally {
workerGroup.shutdownGracefully();
}
}
public void connect_post(String host, int port) throws Exception {
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(workerGroup);
b.channel(NioSocketChannel.class);
b.option(ChannelOption.SO_KEEPALIVE, true);
b.handler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
// 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
ch.pipeline().addLast(new HttpResponseDecoder());
// 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
ch.pipeline().addLast(new HttpRequestEncoder());
ch.pipeline().addLast(new HttpClientInboundHandler());
}
});
ChannelFuture f = b.connect(host, port).sync();
URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
FullHttpRequest request = new DefaultFullHttpRequest(
HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath());
// 构建http请求
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); // or HttpHeaders.Values.CLOSE
request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);
request.headers().add(HttpHeaders.Names.CONTENT_TYPE, "application/json");
ByteBuf bbuf = Unpooled.copiedBuffer("{\"jsonrpc\":\"2.0\",\"method\":\"calc.add\",\"params\":[1,2],\"id\":1}", StandardCharsets.UTF_8);
request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, bbuf.readableBytes());
// 发送http请求
f.channel().write(request);
f.channel().flush();
f.channel().closeFuture().sync();
// }
} finally {
workerGroup.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
HttpClient client = new HttpClient();
//请自行修改成服务端的IP
for(int k=0; k<1;k++){
client.connect("http://gc.ditu.aliyun.com/", 80);
System.out.println(k);
}
}
}
监听代码
package com.nio.test;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpResponse;
public class HttpClientInboundHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
System.out.print("success!~!~");
if (msg instanceof HttpResponse)
{
HttpResponse response = (HttpResponse) msg;
// System.out.println("CONTENT_TYPE:" + response.headers().get(HttpHeaders.Names.CONTENT_TYPE));
System.out.println("HTTP_CODE:" + response.getStatus());
}
if(msg instanceof HttpContent)
{
HttpContent content = (HttpContent)msg;
ByteBuf buf = content.content();
System.out.println(buf.toString(io.netty.util.CharsetUtil.UTF_8));
buf.release();
}
ctx.close();
}
}
python tornado 异步框架压测
import tornado.ioloop
from tornado.httpclient import AsyncHTTPClient
def handle_request(response):
if response.error:
print ("Error:", response.error)
else:
print(response.body)
param = {"msg":"111"}
param["img_msg"] = open("t.jpg",'r',encoding='utf-8')
url = 'http://gc.ditu.aliyun.com/geocoding?a=苏州市'
i = 1
print(param)
req = tornado.httpclient.HTTPRequest(url, 'POST', body=str(param))
http_client = AsyncHTTPClient()
while i<10000:
i += 1
http_client.fetch(req, handle_request)
tornado.ioloop.IOLoop.instance().start()
python 协程压测端
#-*- coding: utf-8 -*-
import urllib.request
# url = "http://r.qzone.qq.com/cgi-bin/user/cgi_personal_card?uin=284772894"
url = "http://gc.ditu.aliyun.com/geocoding?a="+urllib.request.quote("苏州市")
HEADER = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:29.0) Gecko/20100101 Firefox/29.0'}
# 自定义请求的方法,get post
def f(url):
import urllib
import urllib.request
data=urllib.request.urlopen(url).read()
z_data=data.decode('UTF-8')
print(z_data)
# 我自己封装了gevent 的方法,重载了run
from gevent import Greenlet
class MyGreenlet(Greenlet):
def __init__(self, func):
Greenlet.__init__(self)
self.func = func
def _run(self):
# gevent.sleep(self.n)
self.func
count = 3
green_let = []
for i in range(0, count):
green_let.append(MyGreenlet(f(url)))
for j in range(0, count):
green_let[j].start()
for k in range(0, count):
green_let[k].join()
- 当然也可以用多线程，apache ab 等，如果是大量并发可以使用 netty nio 压测 + 虚拟机的方式（其实原理差不多，协程，tornado 等）
远程监控
-
把此 shell 脚本上传到服务器
#!/bin/bash if (( $# != 1)) then echo "please input sum times:" exit 1 fi sar -u 1 $1 | sed '1,3d'|sed '$d' > sar_cpu_1.txt sar -r 1 $1 |sed '1,3d'|sed '$d' > sar_men_1.txt sar -b 1 $1 |sed '1,3d'|sed '$d' > sar_io_1.txt 远程上传 shell 脚本，执行，下载日志
import paramiko
import paramiko
server_ip = '192.168.1.1'
server_user = 'root'
server_passwd = ''
server_port = 22
ssh = paramiko.SSHClient()
def ssh_connect():
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(server_ip, server_port,server_user, server_passwd)
return ssh
def client_connect():
client = paramiko.Transport((server_ip, server_port))
client.connect(username = server_user, password = server_passwd)
return client
def ssh_disconnect(client):
client.close()
def exec_cmd(command, ssh):
'''
windows客户端远程执行linux服务器上命令
'''
stdin, stdout, stderr = ssh.exec_command(command)
err = stderr.readline()
out = stdout.readline()
print(stdout.read())
def win_to_linux(localpath, remotepath,client):
'''
windows向linux服务器上传文件.
localpath 为本地文件的绝对路径。如：D: est.py
remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
'''
sftp = paramiko.SFTPClient.from_transport(client)
sftp.put(localpath,remotepath)
client.close()
def linux_to_win(localpath, remotepath,client):
'''
从linux服务器下载文件到本地
localpath 为本地文件的绝对路径。如：D: est.py
remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
'''
sftp = paramiko.SFTPClient.from_transport(client)
sftp.get(remotepath, localpath)
client.close()
class AllowAllKeys(paramiko.MissingHostKeyPolicy):
def missing_host_key(self, client, hostname, key):
return
def muit_exec_cmd(ssh,cmd):
'''
ssh ssh连接
cmd 多命名
'''
ssh.set_missing_host_key_policy(AllowAllKeys())
channel = ssh.invoke_shell()
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stdin.write(cmd)
print(stdout.read())
stdout.close()
stdin.close()
cl = client_connect()
sh = ssh_connect()
win_to_linux("get_men_cpu.sh","/data/get_men_cpu.sh",cl)
exec_cmd("/data/get_men_cpu.sh 100",sh)
# 要和服务器那边的监控时间结束同步，可以设置下sleep时间
linux_to_win("d:/cpu.txt","/data/sar_cpu_1.txt",cl)
linux_to_win("d:/men.txt","/data/sar_men_1.txt",cl)
linux_to_win("d:/io.txt","/data/sar_io_1.txt",cl)
cl.close()
sh.close()
js 解析日志
- js 分析日志文件.cpu,内存,io 都是基于这样分析 用的 highcharts 插件，自己去学习吧
<script>
var idle= [];
var iowait = [];
var categories = [];
var temp;
$(function () {
$.get( "txt/sar_cpu.txt", function( data ) {
var resourceContent = data.toString(); // can be a global variable too...
var rc = resourceContent.split("\n");
for(var i=0; i<rc.length; i++){
temp = getNotNullArr(rc[i].split(" "));
console.log(temp);
idle.push(parseFloat(temp[temp.length-1]));
categories.push(temp[0]);
iowait.push(parseFloat(temp[6]));
}
set_h_title("cpu使用情况");
set_h_sub_title("test.com");
set_h_xaxis_setp(1);
set_h_tooltip("%");
set_h_yaxis_title("cpu使用情况");
set_h_xaxis_categories(categories);
set_h_series( [{name: 'idle',data:idle}, {name: 'iowait', data: iowait}]);
highchartsinit("#container",get_h_title(),get_h_sub_title(),get_h_xaxis_setp(),get_h_xaxis_categories(),get_h_yaxis_title(),get_h_tooltip(),get_h_series());
});
});
</script>
更多参考
如果觉得我的文章对您有用，请随意打赏。您的支持将鼓励我继续创作！
