文章目录
- 引入
- 传统IO
- 内存映射mmap
- 文件描述符sendFile
- 测试
- 总结
引入
为什么要使用零拷贝技术?
传统写入数据需要4次拷贝,如下图:
传统IO
import java.io.*;
import java.net.Socket;public class TranditionIOClient {private static final int PORT = 8888;private final static String FILE_NAME = "D:\\test.mp4";// 接收缓冲区大小private static final int BUFFER_SIZE = 1024;public static void main(String[] args) throws Exception {try (Socket socket = new Socket("localhost", PORT);InputStream inputStream = new FileInputStream(FILE_NAME);DataOutputStream dos = new DataOutputStream(socket.getOutputStream());) {byte[] buffer = new byte[BUFFER_SIZE];long readCount = 0;long total = 0;long startTime = System.currentTimeMillis();// 读取文件:从硬盘读取到内存,发生2次copy(DMA拷贝和CPU拷贝)while ((readCount = inputStream.read(buffer)) >= 0) {total += readCount;// 网络发送:从内存到网卡,发生2次copy(DMA拷贝和CPU拷贝)dos.write(buffer, 0, (int) readCount);}System.out.println("TranditionIOClient发送总字节数:" + total + ",耗时:" + (System.currentTimeMillis() - startTime) + " ms");} catch (IOException e) {e.printStackTrace();}}
}
内存映射mmap
import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;public class MmapClient {private static final int PORT = 8888;private final static String FILE_NAME = "D:\\test.mp4";public static void main(String[] args) throws Exception {try (SocketChannel socketChannel = SocketChannel.open();FileChannel fileChannel = new FileInputStream(FILE_NAME).getChannel()) {socketChannel.connect(new InetSocketAddress("localhost", PORT));socketChannel.configureBlocking(true);long startTime = System.currentTimeMillis();// 获取文件大小long size = fileChannel.size();// 内存映射整个文件,发生3次copy(DMA拷贝和CPU拷贝)ByteBuffer buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, size);// 发送文件while (buffer.hasRemaining()) {socketChannel.write(buffer);}System.out.println("MmapClient发送总字节数:" + size + ",耗时:" + (System.currentTimeMillis() - startTime) + " ms");} catch (Exception e) {e.printStackTrace();}}
}
文件描述符sendFile
import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;public class SendFileClient {private static final int PORT = 8888;private final static String FILE_NAME = "D:\\test.mp4";public static void main(String[] args) throws Exception {try (SocketChannel socketChannel = SocketChannel.open();FileChannel fileChannel = new FileInputStream(FILE_NAME).getChannel()) {socketChannel.connect(new InetSocketAddress("localhost", PORT));socketChannel.configureBlocking(true);long startTime = System.currentTimeMillis();long position = 0;// 8MB,与系统缓冲区大小匹配或略小以避免问题long chunkSize = 8 * 1024 * 1024;long size = fileChannel.size();while (position < size) {long bytesRemaining = size - position;// 确保最后一次传输不会超过文件大小long count = Math.min(bytesRemaining, chunkSize);// transferTo⽅法⽤到了零拷⻉,底层是sendfile,发生2次copy(DMA拷贝)long transferCount = fileChannel.transferTo(position, count, socketChannel);if (transferCount == 0) {// 如果一次传输没有发生,可能需要检查连接是否仍然活跃或处理其他错误情况break;}position += transferCount;}System.out.println("SendFileClient发送总字节数:" + size + ",耗时:" + (System.currentTimeMillis() - startTime) + " ms");} catch (Exception e) {e.printStackTrace();}}
}
如果发送的文件不大于8M,则可以简单写,如下:
import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;public class SendFileClient {private static final int PORT = 8888;private final static String FILE_NAME = "D:\\test.mp4";public static void main(String[] args) throws Exception {try (SocketChannel socketChannel = SocketChannel.open();FileChannel fileChannel = new FileInputStream(FILE_NAME).getChannel()) {socketChannel.connect(new InetSocketAddress("localhost", PORT));socketChannel.configureBlocking(true);long startTime = System.currentTimeMillis();// transferTo⽅法⽤到了零拷⻉,底层是sendfile,发生2次copy(DMA拷贝)long transferCount = fileChannel.transferTo(0, fileChannel.size(), socketChannel);System.out.println("SendFileClient发送总字节数:" + transferCount + ",耗时:" + (System.currentTimeMillis() - startTime) + " ms");} catch (Exception e) {e.printStackTrace();}}
}
测试
服务端代码如下:
import java.io.DataInputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;public class Server {private static final int PORT = 8888;// 接收缓冲区大小private static final int BUFFER_SIZE = 1024;public static void main(String[] args) throws Exception {try (ServerSocket ss = new ServerSocket(PORT);) {while (true) {try (Socket s = ss.accept();DataInputStream dis = new DataInputStream(s.getInputStream());) {int byteCount = 0;byte[] bytes = new byte[BUFFER_SIZE];while (true) {int readCount = dis.read(bytes, 0, BUFFER_SIZE);if (readCount == -1) {break;}byteCount = byteCount + readCount;}System.out.println("服务端接受字节数:" + byteCount + "字节");} catch (IOException e) {e.printStackTrace();}}} catch (IOException e) {e.printStackTrace();}}
}
我们都使用了test.mp4进行测试,文件大小500M,测试结果如下:
TranditionIOClient发送总字节数:524288000,耗时:9590 ms
MmapClient发送总字节数:524288000,耗时:1182 ms
SendFileClient发送总字节数:524288000,耗时:983 ms
总结
零拷贝并不是不需要拷贝,而是指计算机执行操作时,不需要将数据从内存复制到应用程序
效率高到低:sendFile>mmap>传统IO
明明传了500M的文件,但实际读出来8M?代码如下:
try (SocketChannel socketChannel = SocketChannel.open();FileChannel fileChannel = new FileInputStream(FILE_NAME).getChannel()) {socketChannel.connect(new InetSocketAddress("localhost", PORT));socketChannel.configureBlocking(true);long startTime = System.currentTimeMillis();// transferTo⽅法⽤到了零拷⻉,底层是sendfile,发生2次copy(DMA拷贝)long transferCount = fileChannel.transferTo(0, fileChannel.size(), socketChannel);System.out.println("SendFileClient发送总字节数:" + transferCount + ",耗时:" + (System.currentTimeMillis() - startTime) + " ms");
} catch (Exception e) {e.printStackTrace();
}// 输出结果:SendFileClient发送总字节数:8388608,耗时:15 ms
原因:由于操作系统的默认socket缓冲区大小限制所导致的。当使用transferTo进行大文件传输时,如果文件大小超过了操作系统为socket分配的缓冲区大小,那么transferTo可能在达到这个限制后停止,因为它试图一次性将数据从文件通道转移到socket通道,但缓冲区不足以容纳整个文件内容。
解决:分批次进行文件传输