Merge pull request #4863 from pocoproject/fix-sendFile

Fix StreamSocket::sendFile()
This commit is contained in:
Günter Obiltschnig
2025-02-08 19:05:46 +01:00
committed by GitHub
10 changed files with 529 additions and 71 deletions

View File

@@ -10,7 +10,7 @@ POCO_HEADERS_AUTO(SRCS ${HDRS_G})
POCO_SOURCES_AUTO_PLAT(SRCS WIN32 src/wepoll.c)
POCO_HEADERS_AUTO(SRCS src/wepoll.h)
if (MSVC)
if (MSVC OR MINGW)
set(HAVE_SENDFILE ON)
else()
include(CheckIncludeFiles)
@@ -18,8 +18,8 @@ else()
check_include_files(sys/sendfile.h HAVE_SYS_SENDFILE_H)
if(HAVE_SYS_SENDFILE_H)
check_symbol_exists(sendfile sys/sendfile.h HAVE_SENDFILE)
if (NOT DEFINED HAVE_SENDFILE)
check_symbol_exists(sendfile64 sys/sendfile.h HAVE_SENDFILE)
if (NOT HAVE_SENDFILE)
check_symbol_exists(sendfile64 sys/sendfile.h HAVE_SENDFILE64)
endif()
else()
# BSD version
@@ -27,7 +27,7 @@ else()
endif()
endif()
if (DEFINED HAVE_SENDFILE)
if (HAVE_SENDFILE OR HAVE_SENDFILE64)
message(STATUS "OS has native sendfile function")
add_compile_definitions(POCO_HAVE_SENDFILE)
endif()

View File

@@ -287,6 +287,25 @@ public:
/// The preferred way for a socket to receive urgent data
/// is by enabling the SO_OOBINLINE option.
virtual std::streamsize sendFile(Poco::FileInputStream& FileInputStream, std::streamoff offset = 0, std::streamsize count = 0);
/// Sends the contents of a file in an optimized way, if possible.
///
/// If count is != 0, sends the given number of bytes, otherwise
/// sends all bytes, starting from the given offset.
///
/// On POSIX systems, this means using sendfile() or sendfile64().
/// On Windows, this means using TransmitFile().
///
/// If neither is available, or the socket is a SecureSocketImpl()
/// (secure() returns true), falls back to reading the file
/// block by block and callind sendBytes().
///
/// Returns the number of bytes sent, which may be
/// less than the number of bytes specified.
///
/// Throws NetException (or a subclass) in case of any errors.
/// Also throws if the socket is non-blocking.
virtual int available();
/// Returns the number of bytes available that can be read
/// without causing the socket to block.
@@ -487,17 +506,7 @@ public:
bool initialized() const;
/// Returns true iff the underlying socket is initialized.
#ifdef POCO_HAVE_SENDFILE
Int64 sendFile(FileInputStream &FileInputStream, UInt64 offset = 0);
/// Sends file using system function
/// for posix systems - with sendfile[64](...)
/// for windows - with TransmitFile(...)
///
/// Returns the number of bytes sent, which may be
/// less than the number of bytes specified.
///
/// Throws NetException (or a subclass) in case of any errors.
#endif
protected:
SocketImpl();
/// Creates a SocketImpl.
@@ -538,6 +547,14 @@ protected:
void checkBrokenTimeout(SelectMode mode);
std::streamsize sendFileNative(Poco::FileInputStream& FileInputStream, std::streamoff offset, std::streamsize count);
/// Implements sendFile() using an OS-specific API like
/// sendfile() or TransmitFile().
std::streamsize sendFileBlockwise(Poco::FileInputStream& FileInputStream, std::streamoff offset, std::streamsize count);
/// Implements sendFile() by reading the file blockwise and
/// calling sendBytes() for each block.
static int lastError();
/// Returns the last error code.

View File

@@ -267,17 +267,26 @@ public:
///
/// The preferred way for a socket to receive urgent data
/// is by enabling the SO_OOBINLINE option.
#ifdef POCO_HAVE_SENDFILE
IntPtr sendFile(FileInputStream &FileInputStream, UIntPtr offset = 0);
/// Sends file using system function
/// for posix systems - with sendfile[64](...)
/// for windows - with TransmitFile(...)
std::streamsize sendFile(Poco::FileInputStream& FileInputStream, std::streamoff offset = 0, std::streamsize count = 0);
/// Sends the contents of a file in an optimized way, if possible.
///
/// If count is != 0, sends the given number of bytes, otherwise
/// sends all bytes, starting from the given offset.
///
/// On POSIX systems, this means using sendfile() or sendfile64().
/// On Windows, this means using TransmitFile().
///
/// If neither is available, or the socket is a SecureSocketImpl()
/// (secure() returns true), falls back to reading the file
/// block by block and callind sendBytes().
///
/// Returns the number of bytes sent, which may be
/// less than the number of bytes specified.
///
/// Throws NetException (or a subclass) in case of any errors.
#endif
/// Also throws if the socket is non-blocking.
StreamSocket(SocketImpl* pImpl);
/// Creates the Socket and attaches the given SocketImpl.
/// The socket takes ownership of the SocketImpl.

View File

@@ -135,8 +135,8 @@ void HTTPServerResponseImpl::sendFile(const std::string& path, const std::string
#ifdef POCO_HAVE_SENDFILE
_pStream->flush(); // flush the HTTP headers to the socket, required by HTTP 1.0 and above
Poco::IntPtr sent = 0;
Poco::IntPtr offset = 0;
std::streamsize sent = 0;
std::streamoff offset = 0;
while (sent < length)
{
offset = sent;

View File

@@ -696,6 +696,27 @@ void SocketImpl::sendUrgent(unsigned char data)
}
std::streamsize SocketImpl::sendFile(FileInputStream& fileInputStream, std::streamoff offset, std::streamsize count)
{
if (!getBlocking()) throw NetException("sendFile() not supported for non-blocking sockets");
#ifdef POCO_HAVE_SENDFILE
if (secure())
{
return sendFileBlockwise(fileInputStream, offset, count);
}
else
{
return sendFileNative(fileInputStream, offset, count);
}
#else
return sendFileBlockwise(fileInputStream, offset, count);
#endif
}
int SocketImpl::available()
{
int result = 0;
@@ -1424,13 +1445,16 @@ void SocketImpl::error(int code, const std::string& arg)
throw IOException(NumberFormatter::format(code), arg, code);
}
}
#ifdef POCO_HAVE_SENDFILE
#ifdef POCO_OS_FAMILY_WINDOWS
Poco::Int64 SocketImpl::sendFile(FileInputStream &fileInputStream, Poco::UInt64 offset)
std::streamsize SocketImpl::sendFileNative(FileInputStream& fileInputStream, std::streamoff offset, std::streamsize count)
{
FileIOS::NativeHandle fd = fileInputStream.nativeHandle();
UInt64 fileSize = fileInputStream.size();
std::streamoff sentSize = fileSize - offset;
if (count == 0) count = fileInputStream.size() - offset;
LARGE_INTEGER offsetHelper;
offsetHelper.QuadPart = offset;
OVERLAPPED overlapped;
@@ -1441,57 +1465,75 @@ Poco::Int64 SocketImpl::sendFile(FileInputStream &fileInputStream, Poco::UInt64
if (overlapped.hEvent == nullptr)
{
int err = GetLastError();
error(err, std::string("[sendfile error]") + Error::getMessage(err));
error(err);
}
bool result = TransmitFile(_sockfd, fd, sentSize, 0, &overlapped, nullptr, 0);
bool result = TransmitFile(_sockfd, fd, count, 0, &overlapped, nullptr, 0);
if (!result)
{
int err = WSAGetLastError();
if ((err != ERROR_IO_PENDING) && (WSAGetLastError() != WSA_IO_PENDING)) {
if ((err != ERROR_IO_PENDING) && (WSAGetLastError() != WSA_IO_PENDING))
{
CloseHandle(overlapped.hEvent);
error(err, std::string("[sendfile error]") + Error::getMessage(err));
error(err);
}
WaitForSingleObject(overlapped.hEvent, INFINITE);
}
CloseHandle(overlapped.hEvent);
return sentSize;
}
#else
Int64 _sendfile(poco_socket_t sd, FileIOS::NativeHandle fd, UInt64 offset, std::streamoff sentSize)
{
Int64 sent = 0;
#ifdef __USE_LARGEFILE64
sent = sendfile64(sd, fd, (off64_t *)&offset, sentSize);
#else
#if POCO_OS == POCO_OS_LINUX && !defined(POCO_EMSCRIPTEN)
sent = sendfile(sd, fd, (off_t *)&offset, sentSize);
#elif POCO_OS == POCO_OS_MAC_OS_X
int result = sendfile(fd, sd, offset, &sentSize, nullptr, 0);
if (result < 0)
{
sent = -1;
}
else
{
sent = sentSize;
}
#else
throw Poco::NotImplementedException("sendfile not implemented for this platform");
#endif
#endif
if (errno == EAGAIN || errno == EWOULDBLOCK)
{
sent = 0;
}
return sent;
return count;
}
Int64 SocketImpl::sendFile(FileInputStream &fileInputStream, UInt64 offset)
#else
namespace
{
std::streamsize sendFileUnix(poco_socket_t sd, FileIOS::NativeHandle fd, std::streamsize offset, std::streamoff count)
{
Int64 sent = 0;
#ifdef __USE_LARGEFILE64
sent = sendfile64(sd, fd, (off64_t*) &offset, count);
#else
#if POCO_OS == POCO_OS_LINUX && !defined(POCO_EMSCRIPTEN)
sent = sendfile(sd, fd, (off_t*) &offset, count);
#elif POCO_OS == POCO_OS_MAC_OS_X
int result = sendfile(fd, sd, offset, &count, NULL, 0);
if (result < 0)
{
sent = -1;
}
else
{
sent = count;
}
#elif POCO_OS == POCO_OS_FREE_BSD
int result = sendfile(fd, sd, offset, &count, NULL, NULL, 0);
if (result < 0)
{
sent = -1;
}
else
{
sent = count;
}
#else
throw Poco::NotImplementedException("sendfile not implemented for this platform");
#endif
#endif
if (errno == EAGAIN || errno == EWOULDBLOCK)
{
sent = 0;
}
return sent;
}
}
std::streamsize SocketImpl::sendFileNative(FileInputStream& fileInputStream, std::streamoff offset, std::streamsize count)
{
FileIOS::NativeHandle fd = fileInputStream.nativeHandle();
UInt64 fileSize = fileInputStream.size();
std::streamoff sentSize = fileSize - offset;
Int64 sent = 0;
if (count == 0) count = fileInputStream.size() - offset;
std::streamsize sent = 0;
struct sigaction sa, old_sa;
sa.sa_handler = SIG_IGN;
sigemptyset(&sa.sa_mask);
@@ -1500,20 +1542,53 @@ Int64 SocketImpl::sendFile(FileInputStream &fileInputStream, UInt64 offset)
while (sent == 0)
{
errno = 0;
sent = _sendfile(_sockfd, fd, offset, sentSize);
sent = sendFileUnix(_sockfd, fd, offset, count);
if (sent < 0)
{
error(errno, std::string("[sendfile error]") + Error::getMessage(errno));
error(errno);
}
}
if(old_sa.sa_handler == SIG_ERR)
if (old_sa.sa_handler == SIG_ERR)
{
old_sa.sa_handler = SIG_DFL;
}
sigaction(SIGPIPE, &old_sa, nullptr);
return sent;
}
#endif // POCO_OS_FAMILY_WINDOWS
#endif // POCO_HAVE_SENDFILE
std::streamsize SocketImpl::sendFileBlockwise(FileInputStream& fileInputStream, std::streamoff offset, std::streamsize count)
{
fileInputStream.seekg(offset, std::ios_base::beg);
Poco::Buffer<char> buffer(8192);
std::size_t bufferSize = buffer.size();
if (count > 0 && bufferSize > count) bufferSize = count;
std::streamsize len = 0;
fileInputStream.read(buffer.begin(), bufferSize);
std::streamsize n = fileInputStream.gcount();
while (n > 0 && (count == 0 || len < count))
{
len += n;
sendBytes(buffer.begin(), n);
if (count > 0 && len < count)
{
const std::size_t remaining = count - len;
if (bufferSize > remaining) bufferSize = remaining;
}
if (fileInputStream)
{
fileInputStream.read(buffer.begin(), bufferSize);
n = fileInputStream.gcount();
}
else n = 0;
}
return len;
}
} } // namespace Poco::Net

View File

@@ -212,10 +212,12 @@ void StreamSocket::sendUrgent(unsigned char data)
{
impl()->sendUrgent(data);
}
#ifdef POCO_HAVE_SENDFILE
IntPtr StreamSocket::sendFile(FileInputStream &fileInputStream, UIntPtr offset)
std::streamsize StreamSocket::sendFile(Poco::FileInputStream& fileInputStream, std::streamoff offset, std::streamsize count)
{
return impl()->sendFile(fileInputStream, offset);
return impl()->sendFile(fileInputStream, offset, count);
}
#endif
} } // namespace Poco::Net

View File

@@ -15,6 +15,9 @@
#include "Poco/Net/StreamSocket.h"
#include "Poco/Net/ServerSocket.h"
#include "Poco/Net/SocketAddress.h"
#include "Poco/Net/TCPServerConnection.h"
#include "Poco/Net/TCPServerConnectionFactory.h"
#include "Poco/Net/TCPServer.h"
#include "Poco/Net/NetException.h"
#include "Poco/Timespan.h"
#include "Poco/Stopwatch.h"
@@ -22,7 +25,10 @@
#include "Poco/FIFOBuffer.h"
#include "Poco/Delegate.h"
#include "Poco/File.h"
#include "Poco/TemporaryFile.h"
#include "Poco/FileStream.h"
#include "Poco/Path.h"
#include "Poco/Thread.h"
#include <iostream>
@@ -31,6 +37,9 @@ using Poco::Net::StreamSocket;
using Poco::Net::ServerSocket;
using Poco::Net::SocketAddress;
using Poco::Net::ConnectionRefusedException;
using Poco::Net::TCPServerConnection;
using Poco::Net::TCPServerConnectionFactoryImpl;
using Poco::Net::TCPServer;
using Poco::Timespan;
using Poco::Stopwatch;
using Poco::TimeoutException;
@@ -42,6 +51,58 @@ using Poco::File;
using Poco::delegate;
namespace
{
class CopyToStringConnection: public TCPServerConnection
{
public:
CopyToStringConnection(const StreamSocket& s):
TCPServerConnection(s)
{
}
void run()
{
{
Poco::FastMutex::ScopedLock lock(_mutex);
_data.clear();
}
StreamSocket& ss = socket();
try
{
char buffer[256];
int n = ss.receiveBytes(buffer, sizeof(buffer));
while (n > 0)
{
{
Poco::FastMutex::ScopedLock lock(_mutex);
_data.append(buffer, n);
}
n = ss.receiveBytes(buffer, sizeof(buffer));
}
}
catch (Poco::Exception& exc)
{
std::cerr << "CopyToStringConnection: " << exc.displayText() << std::endl;
}
}
static const std::string data()
{
Poco::FastMutex::ScopedLock lock(_mutex);
return _data;
}
private:
static Poco::FastMutex _mutex;
static std::string _data;
};
Poco::FastMutex CopyToStringConnection::_mutex;
std::string CopyToStringConnection::_data;
}
SocketTest::SocketTest(const std::string& name): CppUnit::TestCase(name)
{
}
@@ -667,6 +728,121 @@ void SocketTest::testUseFd()
}
void SocketTest::testSendFile()
{
ServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
StreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string sentData = "Hello, world!";
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
ostr.write(sentData.data(), sentData.size());
ostr.close();
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr);
assertTrue (n == file.getSize());
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == sentData);
}
void SocketTest::testSendFileLarge()
{
ServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
StreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string sentData;
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
std::string data("0123456789abcdef");
for (int i = 0; i < 10000; i++)
{
ostr.write(data.data(), data.size());
sentData += data;
}
ostr.close();
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr);
assertTrue (n == file.getSize());
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == sentData);
}
void SocketTest::testSendFileRange()
{
ServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
StreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string sentData;
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
std::string data("0123456789abcdef");
for (int i = 0; i < 1024; i++)
{
ostr.write(data.data(), data.size());
sentData += data;
}
ostr.close();
const std::streamoff offset = 4000;
const std::streamsize count = 10000;
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr, offset, count);
assertTrue (n == count);
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == sentData.substr(offset, count));
}
void SocketTest::onReadable(bool& b)
{
if (b) ++_notToReadable;
@@ -724,6 +900,9 @@ CppUnit::Test* SocketTest::suite()
CppUnit_addTest(pSuite, SocketTest, testEchoUnixLocal);
CppUnit_addTest(pSuite, SocketTest, testUnixLocalAbstract);
CppUnit_addTest(pSuite, SocketTest, testUseFd);
CppUnit_addTest(pSuite, SocketTest, testSendFile);
CppUnit_addTest(pSuite, SocketTest, testSendFileLarge);
CppUnit_addTest(pSuite, SocketTest, testSendFileRange);
return pSuite;
}

View File

@@ -45,9 +45,13 @@ public:
void testSelect2();
void testSelect3();
#endif
void testEchoUnixLocal();
void testUnixLocalAbstract();
void testUseFd();
void testSendFile();
void testSendFileLarge();
void testSendFileRange();
void setUp() override;
void tearDown() override;

View File

@@ -25,6 +25,9 @@
#include "Poco/Util/Application.h"
#include "Poco/Util/AbstractConfiguration.h"
#include "Poco/Thread.h"
#include "Poco/File.h"
#include "Poco/TemporaryFile.h"
#include "Poco/FileStream.h"
#include <iostream>
@@ -72,6 +75,54 @@ namespace
}
}
};
class CopyToStringConnection: public TCPServerConnection
{
public:
CopyToStringConnection(const StreamSocket& s):
TCPServerConnection(s)
{
}
void run()
{
{
Poco::FastMutex::ScopedLock lock(_mutex);
_data.clear();
}
StreamSocket& ss = socket();
try
{
char buffer[256];
int n = ss.receiveBytes(buffer, sizeof(buffer));
while (n > 0)
{
{
Poco::FastMutex::ScopedLock lock(_mutex);
_data.append(buffer, n);
}
n = ss.receiveBytes(buffer, sizeof(buffer));
}
}
catch (Poco::Exception& exc)
{
std::cerr << "CopyToStringConnection: " << exc.displayText() << std::endl;
}
}
static const std::string data()
{
Poco::FastMutex::ScopedLock lock(_mutex);
return _data;
}
private:
static Poco::FastMutex _mutex;
static std::string _data;
};
Poco::FastMutex CopyToStringConnection::_mutex;
std::string CopyToStringConnection::_data;
}
@@ -200,6 +251,121 @@ void SecureStreamSocketTest::testNB()
}
void SecureStreamSocketTest::testSendFile()
{
SecureServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
SecureStreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string sentData = "Hello, world!";
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
ostr.write(sentData.data(), sentData.size());
ostr.close();
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr);
assertTrue (n == file.getSize());
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == sentData);
}
void SecureStreamSocketTest::testSendFileLarge()
{
SecureServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
SecureStreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string sentData;
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
std::string data("0123456789abcdef");
for (int i = 0; i < 10000; i++)
{
ostr.write(data.data(), data.size());
sentData += data;
}
ostr.close();
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr);
assertTrue (n == file.getSize());
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == sentData);
}
void SecureStreamSocketTest::testSendFileRange()
{
SecureServerSocket svs(0);
TCPServer srv(new TCPServerConnectionFactoryImpl<CopyToStringConnection>(), svs);
srv.start();
SecureStreamSocket ss;
ss.connect(SocketAddress("127.0.0.1", srv.port()));
std::string fileData;
Poco::TemporaryFile file;
Poco::FileOutputStream ostr(file.path());
std::string data("0123456789abcdef");
for (int i = 0; i < 1024; i++)
{
ostr.write(data.data(), data.size());
fileData += data;
}
ostr.close();
const std::streamoff offset = 4000;
const std::streamsize count = 10000;
Poco::FileInputStream istr(file.path());
std::streamsize n = ss.sendFile(istr, offset, count);
assertTrue (n == count);
istr.close();
ss.close();
Poco::Thread::sleep(200);
while (srv.currentConnections() > 0)
{
Poco::Thread::sleep(100);
}
srv.stop();
assertTrue (CopyToStringConnection::data() == fileData.substr(offset, count));
}
void SecureStreamSocketTest::setUp()
{
}
@@ -217,6 +383,9 @@ CppUnit::Test* SecureStreamSocketTest::suite()
CppUnit_addTest(pSuite, SecureStreamSocketTest, testSendReceive);
CppUnit_addTest(pSuite, SecureStreamSocketTest, testPeek);
CppUnit_addTest(pSuite, SecureStreamSocketTest, testNB);
CppUnit_addTest(pSuite, SecureStreamSocketTest, testSendFile);
CppUnit_addTest(pSuite, SecureStreamSocketTest, testSendFileLarge);
CppUnit_addTest(pSuite, SecureStreamSocketTest, testSendFileRange);
return pSuite;
}

View File

@@ -27,6 +27,9 @@ public:
void testSendReceive();
void testPeek();
void testNB();
void testSendFile();
void testSendFileLarge();
void testSendFileRange();
void setUp();
void tearDown();