IOCP model(Chat Server) - Code

SockIOCP.h

#pragma once
#include <WinSock2.h>
#pragma comment(lib,"ws2_32")

DWORD WINAPI ThreadCompletion(LPVOID pParam);

typedef struct _USERSESSION{
    SOCKET hSocket;    
    SOCKADDR_IN clientAddr;
}USERSESSION;

typedef struct _PERIODATA{
    OVERLAPPED ov;
    WSABUF wsabuf;
    char buffer[1024];
    bool isRecv;
}PERIODATA;

class SockIOCP
{
public:
    SockIOCP(void);
    ~SockIOCP(void);

public:
    HANDLE m_hIOCP;
    int m_nThreadCount;
    SOCKET m_hServerSocket;
    std::list<SOCKET> m_clientSocket_list;
    std::list<SOCKET>::iterator m_it;
    HANDLE m_hEvent;    

public:
    BOOL InitializeWinsock(void);
    BOOL CreateIOCP(void);
    BOOL CreateWorkerThreads(void);
    BOOL CreateServerSocket(void);
    BOOL BindServerSocket(int nPortnum);
    BOOL ChangeToListenStatus(void);
    BOOL AcceptClient(void);
    BOOL CleanupWinsock(void);
    void ReleaseServerSocket(void);
    void ReleaseClientSocket(SOCKET);
    void CloseServer(void);    
};

SockIOCP.cpp

#include "StdAfx.h"
#include "SockIOCP.h"


SockIOCP::SockIOCP(void)
{
    m_hServerSocket = INVALID_SOCKET;
    m_hEvent = NULL;
}


SockIOCP::~SockIOCP(void)
{
    CloseServer();
}

BOOL SockIOCP::InitializeWinsock(void)
{
    WSADATA wsa = {0};
    if(::WSAStartup(MAKEWORD(2,2), &wsa) != 0)
    {
        LOGOUT(LOG_ERROR, _T("ERROR: Winsock initialize failed"));
        return FALSE;
    }
    LOGOUT(LOG_DEBUG, _T("Winsock initialize success"));
    return TRUE;
}

BOOL SockIOCP::CreateIOCP(void)
{
    m_hIOCP = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if(m_hIOCP == NULL)
    {
        LOGOUT(LOG_ERROR, _T("ERROR: Failed to create IOCP"));
        return FALSE;
    }
    LOGOUT(LOG_DEBUG, _T("Create IOCP success"));
    return TRUE;
}

BOOL SockIOCP::CreateServerSocket(void)
{
    m_hServerSocket = ::WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
    if(m_hServerSocket == INVALID_SOCKET)
    {
        LOGOUT(LOG_ERROR, _T("ERROR: Failed to create serversocket"));
        return FALSE;
    }
    LOGOUT(LOG_DEBUG, _T("Create serversocket success"));
    return TRUE;
}

BOOL SockIOCP::CreateWorkerThreads(void)
{
    SYSTEM_INFO sysinfo;
    GetSystemInfo(&sysinfo);
    m_nThreadCount = sysinfo.dwNumberOfProcessors*2;

    m_hEvent = ::CreateEvent(NULL,FALSE,FALSE,NULL);    
    for(int i=0; i<m_nThreadCount; i++)    {
        CreateThread(NULL, 0, ThreadCompletion, (LPVOID)this, 0, NULL);
    }

    return TRUE;
}

BOOL SockIOCP::BindServerSocket(int nPortnum)
{
    SOCKADDR_IN addrsvr = {0};
    addrsvr.sin_family = AF_INET;
    addrsvr.sin_port = htons(nPortnum);
    addrsvr.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
    if(::bind(m_hServerSocket, (SOCKADDR*)&addrsvr, sizeof(addrsvr)) == SOCKET_ERROR)
    {
        LOGOUT(LOG_ERROR, _T("ERROR: Failed to bind serversocket"));
        ReleaseServerSocket();
        return FALSE;
    }
    return TRUE;
}

BOOL SockIOCP::ChangeToListenStatus(void)
{
    if(::listen(m_hServerSocket, SOMAXCONN) == SOCKET_ERROR)
    {
        LOGOUT(LOG_ERROR, _T("ERROR: Failed to change to listenstatus"));
        ReleaseServerSocket();
        return FALSE;
    }
    return TRUE;
}

BOOL SockIOCP::AcceptClient(void)
{
    LOGOUT(LOG_DEBUG, _T("Start server"));
    SOCKADDR_IN clientaddr;
    int nAddrLen = sizeof(SOCKADDR);
    USERSESSION* pUserSession;
    PERIODATA* pPerIOData;

    SOCKET hClientSocket = NULL;
    while((hClientSocket = ::accept(m_hServerSocket, (SOCKADDR*)&clientaddr, &nAddrLen)) != INVALID_SOCKET)
    {
        LOGOUT(LOG_DEBUG, _T("Client connected"));
        m_clientSocket_list.push_back(hClientSocket);
        char* inetntoa = inet_ntoa(clientaddr.sin_addr);

        LOGOUT(LOG_DEBUG, _T("Accept() IP: %s Port: %d\n"),
            inetntoa, ntohs(clientaddr.sin_port));

        pUserSession = new USERSESSION;
        memset(pUserSession, 0, sizeof(USERSESSION));
        pUserSession->hSocket = hClientSocket;
        memcpy(&(pUserSession->clientAddr), &clientaddr, nAddrLen);

        LOGOUT(LOG_DEBUG, _T("A New user logged in.  IP: %s, Port: %d\n"),
            inet_ntoa(pUserSession->clientAddr.sin_addr), ntohs(pUserSession->clientAddr.sin_port));

        if (::CreateIoCompletionPort((HANDLE)hClientSocket, m_hIOCP, (ULONG_PTR)pUserSession, 0) == NULL) {
            LOGOUT(LOG_ERROR, _T("Error: CreateIoCompletionPort() errCd:%d"), WSAGetLastError());
            return FALSE;
        }

        pPerIOData = new PERIODATA;
        memset(&pPerIOData->ov, 0, sizeof(OVERLAPPED));
        pPerIOData->wsabuf.buf = pPerIOData->buffer;
        pPerIOData->wsabuf.len = sizeof(pPerIOData->buffer);
        pPerIOData->isRecv = true;
        DWORD dwReceiveSize = 0, dwFlag = 0;
        if (::WSARecv(pUserSession->hSocket, &(pPerIOData->wsabuf), 1, &dwReceiveSize, &dwFlag, &(pPerIOData->ov), NULL) == SOCKET_ERROR) {
            if (::WSAGetLastError() != WSA_IO_PENDING)//WSA_IO_PENDING과 ERROR_IO_PENDING아무거나 사용해도 된다.
            {
                LOGOUT(LOG_ERROR, _T("ERROR: WSARecv() %d"), GetLastError()); //GetLastError()는 에러코드를 리턴한다.(숫자)
                CloseServer();
                return FALSE;
            }
        }

    }

    return TRUE;
}


DWORD WINAPI ThreadCompletion(LPVOID pParam)
{
    SockIOCP* pSockIOCP = (SockIOCP*)pParam;

    USERSESSION* pUserSession = NULL;
    PERIODATA* pPerIOData = NULL;
    DATATRANSFER* pDataTransfer = NULL;
    DWORD dwTransferredSize=0, dwFlag=0; 
    while(TRUE)
    {
        BOOL bRet = ::GetQueuedCompletionStatus(pSockIOCP->m_hIOCP, &dwTransferredSize, (PULONG_PTR)&pUserSession, (LPOVERLAPPED*)&pPerIOData, INFINITE);
        if (bRet == FALSE || dwTransferredSize == 0)
        {
            if (pUserSession == NULL) {//스레드를 종료하기 위해 의도적으로 completion key에 null이 입력된 경우..
                LOGOUT(LOG_INFO, _T("Thread terminated on purpose. pSession = NULL"));//나머지 소켓 닫기 처리는 ReleaseClientSocket()가 알아서 하므로 스레드만 종료.
                break;
            }
            else {//비정상 종료인 경우...
                LOGOUT(LOG_DEBUG, _T("Unexpected disconnection with client"));
                pSockIOCP->ReleaseClientSocket(pUserSession->hSocket);
                continue;
            }
        }

        LOGOUT(LOG_DEBUG, _T("ThreadCompletion pUserSession. IP: %s, Port: %d\n"),
            inet_ntoa(pUserSession->clientAddr.sin_addr), ntohs(pUserSession->clientAddr.sin_port));

        if (pPerIOData->isRecv) {
            pSockIOCP->m_it = pSockIOCP->m_clientSocket_list.begin();
            while (pSockIOCP->m_it != pSockIOCP->m_clientSocket_list.end()) {
                PERIODATA* pPerIODataForSend = new PERIODATA();
                ZeroMemory(&(pPerIODataForSend->ov), sizeof(OVERLAPPED));
                memset(pPerIODataForSend, 0, sizeof(PERIODATA));
                //_sntprintf(pPerIODataForSend->buffer, _countof(pPerIODataForSend->buffer), _T("IP: %s Port: %d //msg: %s"),
                //    inet_ntoa(pUserSession->clientAddr.sin_addr), ntohs(pUserSession->clientAddr.sin_port), pPerIOData->buffer);
                memcpy(pPerIODataForSend->buffer, pPerIOData->buffer, dwTransferredSize);
                pPerIODataForSend->wsabuf.buf = pPerIODataForSend->buffer;
                pPerIODataForSend->wsabuf.len = dwTransferredSize;
                pPerIODataForSend->isRecv = FALSE;

                DWORD dwWrittenForSend = 0;
                int nRet = ::WSASend(*(pSockIOCP->m_it), &(pPerIODataForSend->wsabuf), 1, &dwWrittenForSend, 0, &pPerIODataForSend->ov, NULL);
                if (nRet == SOCKET_ERROR && WSAGetLastError() != WSA_IO_PENDING) {
                    if (pUserSession != NULL) {
                        if (*pSockIOCP->m_it != NULL) {
                            closesocket(*pSockIOCP->m_it);
                            pSockIOCP->m_it = pSockIOCP->m_clientSocket_list.erase(pSockIOCP->m_it);
                        }
                    }
                    continue;
                }
                else {
                    pSockIOCP->m_it++;
                }
            }

            memset(&(pPerIOData->ov), 0, sizeof(OVERLAPPED));
            memset(pPerIOData, 0, sizeof(PERIODATA));
            pPerIOData->wsabuf.buf = pPerIOData->buffer;
            pPerIOData->wsabuf.len = sizeof(pPerIOData->buffer);
            pPerIOData->isRecv = true;
            DWORD dwReceiveSize = 0, dwFlag = 0;
            ::WSARecv(pUserSession->hSocket, &(pPerIOData->wsabuf), 1, &dwReceiveSize, &dwFlag, &(pPerIOData->ov), NULL);
            continue;
        }
        else {

            LOGOUT(LOG_DEBUG, _T("WSASend() responsed. IP: %s, Port: %d\n"), 
                inet_ntoa(pUserSession->clientAddr.sin_addr), ntohs(pUserSession->clientAddr.sin_port));

            if (pPerIOData != NULL) {
                delete pPerIOData;
                pPerIOData = NULL;
            }
            continue;
        }
    }
    return 0;
}

void SockIOCP::ReleaseServerSocket(void)
{
    if(m_hIOCP!= NULL){
        for(int i=0; i<m_nThreadCount; i++)
        ::PostQueuedCompletionStatus(m_hIOCP, 0, NULL, NULL);
    }
    if(m_hServerSocket != NULL){
        ::shutdown(m_hServerSocket, SD_BOTH);
        ::closesocket(m_hServerSocket);
        m_hServerSocket = NULL;
    }    
}

void SockIOCP::ReleaseClientSocket(SOCKET hSocket)
{
    if(hSocket != NULL){
        ::shutdown(hSocket, SD_BOTH);
        closesocket(hSocket);
        m_clientSocket_list.remove(hSocket);
    }    
}

void SockIOCP::CloseServer(void)
{
    ReleaseServerSocket();

    for(m_it = m_clientSocket_list.begin(); m_it!= m_clientSocket_list.end(); ++m_it){
        ::shutdown(*m_it, SD_BOTH);
        ::closesocket(*m_it);
    }

    m_clientSocket_list.empty();

    if(m_hIOCP != NULL){
        ::CloseHandle(m_hIOCP);    
        m_hIOCP = NULL;        
    }

    ::SetEvent(m_hEvent);    
    LOGOUT(LOG_DEBUG, _T("Server closing finsihed"));
}

BOOL SockIOCP::CleanupWinsock()
{
    ::WaitForSingleObject(m_hEvent, INFINITE); //서버가 죽지 않도록 붙잡는 기능

    if(::WSACleanup() == SOCKET_ERROR){
        LOGOUT(LOG_ERROR, _T("ERROR: Failed to clean up winsock"));
        return false;
    }
    LOGOUT(LOG_DEBUG, _T("Clean up winsock success"));
    return TRUE;
}

IOCPExample.cpp

// IOCPExample.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include "IOCPExample.h"

#ifdef _DEBUG
#define new DEBUG_NEW
#endif


// The one and only application object

CWinApp theApp;

using namespace std;

int main()
{
    int nRetCode = 0;

    HMODULE hModule = ::GetModuleHandle(nullptr);

    if (hModule != nullptr)
    {
        // initialize MFC and print and error on failure
        if (!AfxWinInit(hModule, nullptr, ::GetCommandLine(), 0))
        {
            // TODO: change error code to suit your needs
            wprintf(L"Fatal Error: MFC initialization failed\n");
            nRetCode = 1;
        }
        else
        {
            SockIOCP* pSockIOCP = new SockIOCP();
            if (!pSockIOCP->InitializeWinsock())
                return -1;
            if (!pSockIOCP->CreateIOCP())
                return -1;
            if (!pSockIOCP->CreateWorkerThreads())
                return -1;
            if (!pSockIOCP->CreateServerSocket())
                return -1;
            if (!pSockIOCP->BindServerSocket(21000))
                return -1;
            if (!pSockIOCP->ChangeToListenStatus())
                return -1;
            if (!pSockIOCP->AcceptClient())
                return -1;
            if (!pSockIOCP->CleanupWinsock())
                return -1;
            return 0;
        }
    }
    else
    {
        // TODO: change error code to suit your needs
        wprintf(L"Fatal Error: GetModuleHandle failed\n");
        nRetCode = 1;
    }

    return nRetCode;
}